2024-04-25T00:15:25.625 INFO:root:teuthology version: 0.0.1.dev269+g9481b1d 2024-04-25T00:15:25.626 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/teuthology-2024-04-24_21:24:02-fs-squid-distro-default-smithi/7672261 --owner scheduled_teuthology@teuthology --verbose --name teuthology-2024-04-24_21:24:02-fs-squid-distro-default-smithi --description fs/workload/{0-centos_9.stream begin/{0-install 1-cephadm 2-logrotate 3-modules} clusters/1a11s-mds-1c-client-3node conf/{client mds mgr mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/stock/{centos_9.stream k-stock} ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts pg_health session_timeout} ranks/multi/{balancer/automatic export-check n/3 replication/default} standby-replay tasks/{0-subvolume/{with-no-extra-options} 1-check-counter 2-scrub/yes 3-snaps/yes 4-flush/no 5-workunit/suites/iogen}} -- /home/teuthworker/archive/teuthology-2024-04-24_21:24:02-fs-squid-distro-default-smithi/7672261/orig.config.yaml 2024-04-25T00:15:25.685 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-25T00:15:25.781 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-24_21:24:02-fs-squid-distro-default-smithi/7672261 branch: squid description: fs/workload/{0-centos_9.stream begin/{0-install 1-cephadm 2-logrotate 3-modules} clusters/1a11s-mds-1c-client-3node conf/{client mds mgr mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/stock/{centos_9.stream k-stock} ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts pg_health session_timeout} ranks/multi/{balancer/automatic export-check n/3 replication/default} standby-replay tasks/{0-subvolume/{with-no-extra-options} 1-check-counter 2-scrub/yes 3-snaps/yes 4-flush/no 5-workunit/suites/iogen}} email: ceph-qa@ceph.com first_in_suite: false job_id: '7672261' kernel: client: sha1: distro kdb: 1 sha1: distro ktype: distro last_in_suite: false log-rotate: ceph-mds: 10G ceph-osd: 10G machine_type: smithi mgrmodules: sequential: - print: Enabling mgr modules - exec: mon.a: - ceph mgr module enable snap_schedule - ceph config set mgr mgr/snap_schedule/allow_m_granularity true - ceph config set mgr mgr/snap_schedule/dump_on_update true name: teuthology-2024-04-24_21:24:02-fs-squid-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 20 - machine: disk: 200 os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: cephfs: ec_profile: - m=2 - k=2 - crush-failure-domain=osd max_mds: 3 session_timeout: 300 standby_replay: true conf: client: client mount timeout: 600 debug client: 20 debug ms: 1 rados mon op timeout: 900 rados osd op timeout: 900 global: ms die on skipped message: false mds: debug mds: 20 debug mds balancer: 20 debug ms: 1 mds bal fragment size max: 10000 mds bal merge size: 5 mds bal split bits: 3 mds bal split size: 100 mds debug frag: true mds debug scatterstat: true mds op complaint time: 180 mds verify scatter: true osd op complaint time: 180 rados mon op timeout: 900 rados osd op timeout: 900 mgr: cephsqlite lock renewal timeout: 900000 debug cephsqlite: 20 debug client: 20 debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 mon down mkfs grace: 300 mon op complaint time: 120 osd: bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 20 debug bluestore: 20 debug ms: 1 debug osd: 20 debug rocksdb: 10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore osd op complaint time: 180 osd shutdown pgref assert: true osd_max_omap_entries_per_request: 10 flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - FS_DEGRADED - FS_INLINE_DATA_DEPRECATED - FS_WITH_FAILED_MDS - MDS_ALL_DOWN - MDS_DAMAGE - MDS_DEGRADED - MDS_FAILED - MDS_INSUFFICIENT_STANDBY - MDS_UP_LESS_THAN_MAX - filesystem is online with fewer MDS than max_mds - POOL_APP_NOT_ENABLED - overall HEALTH_ - Replacing daemon - deprecated feature inline_data - overall HEALTH_ - \(OSD_DOWN\) - \(OSD_ - but it is still running - is not responding - MON_DOWN - PG_AVAILABILITY - PG_DEGRADED - Reduced data availability - slow metadata IO - SLOW_OPS - slow request sha1: d2c45d977142e024a59b68a8f1dce8f0b9840551 subvols: create: 2 ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} osd: bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 20 debug bluestore: 20 debug rocksdb: 10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs ceph-fuse: client.0: mount_subvol_num: 0 check-counter: counters: mds: - mds.exported - mds.imported - mds.root_rsnaps - mds_server.req_mksnap_latency.avgcount - mds_server.req_rmsnap_latency.avgcount dry_run: true install: ceph: flavor: default sha1: d2c45d977142e024a59b68a8f1dce8f0b9840551 kclient: client.0: mount_subvol_num: 1 mntopts: - ms_mode=crc - wsync syntax: v1 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 - scontext=system_u:system_r:getty_t:s0 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: squid sha1: d2c45d977142e024a59b68a8f1dce8f0b9840551 owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mgr.x - mds.a - mds.d - mds.g - mds.j - osd.0 - osd.3 - osd.6 - osd.9 - client.0 - - mon.b - mgr.y - mds.b - mds.e - mds.h - mds.k - osd.1 - osd.4 - osd.7 - osd.10 - - mon.c - mgr.z - mds.c - mds.f - mds.i - mds.l - osd.2 - osd.5 - osd.8 - osd.11 seed: 8528 sha1: d2c45d977142e024a59b68a8f1dce8f0b9840551 sleep_before_teardown: 0 subset: 122/512 suite: fs suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_d2c45d977142e024a59b68a8f1dce8f0b9840551/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: d2c45d977142e024a59b68a8f1dce8f0b9840551 syslog: ignorelist: - WARNING*.*check_session_state - WARNING*.*__ceph_remove_cap targets: smithi032.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBD7DrckUBqtLVOh4N21KpzV2c5DYCQ4bNPwoz1xlRWR3s5dZJe2UHzqCWVTXjf2RXCo+Uax58IeB8SHs1nwGyDQ= smithi121.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDc3ni7OjyBoF/W+5fbYLxj3kDcEq+yuFnaEXYGj35ajB7s1IGOeGWTpkBzEK3HH72BAhhtYvuA9ng2PWcevtHA= smithi130.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJEqQth7lAp0ElBkxSulfTqJDM8G2Xx2ESozNk909PMdxh3PD482xBapBNkUUkg7JMf7+qYQSeTk7wRTvFInNGw= tasks: - install: extra_packages: deb: - python3-cephfs - cephfs-shell - cephfs-top - cephfs-mirror rpm: - python3-cephfs - cephfs-shell - cephfs-top - cephfs-mirror extra_system_packages: deb: - bison - flex - libelf-dev - libssl-dev - network-manager - iproute2 - util-linux - dump - indent - libaio-dev - libtool-bin - uuid-dev - xfslibs-dev - postgresql - postgresql-client - postgresql-common - postgresql-contrib rpm: - bison - flex - elfutils-libelf-devel - openssl-devel - NetworkManager - iproute - util-linux - libacl-devel - libaio-devel - libattr-devel - libtool - libuuid-devel - xfsdump - xfsprogs - xfsprogs-devel - libaio-devel - libtool - libuuid-devel - xfsprogs-devel - postgresql - postgresql-server - postgresql-contrib - cephadm: roleless: false - cephadm.shell: mon.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - cephadm.shell: mon.a: - ceph fs dump - ceph osd dump - fs.ready: timeout: 300 - sequential: - mgrmodules - kclient: null - exec: mon.a: - ceph fs set cephfs balance_automate true - check-counter: {} - fwd_scrub: scrub_timeout: 900 sleep_between_iterations: 1 - exec: mon.a: - ceph fs snap-schedule add --fs=cephfs --path=/ --snap_schedule=1m - ceph fs snap-schedule retention add --fs=cephfs --path=/ --retention-spec-or-period=6m3h - ceph fs snap-schedule status --fs=cephfs --path=/ - ceph fs snap-schedule list --fs=cephfs --path=/ --recursive=true - date +%s > START_TIME - full_sequential_finally: - exec: mon.a: - date +%s > END_TIME - START_TIME=$(cat START_TIME); END_TIME=$(cat END_TIME); DIFF_TIME=$((600-(END_TIME-START_TIME))); if [ "$DIFF_TIME" -gt 0 ]; then sleep "$DIFF_TIME"; fi - ceph fs snap-schedule status --fs=cephfs --path=/ - ceph fs snap-schedule list --fs=cephfs --path=/ --recursive=true - workunit: clients: all: - suites/iogen.sh teuthology: fragments_dropped: [] meta: {} postmerge: - local function is_kupstream() return yaml.ktype == 'upstream' end - local function is_kdistro() return yaml.ktype == 'distro' end - local function is_fuse() return false end - local function syntax_version() return yaml.overrides.kclient.syntax end teuthology_branch: main teuthology_sha1: 9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17 timestamp: 2024-04-24_21:24:02 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-25T00:15:25.781 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_d2c45d977142e024a59b68a8f1dce8f0b9840551/qa; will attempt to use it 2024-04-25T00:15:25.782 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_d2c45d977142e024a59b68a8f1dce8f0b9840551/qa/tasks 2024-04-25T00:15:25.782 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-25T00:15:25.784 INFO:teuthology.task.internal:Checking packages... 2024-04-25T00:15:25.807 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'd2c45d977142e024a59b68a8f1dce8f0b9840551' 2024-04-25T00:15:25.807 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-25T00:15:25.807 INFO:teuthology.packaging:ref: None 2024-04-25T00:15:25.807 INFO:teuthology.packaging:tag: None 2024-04-25T00:15:25.808 INFO:teuthology.packaging:branch: squid 2024-04-25T00:15:25.808 INFO:teuthology.packaging:sha1: d2c45d977142e024a59b68a8f1dce8f0b9840551 2024-04-25T00:15:25.808 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-04-25T00:15:26.022 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2445.gd2c45d97 2024-04-25T00:15:26.023 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-25T00:15:26.030 INFO:teuthology.task.internal:no buildpackages task found 2024-04-25T00:15:26.030 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-25T00:15:26.046 INFO:teuthology.task.internal:Saving configuration 2024-04-25T00:15:26.068 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-25T00:15:26.075 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-25T00:15:26.101 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi032.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-24_21:24:02-fs-squid-distro-default-smithi/7672261', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-25 00:10:23.147174', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBD7DrckUBqtLVOh4N21KpzV2c5DYCQ4bNPwoz1xlRWR3s5dZJe2UHzqCWVTXjf2RXCo+Uax58IeB8SHs1nwGyDQ='} 2024-04-25T00:15:26.124 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi121.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-24_21:24:02-fs-squid-distro-default-smithi/7672261', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-25 00:10:23.149934', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDc3ni7OjyBoF/W+5fbYLxj3kDcEq+yuFnaEXYGj35ajB7s1IGOeGWTpkBzEK3HH72BAhhtYvuA9ng2PWcevtHA='} 2024-04-25T00:15:26.149 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi130.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-24_21:24:02-fs-squid-distro-default-smithi/7672261', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-25 00:10:23.148903', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJEqQth7lAp0ElBkxSulfTqJDM8G2Xx2ESozNk909PMdxh3PD482xBapBNkUUkg7JMf7+qYQSeTk7wRTvFInNGw='} 2024-04-25T00:15:26.149 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-25T00:15:26.155 INFO:teuthology.task.internal:roles: ubuntu@smithi032.front.sepia.ceph.com - ['mon.a', 'mgr.x', 'mds.a', 'mds.d', 'mds.g', 'mds.j', 'osd.0', 'osd.3', 'osd.6', 'osd.9', 'client.0'] 2024-04-25T00:15:26.155 INFO:teuthology.task.internal:roles: ubuntu@smithi121.front.sepia.ceph.com - ['mon.b', 'mgr.y', 'mds.b', 'mds.e', 'mds.h', 'mds.k', 'osd.1', 'osd.4', 'osd.7', 'osd.10'] 2024-04-25T00:15:26.155 INFO:teuthology.task.internal:roles: ubuntu@smithi130.front.sepia.ceph.com - ['mon.c', 'mgr.z', 'mds.c', 'mds.f', 'mds.i', 'mds.l', 'osd.2', 'osd.5', 'osd.8', 'osd.11'] 2024-04-25T00:15:26.155 INFO:teuthology.run_tasks:Running task console_log... 2024-04-25T00:15:26.265 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f427f203160>, signals=[15]) 2024-04-25T00:15:26.266 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-25T00:15:26.272 INFO:teuthology.task.internal:Opening connections... 2024-04-25T00:15:26.272 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi032.front.sepia.ceph.com 2024-04-25T00:15:26.274 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi032.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-25T00:15:26.353 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi121.front.sepia.ceph.com 2024-04-25T00:15:26.354 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi121.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-25T00:15:26.430 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi130.front.sepia.ceph.com 2024-04-25T00:15:26.431 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi130.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-25T00:15:26.504 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-25T00:15:26.513 DEBUG:teuthology.orchestra.run.smithi032:> uname -m 2024-04-25T00:15:26.532 INFO:teuthology.orchestra.run.smithi032.stdout:x86_64 2024-04-25T00:15:26.532 DEBUG:teuthology.orchestra.run.smithi032:> cat /etc/os-release 2024-04-25T00:15:26.590 INFO:teuthology.orchestra.run.smithi032.stdout:NAME="CentOS Stream" 2024-04-25T00:15:26.590 INFO:teuthology.orchestra.run.smithi032.stdout:VERSION="9" 2024-04-25T00:15:26.590 INFO:teuthology.orchestra.run.smithi032.stdout:ID="centos" 2024-04-25T00:15:26.590 INFO:teuthology.orchestra.run.smithi032.stdout:ID_LIKE="rhel fedora" 2024-04-25T00:15:26.590 INFO:teuthology.orchestra.run.smithi032.stdout:VERSION_ID="9" 2024-04-25T00:15:26.590 INFO:teuthology.orchestra.run.smithi032.stdout:PLATFORM_ID="platform:el9" 2024-04-25T00:15:26.590 INFO:teuthology.orchestra.run.smithi032.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-25T00:15:26.591 INFO:teuthology.orchestra.run.smithi032.stdout:ANSI_COLOR="0;31" 2024-04-25T00:15:26.591 INFO:teuthology.orchestra.run.smithi032.stdout:LOGO="fedora-logo-icon" 2024-04-25T00:15:26.591 INFO:teuthology.orchestra.run.smithi032.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-25T00:15:26.591 INFO:teuthology.orchestra.run.smithi032.stdout:HOME_URL="https://centos.org/" 2024-04-25T00:15:26.591 INFO:teuthology.orchestra.run.smithi032.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-25T00:15:26.591 INFO:teuthology.orchestra.run.smithi032.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-25T00:15:26.591 INFO:teuthology.orchestra.run.smithi032.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-25T00:15:26.591 INFO:teuthology.lock.ops:Updating smithi032.front.sepia.ceph.com on lock server 2024-04-25T00:15:26.615 DEBUG:teuthology.orchestra.run.smithi121:> uname -m 2024-04-25T00:15:26.633 INFO:teuthology.orchestra.run.smithi121.stdout:x86_64 2024-04-25T00:15:26.633 DEBUG:teuthology.orchestra.run.smithi121:> cat /etc/os-release 2024-04-25T00:15:26.689 INFO:teuthology.orchestra.run.smithi121.stdout:NAME="CentOS Stream" 2024-04-25T00:15:26.690 INFO:teuthology.orchestra.run.smithi121.stdout:VERSION="9" 2024-04-25T00:15:26.690 INFO:teuthology.orchestra.run.smithi121.stdout:ID="centos" 2024-04-25T00:15:26.690 INFO:teuthology.orchestra.run.smithi121.stdout:ID_LIKE="rhel fedora" 2024-04-25T00:15:26.690 INFO:teuthology.orchestra.run.smithi121.stdout:VERSION_ID="9" 2024-04-25T00:15:26.690 INFO:teuthology.orchestra.run.smithi121.stdout:PLATFORM_ID="platform:el9" 2024-04-25T00:15:26.690 INFO:teuthology.orchestra.run.smithi121.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-25T00:15:26.690 INFO:teuthology.orchestra.run.smithi121.stdout:ANSI_COLOR="0;31" 2024-04-25T00:15:26.690 INFO:teuthology.orchestra.run.smithi121.stdout:LOGO="fedora-logo-icon" 2024-04-25T00:15:26.690 INFO:teuthology.orchestra.run.smithi121.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-25T00:15:26.690 INFO:teuthology.orchestra.run.smithi121.stdout:HOME_URL="https://centos.org/" 2024-04-25T00:15:26.690 INFO:teuthology.orchestra.run.smithi121.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-25T00:15:26.690 INFO:teuthology.orchestra.run.smithi121.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-25T00:15:26.691 INFO:teuthology.orchestra.run.smithi121.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-25T00:15:26.691 INFO:teuthology.lock.ops:Updating smithi121.front.sepia.ceph.com on lock server 2024-04-25T00:15:26.707 DEBUG:teuthology.orchestra.run.smithi130:> uname -m 2024-04-25T00:15:26.726 INFO:teuthology.orchestra.run.smithi130.stdout:x86_64 2024-04-25T00:15:26.726 DEBUG:teuthology.orchestra.run.smithi130:> cat /etc/os-release 2024-04-25T00:15:26.783 INFO:teuthology.orchestra.run.smithi130.stdout:NAME="CentOS Stream" 2024-04-25T00:15:26.783 INFO:teuthology.orchestra.run.smithi130.stdout:VERSION="9" 2024-04-25T00:15:26.783 INFO:teuthology.orchestra.run.smithi130.stdout:ID="centos" 2024-04-25T00:15:26.784 INFO:teuthology.orchestra.run.smithi130.stdout:ID_LIKE="rhel fedora" 2024-04-25T00:15:26.784 INFO:teuthology.orchestra.run.smithi130.stdout:VERSION_ID="9" 2024-04-25T00:15:26.784 INFO:teuthology.orchestra.run.smithi130.stdout:PLATFORM_ID="platform:el9" 2024-04-25T00:15:26.784 INFO:teuthology.orchestra.run.smithi130.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-25T00:15:26.784 INFO:teuthology.orchestra.run.smithi130.stdout:ANSI_COLOR="0;31" 2024-04-25T00:15:26.784 INFO:teuthology.orchestra.run.smithi130.stdout:LOGO="fedora-logo-icon" 2024-04-25T00:15:26.784 INFO:teuthology.orchestra.run.smithi130.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-25T00:15:26.784 INFO:teuthology.orchestra.run.smithi130.stdout:HOME_URL="https://centos.org/" 2024-04-25T00:15:26.784 INFO:teuthology.orchestra.run.smithi130.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-25T00:15:26.784 INFO:teuthology.orchestra.run.smithi130.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-25T00:15:26.784 INFO:teuthology.orchestra.run.smithi130.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-25T00:15:26.785 INFO:teuthology.lock.ops:Updating smithi130.front.sepia.ceph.com on lock server 2024-04-25T00:15:26.806 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-25T00:15:26.818 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-25T00:15:26.824 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-25T00:15:26.825 DEBUG:teuthology.orchestra.run.smithi032:> test '!' -e /home/ubuntu/cephtest 2024-04-25T00:15:26.828 DEBUG:teuthology.orchestra.run.smithi121:> test '!' -e /home/ubuntu/cephtest 2024-04-25T00:15:26.831 DEBUG:teuthology.orchestra.run.smithi130:> test '!' -e /home/ubuntu/cephtest 2024-04-25T00:15:26.847 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-25T00:15:26.853 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-25T00:15:26.853 DEBUG:teuthology.orchestra.run.smithi032:> test -z $(ls -A /var/lib/ceph) 2024-04-25T00:15:26.885 DEBUG:teuthology.orchestra.run.smithi121:> test -z $(ls -A /var/lib/ceph) 2024-04-25T00:15:26.887 DEBUG:teuthology.orchestra.run.smithi130:> test -z $(ls -A /var/lib/ceph) 2024-04-25T00:15:26.994 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-25T00:15:27.068 INFO:teuthology.run_tasks:Running task kernel... 2024-04-25T00:15:27.084 INFO:teuthology.task.kernel:normalize config orig: {'client': {'sha1': 'distro'}, 'kdb': 1, 'sha1': 'distro'} 2024-04-25T00:15:27.084 INFO:teuthology.task.kernel:normalize config final: {'client.0': {'sha1': 'distro'}} 2024-04-25T00:15:27.084 INFO:teuthology.task.kernel:config {'client.0': {'sha1': 'distro'}}, timeout 300 2024-04-25T00:15:27.084 DEBUG:teuthology.orchestra.run.smithi032:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-25T00:15:27.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-25T00:15:27.102 DEBUG:teuthology.orchestra.run.smithi032:> uname -r 2024-04-25T00:15:27.160 INFO:teuthology.orchestra.run.smithi032.stdout:5.14.0-437.el9.x86_64 2024-04-25T00:15:27.160 INFO:teuthology.task.kernel:Running kernel on smithi032: 5.14.0-437.el9.x86_64 2024-04-25T00:15:27.160 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum install -y kernel 2024-04-25T00:15:29.965 INFO:teuthology.orchestra.run.smithi032.stdout:CentOS Stream 9 - BaseOS 6.8 MB/s | 8.0 MB 00:01 2024-04-25T00:15:31.858 INFO:teuthology.orchestra.run.smithi032.stdout:CentOS Stream 9 - AppStream 25 MB/s | 19 MB 00:00 2024-04-25T00:15:36.259 INFO:teuthology.orchestra.run.smithi032.stdout:CentOS Stream 9 - CRB 8.5 MB/s | 6.2 MB 00:00 2024-04-25T00:15:37.968 INFO:teuthology.orchestra.run.smithi032.stdout:CentOS Stream 9 - Extras packages 32 kB/s | 16 kB 00:00 2024-04-25T00:15:39.661 INFO:teuthology.orchestra.run.smithi032.stdout:Extra Packages for Enterprise Linux 13 MB/s | 21 MB 00:01 2024-04-25T00:15:44.914 INFO:teuthology.orchestra.run.smithi032.stdout:lab-extras 25 kB/s | 1.7 kB 00:00 2024-04-25T00:15:45.107 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:00:01 ago on Thu 25 Apr 2024 12:15:44 AM UTC. 2024-04-25T00:15:46.876 INFO:teuthology.orchestra.run.smithi032.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-25T00:15:46.876 INFO:teuthology.orchestra.run.smithi032.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-25T00:15:46.972 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-04-25T00:15:46.973 INFO:teuthology.orchestra.run.smithi032.stdout:Nothing to do. 2024-04-25T00:15:46.973 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-04-25T00:15:47.072 DEBUG:teuthology.orchestra.run.smithi032:> echo no | sudo yum reinstall kernel || true 2024-04-25T00:15:47.595 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:00:03 ago on Thu 25 Apr 2024 12:15:44 AM UTC. 2024-04-25T00:15:47.719 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-04-25T00:15:47.732 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-04-25T00:15:47.732 INFO:teuthology.orchestra.run.smithi032.stdout: Package Architecture Version Repository Size 2024-04-25T00:15:47.732 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-04-25T00:15:47.732 INFO:teuthology.orchestra.run.smithi032.stdout:Reinstalling: 2024-04-25T00:15:47.733 INFO:teuthology.orchestra.run.smithi032.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-25T00:15:47.733 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:15:47.733 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction Summary 2024-04-25T00:15:47.733 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-04-25T00:15:47.733 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:15:47.733 INFO:teuthology.orchestra.run.smithi032.stdout:Total download size: 255 k 2024-04-25T00:15:47.734 INFO:teuthology.orchestra.run.smithi032.stdout:Installed size: 0 2024-04-25T00:15:47.734 INFO:teuthology.orchestra.run.smithi032.stderr:Operation aborted. 2024-04-25T00:15:47.792 INFO:teuthology.orchestra.run.smithi032.stdout:Is this ok [y/N]: 2024-04-25T00:15:47.792 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum reinstall -y kernel || true 2024-04-25T00:15:48.311 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:00:04 ago on Thu 25 Apr 2024 12:15:44 AM UTC. 2024-04-25T00:15:48.432 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-04-25T00:15:48.433 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-04-25T00:15:48.433 INFO:teuthology.orchestra.run.smithi032.stdout: Package Architecture Version Repository Size 2024-04-25T00:15:48.433 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-04-25T00:15:48.433 INFO:teuthology.orchestra.run.smithi032.stdout:Reinstalling: 2024-04-25T00:15:48.433 INFO:teuthology.orchestra.run.smithi032.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-25T00:15:48.433 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:15:48.433 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction Summary 2024-04-25T00:15:48.433 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-04-25T00:15:48.433 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:15:48.433 INFO:teuthology.orchestra.run.smithi032.stdout:Total download size: 255 k 2024-04-25T00:15:48.433 INFO:teuthology.orchestra.run.smithi032.stdout:Installed size: 0 2024-04-25T00:15:48.433 INFO:teuthology.orchestra.run.smithi032.stdout:Downloading Packages: 2024-04-25T00:15:48.697 INFO:teuthology.orchestra.run.smithi032.stdout:kernel-5.14.0-437.el9.x86_64.rpm 1.8 MB/s | 255 kB 00:00 2024-04-25T00:15:48.697 INFO:teuthology.orchestra.run.smithi032.stdout:-------------------------------------------------------------------------------- 2024-04-25T00:15:48.697 INFO:teuthology.orchestra.run.smithi032.stdout:Total 967 kB/s | 255 kB 00:00 2024-04-25T00:15:48.715 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction check 2024-04-25T00:15:48.906 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction check succeeded. 2024-04-25T00:15:48.906 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction test 2024-04-25T00:15:48.911 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction test succeeded. 2024-04-25T00:15:48.911 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction 2024-04-25T00:15:49.058 INFO:teuthology.orchestra.run.smithi032.stdout: Preparing : 1/1 2024-04-25T00:15:49.114 INFO:teuthology.orchestra.run.smithi032.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-25T00:15:49.115 INFO:teuthology.orchestra.run.smithi032.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-25T00:15:49.796 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-25T00:15:49.796 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-25T00:15:50.521 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-25T00:15:50.521 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:15:50.521 INFO:teuthology.orchestra.run.smithi032.stdout:Reinstalled: 2024-04-25T00:15:50.521 INFO:teuthology.orchestra.run.smithi032.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-25T00:15:50.521 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:15:50.521 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-04-25T00:15:50.719 DEBUG:teuthology.orchestra.run.smithi032:> rpm -q kernel | sort -rV | head -n 1 2024-04-25T00:15:50.768 INFO:teuthology.orchestra.run.smithi032.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-25T00:15:50.768 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-25T00:15:50.768 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-25T00:15:50.768 DEBUG:teuthology.parallel:result is None 2024-04-25T00:15:50.769 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-25T00:15:50.776 INFO:teuthology.task.internal:Creating test directory... 2024-04-25T00:15:50.776 DEBUG:teuthology.orchestra.run.smithi032:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-25T00:15:50.779 DEBUG:teuthology.orchestra.run.smithi121:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-25T00:15:50.781 DEBUG:teuthology.orchestra.run.smithi130:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-25T00:15:50.799 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-25T00:15:50.806 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-25T00:15:50.813 INFO:teuthology.task.internal:Creating archive directory... 2024-04-25T00:15:50.814 DEBUG:teuthology.orchestra.run.smithi032:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-25T00:15:50.834 DEBUG:teuthology.orchestra.run.smithi121:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-25T00:15:50.839 DEBUG:teuthology.orchestra.run.smithi130:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-25T00:15:50.942 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-25T00:15:50.949 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-25T00:15:50.949 DEBUG:teuthology.orchestra.run.smithi121:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-25T00:15:50.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-25T00:15:50.967 DEBUG:teuthology.orchestra.run.smithi130:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-25T00:15:50.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-25T00:15:50.984 DEBUG:teuthology.orchestra.run.smithi032:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-25T00:15:50.986 DEBUG:teuthology.orchestra.run.smithi121:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-25T00:15:51.009 DEBUG:teuthology.orchestra.run.smithi130:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-25T00:15:51.014 INFO:teuthology.orchestra.run.smithi032.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-25T00:15:51.034 INFO:teuthology.orchestra.run.smithi032.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-25T00:15:51.052 INFO:teuthology.orchestra.run.smithi121.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-25T00:15:51.069 INFO:teuthology.orchestra.run.smithi130.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-25T00:15:51.078 INFO:teuthology.orchestra.run.smithi121.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-25T00:15:51.115 INFO:teuthology.orchestra.run.smithi130.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-25T00:15:51.117 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-25T00:15:51.124 INFO:teuthology.task.internal:Configuring sudo... 2024-04-25T00:15:51.124 DEBUG:teuthology.orchestra.run.smithi032:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-25T00:15:51.127 DEBUG:teuthology.orchestra.run.smithi121:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-25T00:15:51.129 DEBUG:teuthology.orchestra.run.smithi130:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-25T00:15:51.156 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-25T00:15:51.164 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-25T00:15:51.165 DEBUG:teuthology.orchestra.run.smithi032:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-25T00:15:51.191 DEBUG:teuthology.orchestra.run.smithi121:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-25T00:15:51.194 DEBUG:teuthology.orchestra.run.smithi130:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-25T00:15:51.212 DEBUG:teuthology.orchestra.run.smithi032:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-25T00:15:51.289 DEBUG:teuthology.orchestra.run.smithi032:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-25T00:15:51.341 DEBUG:teuthology.orchestra.run.smithi032:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-25T00:15:51.424 DEBUG:teuthology.orchestra.run.smithi032:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-25T00:15:51.462 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-04-25T00:15:51.462 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-25T00:15:51.526 DEBUG:teuthology.orchestra.run.smithi121:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-25T00:15:51.568 DEBUG:teuthology.orchestra.run.smithi121:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-25T00:15:51.617 DEBUG:teuthology.orchestra.run.smithi121:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-25T00:15:51.702 DEBUG:teuthology.orchestra.run.smithi121:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-25T00:15:51.740 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-04-25T00:15:51.741 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-25T00:15:51.806 DEBUG:teuthology.orchestra.run.smithi130:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-25T00:15:51.849 DEBUG:teuthology.orchestra.run.smithi130:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-25T00:15:51.908 DEBUG:teuthology.orchestra.run.smithi130:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-25T00:15:51.993 DEBUG:teuthology.orchestra.run.smithi130:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-25T00:15:52.034 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-04-25T00:15:52.035 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-25T00:15:52.101 DEBUG:teuthology.orchestra.run.smithi032:> sudo service rsyslog restart 2024-04-25T00:15:52.103 DEBUG:teuthology.orchestra.run.smithi121:> sudo service rsyslog restart 2024-04-25T00:15:52.106 DEBUG:teuthology.orchestra.run.smithi130:> sudo service rsyslog restart 2024-04-25T00:15:52.134 INFO:teuthology.orchestra.run.smithi121.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-25T00:15:52.136 INFO:teuthology.orchestra.run.smithi032.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-25T00:15:52.177 INFO:teuthology.orchestra.run.smithi130.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-25T00:15:52.677 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-25T00:15:52.686 INFO:teuthology.task.internal:Starting timer... 2024-04-25T00:15:52.686 INFO:teuthology.run_tasks:Running task pcp... 2024-04-25T00:15:52.700 INFO:teuthology.run_tasks:Running task selinux... 2024-04-25T00:15:52.710 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-04-25T00:15:52.711 DEBUG:teuthology.orchestra.run.smithi032:> sudo service auditd rotate 2024-04-25T00:15:52.785 INFO:teuthology.orchestra.run.smithi032.stdout:Rotating logs: 2024-04-25T00:15:52.787 DEBUG:teuthology.orchestra.run.smithi121:> sudo service auditd rotate 2024-04-25T00:15:52.837 INFO:teuthology.orchestra.run.smithi121.stdout:Rotating logs: 2024-04-25T00:15:52.839 DEBUG:teuthology.orchestra.run.smithi130:> sudo service auditd rotate 2024-04-25T00:15:52.906 INFO:teuthology.orchestra.run.smithi130.stdout:Rotating logs: 2024-04-25T00:15:52.908 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-25T00:15:52.908 DEBUG:teuthology.orchestra.run.smithi032:> /usr/sbin/getenforce 2024-04-25T00:15:52.934 INFO:teuthology.orchestra.run.smithi032.stdout:Permissive 2024-04-25T00:15:52.934 DEBUG:teuthology.orchestra.run.smithi121:> /usr/sbin/getenforce 2024-04-25T00:15:52.957 INFO:teuthology.orchestra.run.smithi121.stdout:Permissive 2024-04-25T00:15:52.957 DEBUG:teuthology.orchestra.run.smithi130:> /usr/sbin/getenforce 2024-04-25T00:15:52.982 INFO:teuthology.orchestra.run.smithi130.stdout:Permissive 2024-04-25T00:15:52.982 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi032.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi121.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi130.front.sepia.ceph.com': 'permissive'} 2024-04-25T00:15:52.983 DEBUG:teuthology.orchestra.run.smithi032:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-25T00:15:53.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-25T00:15:53.008 DEBUG:teuthology.orchestra.run.smithi121:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-25T00:15:53.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-25T00:15:53.035 DEBUG:teuthology.orchestra.run.smithi130:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-25T00:15:53.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-25T00:15:53.063 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-25T00:15:53.063 DEBUG:teuthology.orchestra.run.smithi032:> sudo /usr/sbin/setenforce permissive 2024-04-25T00:15:53.088 DEBUG:teuthology.orchestra.run.smithi121:> sudo /usr/sbin/setenforce permissive 2024-04-25T00:15:53.118 DEBUG:teuthology.orchestra.run.smithi130:> sudo /usr/sbin/setenforce permissive 2024-04-25T00:15:53.149 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-25T00:15:53.158 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-25T00:15:53.179 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-25T00:15:53.180 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi032.front.sepia.ceph.com,smithi121.front.sepia.ceph.com,smithi130.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-25T00:19:51.040 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi032.front.sepia.ceph.com'), Remote(name='ubuntu@smithi121.front.sepia.ceph.com'), Remote(name='ubuntu@smithi130.front.sepia.ceph.com')] 2024-04-25T00:19:51.042 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi032.front.sepia.ceph.com' 2024-04-25T00:19:51.043 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi032.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-25T00:19:51.118 DEBUG:teuthology.orchestra.run.smithi032:> true 2024-04-25T00:19:51.206 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi032.front.sepia.ceph.com' 2024-04-25T00:19:51.207 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi121.front.sepia.ceph.com' 2024-04-25T00:19:51.207 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi121.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-25T00:19:51.282 DEBUG:teuthology.orchestra.run.smithi121:> true 2024-04-25T00:19:51.362 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi121.front.sepia.ceph.com' 2024-04-25T00:19:51.363 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi130.front.sepia.ceph.com' 2024-04-25T00:19:51.363 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi130.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-25T00:19:51.435 DEBUG:teuthology.orchestra.run.smithi130:> true 2024-04-25T00:19:51.517 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi130.front.sepia.ceph.com' 2024-04-25T00:19:51.518 INFO:teuthology.run_tasks:Running task clock... 2024-04-25T00:19:51.530 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-25T00:19:51.531 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-25T00:19:51.531 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-25T00:19:51.533 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-25T00:19:51.533 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-25T00:19:51.536 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-25T00:19:51.536 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-25T00:19:51.563 INFO:teuthology.orchestra.run.smithi032.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-25T00:19:51.564 INFO:teuthology.orchestra.run.smithi121.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-25T00:19:51.578 INFO:teuthology.orchestra.run.smithi032.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-25T00:19:51.579 INFO:teuthology.orchestra.run.smithi121.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-25T00:19:51.585 INFO:teuthology.orchestra.run.smithi130.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-25T00:19:51.599 INFO:teuthology.orchestra.run.smithi130.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-25T00:19:51.602 INFO:teuthology.orchestra.run.smithi121.stderr:sudo: ntpd: command not found 2024-04-25T00:19:51.603 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: ntpd: command not found 2024-04-25T00:19:51.613 INFO:teuthology.orchestra.run.smithi121.stdout:506 Cannot talk to daemon 2024-04-25T00:19:51.614 INFO:teuthology.orchestra.run.smithi032.stdout:506 Cannot talk to daemon 2024-04-25T00:19:51.621 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: ntpd: command not found 2024-04-25T00:19:51.627 INFO:teuthology.orchestra.run.smithi121.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-25T00:19:51.629 INFO:teuthology.orchestra.run.smithi032.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-25T00:19:51.631 INFO:teuthology.orchestra.run.smithi130.stdout:506 Cannot talk to daemon 2024-04-25T00:19:51.640 INFO:teuthology.orchestra.run.smithi121.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-25T00:19:51.642 INFO:teuthology.orchestra.run.smithi032.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-25T00:19:51.644 INFO:teuthology.orchestra.run.smithi130.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-25T00:19:51.658 INFO:teuthology.orchestra.run.smithi130.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-25T00:19:51.695 INFO:teuthology.orchestra.run.smithi121.stderr:bash: line 1: ntpq: command not found 2024-04-25T00:19:51.698 INFO:teuthology.orchestra.run.smithi121.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-25T00:19:51.698 INFO:teuthology.orchestra.run.smithi121.stdout:=============================================================================== 2024-04-25T00:19:51.698 INFO:teuthology.orchestra.run.smithi121.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T00:19:51.698 INFO:teuthology.orchestra.run.smithi121.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T00:19:51.698 INFO:teuthology.orchestra.run.smithi121.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T00:19:51.698 INFO:teuthology.orchestra.run.smithi121.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T00:19:51.704 INFO:teuthology.orchestra.run.smithi032.stderr:bash: line 1: ntpq: command not found 2024-04-25T00:19:51.708 INFO:teuthology.orchestra.run.smithi032.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-25T00:19:51.708 INFO:teuthology.orchestra.run.smithi032.stdout:=============================================================================== 2024-04-25T00:19:51.708 INFO:teuthology.orchestra.run.smithi032.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T00:19:51.708 INFO:teuthology.orchestra.run.smithi032.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T00:19:51.708 INFO:teuthology.orchestra.run.smithi032.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T00:19:51.708 INFO:teuthology.orchestra.run.smithi032.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T00:19:51.714 INFO:teuthology.orchestra.run.smithi130.stderr:bash: line 1: ntpq: command not found 2024-04-25T00:19:51.718 INFO:teuthology.orchestra.run.smithi130.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-25T00:19:51.718 INFO:teuthology.orchestra.run.smithi130.stdout:=============================================================================== 2024-04-25T00:19:51.718 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T00:19:51.719 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T00:19:51.719 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T00:19:51.719 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T00:19:51.720 INFO:teuthology.run_tasks:Running task install... 2024-04-25T00:19:51.734 DEBUG:teuthology.task.install:project ceph 2024-04-25T00:19:51.734 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'd2c45d977142e024a59b68a8f1dce8f0b9840551'}} 2024-04-25T00:19:51.734 DEBUG:teuthology.task.install:config {'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'flavor': 'default', 'sha1': 'd2c45d977142e024a59b68a8f1dce8f0b9840551'} 2024-04-25T00:19:51.734 INFO:teuthology.task.install:Using flavor: default 2024-04-25T00:19:51.740 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-04-25T00:19:51.740 INFO:teuthology.task.install:extra packages: {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']} 2024-04-25T00:19:51.740 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'd2c45d977142e024a59b68a8f1dce8f0b9840551', 'tag': None, 'wait_for_package': False} 2024-04-25T00:19:51.741 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=d2c45d977142e024a59b68a8f1dce8f0b9840551 2024-04-25T00:19:51.743 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'd2c45d977142e024a59b68a8f1dce8f0b9840551', 'tag': None, 'wait_for_package': False} 2024-04-25T00:19:51.743 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=d2c45d977142e024a59b68a8f1dce8f0b9840551 2024-04-25T00:19:51.744 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'd2c45d977142e024a59b68a8f1dce8f0b9840551', 'tag': None, 'wait_for_package': False} 2024-04-25T00:19:51.744 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=d2c45d977142e024a59b68a8f1dce8f0b9840551 2024-04-25T00:19:51.898 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/squid/d2c45d977142e024a59b68a8f1dce8f0b9840551/centos/9/flavors/default/ 2024-04-25T00:19:51.898 INFO:teuthology.task.install.rpm:Package version is 19.0.0-2445.gd2c45d97 2024-04-25T00:19:51.963 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/squid/d2c45d977142e024a59b68a8f1dce8f0b9840551/centos/9/flavors/default/ 2024-04-25T00:19:51.964 INFO:teuthology.task.install.rpm:Package version is 19.0.0-2445.gd2c45d97 2024-04-25T00:19:51.970 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/squid/d2c45d977142e024a59b68a8f1dce8f0b9840551/centos/9/flavors/default/ 2024-04-25T00:19:51.970 INFO:teuthology.task.install.rpm:Package version is 19.0.0-2445.gd2c45d97 2024-04-25T00:19:52.016 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/squid/d2c45d977142e024a59b68a8f1dce8f0b9840551/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/squid/d2c45d977142e024a59b68a8f1dce8f0b9840551/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/squid/d2c45d977142e024a59b68a8f1dce8f0b9840551/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-25T00:19:52.016 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-04-25T00:19:52.016 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-25T00:19:52.044 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-cephfs, cephfs-shell, cephfs-top, cephfs-mirror, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib on remote rpm x86_64 2024-04-25T00:19:52.044 DEBUG:teuthology.orchestra.run.smithi032:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/d2c45d977142e024a59b68a8f1dce8f0b9840551/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-25T00:19:52.089 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/squid/d2c45d977142e024a59b68a8f1dce8f0b9840551/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/squid/d2c45d977142e024a59b68a8f1dce8f0b9840551/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/squid/d2c45d977142e024a59b68a8f1dce8f0b9840551/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-25T00:19:52.089 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-04-25T00:19:52.089 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-25T00:19:52.096 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/squid/d2c45d977142e024a59b68a8f1dce8f0b9840551/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/squid/d2c45d977142e024a59b68a8f1dce8f0b9840551/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/squid/d2c45d977142e024a59b68a8f1dce8f0b9840551/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-25T00:19:52.096 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-04-25T00:19:52.096 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-25T00:19:52.109 DEBUG:teuthology.orchestra.run.smithi032:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-25T00:19:52.114 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-cephfs, cephfs-shell, cephfs-top, cephfs-mirror, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib on remote rpm x86_64 2024-04-25T00:19:52.114 DEBUG:teuthology.orchestra.run.smithi121:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/d2c45d977142e024a59b68a8f1dce8f0b9840551/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-25T00:19:52.122 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-cephfs, cephfs-shell, cephfs-top, cephfs-mirror, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib on remote rpm x86_64 2024-04-25T00:19:52.122 DEBUG:teuthology.orchestra.run.smithi130:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/d2c45d977142e024a59b68a8f1dce8f0b9840551/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-25T00:19:52.179 DEBUG:teuthology.orchestra.run.smithi121:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-25T00:19:52.185 DEBUG:teuthology.orchestra.run.smithi032:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-25T00:19:52.186 DEBUG:teuthology.orchestra.run.smithi130:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-25T00:19:52.249 INFO:teuthology.orchestra.run.smithi032.stdout:check_obsoletes = 1 2024-04-25T00:19:52.251 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum clean all 2024-04-25T00:19:52.252 DEBUG:teuthology.orchestra.run.smithi121:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-25T00:19:52.259 DEBUG:teuthology.orchestra.run.smithi130:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-25T00:19:52.314 INFO:teuthology.orchestra.run.smithi121.stdout:check_obsoletes = 1 2024-04-25T00:19:52.316 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum clean all 2024-04-25T00:19:52.322 INFO:teuthology.orchestra.run.smithi130.stdout:check_obsoletes = 1 2024-04-25T00:19:52.323 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum clean all 2024-04-25T00:19:52.486 INFO:teuthology.orchestra.run.smithi032.stdout:48 files removed 2024-04-25T00:19:52.513 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-shell cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib 2024-04-25T00:19:52.540 INFO:teuthology.orchestra.run.smithi121.stdout:48 files removed 2024-04-25T00:19:52.546 INFO:teuthology.orchestra.run.smithi130.stdout:48 files removed 2024-04-25T00:19:52.567 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-shell cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib 2024-04-25T00:19:52.573 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-shell cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib 2024-04-25T00:19:53.198 INFO:teuthology.orchestra.run.smithi032.stdout:ceph packages for x86_64 201 kB/s | 83 kB 00:00 2024-04-25T00:19:53.235 INFO:teuthology.orchestra.run.smithi121.stdout:ceph packages for x86_64 204 kB/s | 83 kB 00:00 2024-04-25T00:19:53.302 INFO:teuthology.orchestra.run.smithi130.stdout:ceph packages for x86_64 178 kB/s | 83 kB 00:00 2024-04-25T00:19:53.566 INFO:teuthology.orchestra.run.smithi032.stdout:ceph noarch packages 36 kB/s | 11 kB 00:00 2024-04-25T00:19:53.586 INFO:teuthology.orchestra.run.smithi121.stdout:ceph noarch packages 38 kB/s | 11 kB 00:00 2024-04-25T00:19:53.686 INFO:teuthology.orchestra.run.smithi130.stdout:ceph noarch packages 34 kB/s | 11 kB 00:00 2024-04-25T00:19:53.936 INFO:teuthology.orchestra.run.smithi121.stdout:ceph source packages 6.0 kB/s | 1.9 kB 00:00 2024-04-25T00:19:53.941 INFO:teuthology.orchestra.run.smithi032.stdout:ceph source packages 5.6 kB/s | 1.9 kB 00:00 2024-04-25T00:19:54.120 INFO:teuthology.orchestra.run.smithi130.stdout:ceph source packages 4.8 kB/s | 1.9 kB 00:00 2024-04-25T00:19:54.611 INFO:teuthology.orchestra.run.smithi130.stdout:CentOS Stream 9 - BaseOS 18 MB/s | 8.0 MB 00:00 2024-04-25T00:19:55.075 INFO:teuthology.orchestra.run.smithi032.stdout:CentOS Stream 9 - BaseOS 7.3 MB/s | 8.0 MB 00:01 2024-04-25T00:19:55.878 INFO:teuthology.orchestra.run.smithi121.stdout:CentOS Stream 9 - BaseOS 4.2 MB/s | 8.0 MB 00:01 2024-04-25T00:19:56.429 INFO:teuthology.orchestra.run.smithi130.stdout:CentOS Stream 9 - AppStream 25 MB/s | 19 MB 00:00 2024-04-25T00:19:58.010 INFO:teuthology.orchestra.run.smithi032.stdout:CentOS Stream 9 - AppStream 11 MB/s | 19 MB 00:01 2024-04-25T00:20:00.847 INFO:teuthology.orchestra.run.smithi130.stdout:CentOS Stream 9 - CRB 7.4 MB/s | 6.2 MB 00:00 2024-04-25T00:20:02.219 INFO:teuthology.orchestra.run.smithi032.stdout:CentOS Stream 9 - CRB 12 MB/s | 6.2 MB 00:00 2024-04-25T00:20:03.214 INFO:teuthology.orchestra.run.smithi130.stdout:CentOS Stream 9 - Extras packages 13 kB/s | 16 kB 00:01 2024-04-25T00:20:03.912 INFO:teuthology.orchestra.run.smithi032.stdout:CentOS Stream 9 - Extras packages 33 kB/s | 16 kB 00:00 2024-04-25T00:20:04.885 INFO:teuthology.orchestra.run.smithi130.stdout:Extra Packages for Enterprise Linux 14 MB/s | 21 MB 00:01 2024-04-25T00:20:05.863 INFO:teuthology.orchestra.run.smithi032.stdout:Extra Packages for Enterprise Linux 11 MB/s | 21 MB 00:01 2024-04-25T00:20:09.975 INFO:teuthology.orchestra.run.smithi130.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-04-25T00:20:11.096 INFO:teuthology.orchestra.run.smithi032.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-04-25T00:20:12.093 INFO:teuthology.orchestra.run.smithi130.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-04-25T00:20:12.093 INFO:teuthology.orchestra.run.smithi130.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-04-25T00:20:12.104 INFO:teuthology.orchestra.run.smithi130.stdout:Package NetworkManager-1:1.47.4-1.el9.x86_64 is already installed. 2024-04-25T00:20:12.105 INFO:teuthology.orchestra.run.smithi130.stdout:Package iproute-6.2.0-5.el9.x86_64 is already installed. 2024-04-25T00:20:12.106 INFO:teuthology.orchestra.run.smithi130.stdout:Package util-linux-2.37.4-18.el9.x86_64 is already installed. 2024-04-25T00:20:12.107 INFO:teuthology.orchestra.run.smithi130.stdout:Package libacl-devel-2.3.1-4.el9.x86_64 is already installed. 2024-04-25T00:20:12.108 INFO:teuthology.orchestra.run.smithi130.stdout:Package libattr-devel-2.5.1-3.el9.x86_64 is already installed. 2024-04-25T00:20:12.111 INFO:teuthology.orchestra.run.smithi130.stdout:Package xfsdump-3.1.12-4.el9.x86_64 is already installed. 2024-04-25T00:20:12.111 INFO:teuthology.orchestra.run.smithi130.stdout:Package xfsprogs-6.3.0-1.el9.x86_64 is already installed. 2024-04-25T00:20:12.173 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-04-25T00:20:12.190 INFO:teuthology.orchestra.run.smithi130.stdout:======================================================================================= 2024-04-25T00:20:12.190 INFO:teuthology.orchestra.run.smithi130.stdout: Package Arch Version Repository Size 2024-04-25T00:20:12.190 INFO:teuthology.orchestra.run.smithi130.stdout:======================================================================================= 2024-04-25T00:20:12.190 INFO:teuthology.orchestra.run.smithi130.stdout:Installing: 2024-04-25T00:20:12.190 INFO:teuthology.orchestra.run.smithi130.stdout: bison x86_64 3.7.4-5.el9 appstream 943 k 2024-04-25T00:20:12.190 INFO:teuthology.orchestra.run.smithi130.stdout: ceph x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 6.6 k 2024-04-25T00:20:12.191 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-base x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 5.4 M 2024-04-25T00:20:12.191 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-fuse x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 890 k 2024-04-25T00:20:12.191 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-immutable-object-cache x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 145 k 2024-04-25T00:20:12.191 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 1.5 M 2024-04-25T00:20:12.191 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-cephadm noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 141 k 2024-04-25T00:20:12.191 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-dashboard noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 3.5 M 2024-04-25T00:20:12.191 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-diskprediction-local noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 7.4 M 2024-04-25T00:20:12.191 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-rook noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 49 k 2024-04-25T00:20:12.191 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-radosgw x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 10 M 2024-04-25T00:20:12.191 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-test x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 48 M 2024-04-25T00:20:12.191 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-volume noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 264 k 2024-04-25T00:20:12.191 INFO:teuthology.orchestra.run.smithi130.stdout: cephadm noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 762 k 2024-04-25T00:20:12.191 INFO:teuthology.orchestra.run.smithi130.stdout: cephfs-mirror x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 222 k 2024-04-25T00:20:12.192 INFO:teuthology.orchestra.run.smithi130.stdout: cephfs-shell x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 27 k 2024-04-25T00:20:12.192 INFO:teuthology.orchestra.run.smithi130.stdout: cephfs-top noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 18 k 2024-04-25T00:20:12.192 INFO:teuthology.orchestra.run.smithi130.stdout: elfutils-libelf-devel x86_64 0.190-2.el9 appstream 21 k 2024-04-25T00:20:12.192 INFO:teuthology.orchestra.run.smithi130.stdout: flex x86_64 2.6.4-9.el9 appstream 313 k 2024-04-25T00:20:12.192 INFO:teuthology.orchestra.run.smithi130.stdout: libaio-devel x86_64 0.3.111-13.el9 appstream 11 k 2024-04-25T00:20:12.192 INFO:teuthology.orchestra.run.smithi130.stdout: libcephfs-devel x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 32 k 2024-04-25T00:20:12.192 INFO:teuthology.orchestra.run.smithi130.stdout: libcephfs2 x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 745 k 2024-04-25T00:20:12.192 INFO:teuthology.orchestra.run.smithi130.stdout: librados-devel x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 127 k 2024-04-25T00:20:12.192 INFO:teuthology.orchestra.run.smithi130.stdout: libtool x86_64 2.4.6-45.el9 appstream 582 k 2024-04-25T00:20:12.192 INFO:teuthology.orchestra.run.smithi130.stdout: libuuid-devel x86_64 2.37.4-18.el9 appstream 25 k 2024-04-25T00:20:12.192 INFO:teuthology.orchestra.run.smithi130.stdout: openssl-devel x86_64 1:3.2.1-1.el9 appstream 4.4 M 2024-04-25T00:20:12.192 INFO:teuthology.orchestra.run.smithi130.stdout: postgresql x86_64 13.14-1.el9 appstream 1.6 M 2024-04-25T00:20:12.192 INFO:teuthology.orchestra.run.smithi130.stdout: postgresql-contrib x86_64 13.14-1.el9 appstream 911 k 2024-04-25T00:20:12.193 INFO:teuthology.orchestra.run.smithi130.stdout: postgresql-server x86_64 13.14-1.el9 appstream 5.8 M 2024-04-25T00:20:12.193 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cephfs x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 162 k 2024-04-25T00:20:12.193 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rados x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 321 k 2024-04-25T00:20:12.193 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rbd x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 299 k 2024-04-25T00:20:12.193 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rgw x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 99 k 2024-04-25T00:20:12.193 INFO:teuthology.orchestra.run.smithi130.stdout: rbd-fuse x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 85 k 2024-04-25T00:20:12.193 INFO:teuthology.orchestra.run.smithi130.stdout: rbd-mirror x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 3.1 M 2024-04-25T00:20:12.193 INFO:teuthology.orchestra.run.smithi130.stdout: rbd-nbd x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 171 k 2024-04-25T00:20:12.193 INFO:teuthology.orchestra.run.smithi130.stdout: xfsprogs-devel x86_64 6.3.0-1.el9 appstream 94 k 2024-04-25T00:20:12.193 INFO:teuthology.orchestra.run.smithi130.stdout:Upgrading: 2024-04-25T00:20:12.193 INFO:teuthology.orchestra.run.smithi130.stdout: librados2 x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 3.4 M 2024-04-25T00:20:12.193 INFO:teuthology.orchestra.run.smithi130.stdout: librbd1 x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 3.1 M 2024-04-25T00:20:12.193 INFO:teuthology.orchestra.run.smithi130.stdout: openssl x86_64 1:3.2.1-1.el9 baseos 1.3 M 2024-04-25T00:20:12.194 INFO:teuthology.orchestra.run.smithi130.stdout: openssl-libs x86_64 1:3.2.1-1.el9 baseos 2.4 M 2024-04-25T00:20:12.194 INFO:teuthology.orchestra.run.smithi130.stdout:Installing dependencies: 2024-04-25T00:20:12.194 INFO:teuthology.orchestra.run.smithi130.stdout: abseil-cpp x86_64 20211102.0-3.el9 epel 548 k 2024-04-25T00:20:12.194 INFO:teuthology.orchestra.run.smithi130.stdout: automake noarch 1.16.2-8.el9 appstream 689 k 2024-04-25T00:20:12.194 INFO:teuthology.orchestra.run.smithi130.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-04-25T00:20:12.194 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-common x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 21 M 2024-04-25T00:20:12.194 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-grafana-dashboards noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 32 k 2024-04-25T00:20:12.194 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mds x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 2.4 M 2024-04-25T00:20:12.194 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-modules-core noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 249 k 2024-04-25T00:20:12.194 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mon x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 4.7 M 2024-04-25T00:20:12.194 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-osd x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 17 M 2024-04-25T00:20:12.194 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-prometheus-alerts noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 15 k 2024-04-25T00:20:12.195 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-selinux x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 25 k 2024-04-25T00:20:12.195 INFO:teuthology.orchestra.run.smithi130.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-04-25T00:20:12.195 INFO:teuthology.orchestra.run.smithi130.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-04-25T00:20:12.195 INFO:teuthology.orchestra.run.smithi130.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-04-25T00:20:12.195 INFO:teuthology.orchestra.run.smithi130.stdout: fuse x86_64 2.9.9-15.el9 baseos 80 k 2024-04-25T00:20:12.195 INFO:teuthology.orchestra.run.smithi130.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-25T00:20:12.195 INFO:teuthology.orchestra.run.smithi130.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2024-04-25T00:20:12.195 INFO:teuthology.orchestra.run.smithi130.stdout: libarrow x86_64 9.0.0-10.el9 epel 4.4 M 2024-04-25T00:20:12.196 INFO:teuthology.orchestra.run.smithi130.stdout: libarrow-doc noarch 9.0.0-10.el9 epel 26 k 2024-04-25T00:20:12.196 INFO:teuthology.orchestra.run.smithi130.stdout: libcephsqlite x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 163 k 2024-04-25T00:20:12.196 INFO:teuthology.orchestra.run.smithi130.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-25T00:20:12.196 INFO:teuthology.orchestra.run.smithi130.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-25T00:20:12.196 INFO:teuthology.orchestra.run.smithi130.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-04-25T00:20:12.196 INFO:teuthology.orchestra.run.smithi130.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-25T00:20:12.196 INFO:teuthology.orchestra.run.smithi130.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-04-25T00:20:12.196 INFO:teuthology.orchestra.run.smithi130.stdout: libradosstriper1 x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 501 k 2024-04-25T00:20:12.196 INFO:teuthology.orchestra.run.smithi130.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-04-25T00:20:12.197 INFO:teuthology.orchestra.run.smithi130.stdout: librgw2 x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 5.3 M 2024-04-25T00:20:12.197 INFO:teuthology.orchestra.run.smithi130.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-25T00:20:12.197 INFO:teuthology.orchestra.run.smithi130.stdout: libzstd-devel x86_64 1.5.1-2.el9 appstream 47 k 2024-04-25T00:20:12.197 INFO:teuthology.orchestra.run.smithi130.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-04-25T00:20:12.197 INFO:teuthology.orchestra.run.smithi130.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2024-04-25T00:20:12.197 INFO:teuthology.orchestra.run.smithi130.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-04-25T00:20:12.197 INFO:teuthology.orchestra.run.smithi130.stdout: luarocks noarch 3.9.2-1.el9 epel 151 k 2024-04-25T00:20:12.197 INFO:teuthology.orchestra.run.smithi130.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-04-25T00:20:12.197 INFO:teuthology.orchestra.run.smithi130.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-04-25T00:20:12.197 INFO:teuthology.orchestra.run.smithi130.stdout: parquet-libs x86_64 9.0.0-10.el9 epel 838 k 2024-04-25T00:20:12.197 INFO:teuthology.orchestra.run.smithi130.stdout: postgresql-private-libs x86_64 13.14-1.el9 appstream 133 k 2024-04-25T00:20:12.197 INFO:teuthology.orchestra.run.smithi130.stdout: protobuf x86_64 3.14.0-13.el9 appstream 1.0 M 2024-04-25T00:20:12.197 INFO:teuthology.orchestra.run.smithi130.stdout: protobuf-compiler x86_64 3.14.0-13.el9 crb 863 k 2024-04-25T00:20:12.197 INFO:teuthology.orchestra.run.smithi130.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-25T00:20:12.198 INFO:teuthology.orchestra.run.smithi130.stdout: python3-attrs noarch 20.3.0-7.el9 appstream 87 k 2024-04-25T00:20:12.198 INFO:teuthology.orchestra.run.smithi130.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-04-25T00:20:12.198 INFO:teuthology.orchestra.run.smithi130.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-25T00:20:12.198 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-25T00:20:12.198 INFO:teuthology.orchestra.run.smithi130.stdout: python3-ceph-argparse x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 45 k 2024-04-25T00:20:12.198 INFO:teuthology.orchestra.run.smithi130.stdout: python3-ceph-common x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 133 k 2024-04-25T00:20:12.198 INFO:teuthology.orchestra.run.smithi130.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-25T00:20:12.198 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-04-25T00:20:12.198 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-25T00:20:12.198 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-25T00:20:12.198 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cmd2 noarch 2.4.2-2.el9 epel 303 k 2024-04-25T00:20:12.198 INFO:teuthology.orchestra.run.smithi130.stdout: python3-colorama noarch 0.4.4-12.el9 epel 35 k 2024-04-25T00:20:12.198 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-04-25T00:20:12.199 INFO:teuthology.orchestra.run.smithi130.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-04-25T00:20:12.199 INFO:teuthology.orchestra.run.smithi130.stdout: python3-google-auth noarch 1:2.29.0-1.el9 epel 206 k 2024-04-25T00:20:12.199 INFO:teuthology.orchestra.run.smithi130.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2024-04-25T00:20:12.199 INFO:teuthology.orchestra.run.smithi130.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2024-04-25T00:20:12.199 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-25T00:20:12.199 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-25T00:20:12.199 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-04-25T00:20:12.199 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-25T00:20:12.199 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-25T00:20:12.199 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-25T00:20:12.199 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-04-25T00:20:12.199 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-04-25T00:20:12.199 INFO:teuthology.orchestra.run.smithi130.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-25T00:20:12.199 INFO:teuthology.orchestra.run.smithi130.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-25T00:20:12.200 INFO:teuthology.orchestra.run.smithi130.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-04-25T00:20:12.200 INFO:teuthology.orchestra.run.smithi130.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-04-25T00:20:12.200 INFO:teuthology.orchestra.run.smithi130.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-25T00:20:12.200 INFO:teuthology.orchestra.run.smithi130.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-25T00:20:12.200 INFO:teuthology.orchestra.run.smithi130.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-04-25T00:20:12.200 INFO:teuthology.orchestra.run.smithi130.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-04-25T00:20:12.200 INFO:teuthology.orchestra.run.smithi130.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-04-25T00:20:12.200 INFO:teuthology.orchestra.run.smithi130.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-04-25T00:20:12.200 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-25T00:20:12.200 INFO:teuthology.orchestra.run.smithi130.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-04-25T00:20:12.200 INFO:teuthology.orchestra.run.smithi130.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-25T00:20:12.200 INFO:teuthology.orchestra.run.smithi130.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-04-25T00:20:12.200 INFO:teuthology.orchestra.run.smithi130.stdout: python3-protobuf noarch 3.14.0-13.el9 appstream 269 k 2024-04-25T00:20:12.200 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-25T00:20:12.200 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-04-25T00:20:12.201 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-04-25T00:20:12.201 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-04-25T00:20:12.201 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pyperclip noarch 1.8.2-2.el9 epel 28 k 2024-04-25T00:20:12.201 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-04-25T00:20:12.201 INFO:teuthology.orchestra.run.smithi130.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-25T00:20:12.201 INFO:teuthology.orchestra.run.smithi130.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-25T00:20:12.201 INFO:teuthology.orchestra.run.smithi130.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-04-25T00:20:12.201 INFO:teuthology.orchestra.run.smithi130.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-25T00:20:12.201 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-25T00:20:12.201 INFO:teuthology.orchestra.run.smithi130.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-04-25T00:20:12.201 INFO:teuthology.orchestra.run.smithi130.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-25T00:20:12.201 INFO:teuthology.orchestra.run.smithi130.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-04-25T00:20:12.201 INFO:teuthology.orchestra.run.smithi130.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-25T00:20:12.201 INFO:teuthology.orchestra.run.smithi130.stdout: python3-urllib3 noarch 1.26.5-5.el9 baseos 215 k 2024-04-25T00:20:12.202 INFO:teuthology.orchestra.run.smithi130.stdout: python3-wcwidth noarch 0.2.5-8.el9 appstream 44 k 2024-04-25T00:20:12.202 INFO:teuthology.orchestra.run.smithi130.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-25T00:20:12.202 INFO:teuthology.orchestra.run.smithi130.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-25T00:20:12.202 INFO:teuthology.orchestra.run.smithi130.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-25T00:20:12.202 INFO:teuthology.orchestra.run.smithi130.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-25T00:20:12.202 INFO:teuthology.orchestra.run.smithi130.stdout: qatlib x86_64 24.02.0-1.el9 appstream 221 k 2024-04-25T00:20:12.202 INFO:teuthology.orchestra.run.smithi130.stdout: qatzip-libs x86_64 1.2.0-1.el9 appstream 47 k 2024-04-25T00:20:12.202 INFO:teuthology.orchestra.run.smithi130.stdout: re2 x86_64 1:20211101-3.el9 epel 192 k 2024-04-25T00:20:12.202 INFO:teuthology.orchestra.run.smithi130.stdout: socat x86_64 1.7.4.1-5.el9 appstream 305 k 2024-04-25T00:20:12.202 INFO:teuthology.orchestra.run.smithi130.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-25T00:20:12.202 INFO:teuthology.orchestra.run.smithi130.stdout: uuid x86_64 1.6.2-55.el9 appstream 58 k 2024-04-25T00:20:12.202 INFO:teuthology.orchestra.run.smithi130.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-04-25T00:20:12.202 INFO:teuthology.orchestra.run.smithi130.stdout: zlib-devel x86_64 1.2.11-41.el9 appstream 45 k 2024-04-25T00:20:12.202 INFO:teuthology.orchestra.run.smithi130.stdout:Installing weak dependencies: 2024-04-25T00:20:12.203 INFO:teuthology.orchestra.run.smithi130.stdout: qatlib-service x86_64 24.02.0-1.el9 appstream 36 k 2024-04-25T00:20:12.203 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-25T00:20:12.203 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction Summary 2024-04-25T00:20:12.203 INFO:teuthology.orchestra.run.smithi130.stdout:======================================================================================= 2024-04-25T00:20:12.203 INFO:teuthology.orchestra.run.smithi130.stdout:Install 148 Packages 2024-04-25T00:20:12.203 INFO:teuthology.orchestra.run.smithi130.stdout:Upgrade 4 Packages 2024-04-25T00:20:12.203 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-25T00:20:12.204 INFO:teuthology.orchestra.run.smithi130.stdout:Total download size: 220 M 2024-04-25T00:20:12.204 INFO:teuthology.orchestra.run.smithi130.stdout:Downloading Packages: 2024-04-25T00:20:12.809 INFO:teuthology.orchestra.run.smithi121.stdout:CentOS Stream 9 - AppStream 1.2 MB/s | 19 MB 00:15 2024-04-25T00:20:13.076 INFO:teuthology.orchestra.run.smithi130.stdout:(1/152): ceph-19.0.0-2445.gd2c45d97.el9.x86_64. 29 kB/s | 6.6 kB 00:00 2024-04-25T00:20:13.315 INFO:teuthology.orchestra.run.smithi032.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-04-25T00:20:13.316 INFO:teuthology.orchestra.run.smithi032.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-04-25T00:20:13.328 INFO:teuthology.orchestra.run.smithi032.stdout:Package NetworkManager-1:1.47.4-1.el9.x86_64 is already installed. 2024-04-25T00:20:13.329 INFO:teuthology.orchestra.run.smithi032.stdout:Package iproute-6.2.0-5.el9.x86_64 is already installed. 2024-04-25T00:20:13.329 INFO:teuthology.orchestra.run.smithi032.stdout:Package util-linux-2.37.4-18.el9.x86_64 is already installed. 2024-04-25T00:20:13.330 INFO:teuthology.orchestra.run.smithi032.stdout:Package libacl-devel-2.3.1-4.el9.x86_64 is already installed. 2024-04-25T00:20:13.332 INFO:teuthology.orchestra.run.smithi032.stdout:Package libattr-devel-2.5.1-3.el9.x86_64 is already installed. 2024-04-25T00:20:13.335 INFO:teuthology.orchestra.run.smithi032.stdout:Package xfsdump-3.1.12-4.el9.x86_64 is already installed. 2024-04-25T00:20:13.335 INFO:teuthology.orchestra.run.smithi032.stdout:Package xfsprogs-6.3.0-1.el9.x86_64 is already installed. 2024-04-25T00:20:13.400 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-04-25T00:20:13.401 INFO:teuthology.orchestra.run.smithi130.stdout:(2/152): ceph-fuse-19.0.0-2445.gd2c45d97.el9.x8 2.7 MB/s | 890 kB 00:00 2024-04-25T00:20:13.409 INFO:teuthology.orchestra.run.smithi032.stdout:======================================================================================= 2024-04-25T00:20:13.410 INFO:teuthology.orchestra.run.smithi032.stdout: Package Arch Version Repository Size 2024-04-25T00:20:13.410 INFO:teuthology.orchestra.run.smithi032.stdout:======================================================================================= 2024-04-25T00:20:13.410 INFO:teuthology.orchestra.run.smithi032.stdout:Installing: 2024-04-25T00:20:13.410 INFO:teuthology.orchestra.run.smithi032.stdout: bison x86_64 3.7.4-5.el9 appstream 943 k 2024-04-25T00:20:13.410 INFO:teuthology.orchestra.run.smithi032.stdout: ceph x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 6.6 k 2024-04-25T00:20:13.410 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-base x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 5.4 M 2024-04-25T00:20:13.410 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-fuse x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 890 k 2024-04-25T00:20:13.410 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-immutable-object-cache x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 145 k 2024-04-25T00:20:13.410 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mgr x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 1.5 M 2024-04-25T00:20:13.410 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mgr-cephadm noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 141 k 2024-04-25T00:20:13.410 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mgr-dashboard noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 3.5 M 2024-04-25T00:20:13.410 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mgr-diskprediction-local noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 7.4 M 2024-04-25T00:20:13.411 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mgr-rook noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 49 k 2024-04-25T00:20:13.411 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-radosgw x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 10 M 2024-04-25T00:20:13.411 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-test x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 48 M 2024-04-25T00:20:13.411 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-volume noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 264 k 2024-04-25T00:20:13.411 INFO:teuthology.orchestra.run.smithi032.stdout: cephadm noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 762 k 2024-04-25T00:20:13.411 INFO:teuthology.orchestra.run.smithi032.stdout: cephfs-mirror x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 222 k 2024-04-25T00:20:13.411 INFO:teuthology.orchestra.run.smithi032.stdout: cephfs-shell x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 27 k 2024-04-25T00:20:13.411 INFO:teuthology.orchestra.run.smithi032.stdout: cephfs-top noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 18 k 2024-04-25T00:20:13.411 INFO:teuthology.orchestra.run.smithi032.stdout: elfutils-libelf-devel x86_64 0.190-2.el9 appstream 21 k 2024-04-25T00:20:13.411 INFO:teuthology.orchestra.run.smithi032.stdout: flex x86_64 2.6.4-9.el9 appstream 313 k 2024-04-25T00:20:13.411 INFO:teuthology.orchestra.run.smithi032.stdout: libaio-devel x86_64 0.3.111-13.el9 appstream 11 k 2024-04-25T00:20:13.411 INFO:teuthology.orchestra.run.smithi032.stdout: libcephfs-devel x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 32 k 2024-04-25T00:20:13.411 INFO:teuthology.orchestra.run.smithi032.stdout: libcephfs2 x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 745 k 2024-04-25T00:20:13.412 INFO:teuthology.orchestra.run.smithi032.stdout: librados-devel x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 127 k 2024-04-25T00:20:13.412 INFO:teuthology.orchestra.run.smithi032.stdout: libtool x86_64 2.4.6-45.el9 appstream 582 k 2024-04-25T00:20:13.412 INFO:teuthology.orchestra.run.smithi032.stdout: libuuid-devel x86_64 2.37.4-18.el9 appstream 25 k 2024-04-25T00:20:13.412 INFO:teuthology.orchestra.run.smithi032.stdout: openssl-devel x86_64 1:3.2.1-1.el9 appstream 4.4 M 2024-04-25T00:20:13.412 INFO:teuthology.orchestra.run.smithi032.stdout: postgresql x86_64 13.14-1.el9 appstream 1.6 M 2024-04-25T00:20:13.412 INFO:teuthology.orchestra.run.smithi032.stdout: postgresql-contrib x86_64 13.14-1.el9 appstream 911 k 2024-04-25T00:20:13.412 INFO:teuthology.orchestra.run.smithi032.stdout: postgresql-server x86_64 13.14-1.el9 appstream 5.8 M 2024-04-25T00:20:13.412 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cephfs x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 162 k 2024-04-25T00:20:13.412 INFO:teuthology.orchestra.run.smithi032.stdout: python3-rados x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 321 k 2024-04-25T00:20:13.412 INFO:teuthology.orchestra.run.smithi032.stdout: python3-rbd x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 299 k 2024-04-25T00:20:13.412 INFO:teuthology.orchestra.run.smithi032.stdout: python3-rgw x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 99 k 2024-04-25T00:20:13.412 INFO:teuthology.orchestra.run.smithi032.stdout: rbd-fuse x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 85 k 2024-04-25T00:20:13.412 INFO:teuthology.orchestra.run.smithi032.stdout: rbd-mirror x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 3.1 M 2024-04-25T00:20:13.413 INFO:teuthology.orchestra.run.smithi032.stdout: rbd-nbd x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 171 k 2024-04-25T00:20:13.413 INFO:teuthology.orchestra.run.smithi032.stdout: xfsprogs-devel x86_64 6.3.0-1.el9 appstream 94 k 2024-04-25T00:20:13.413 INFO:teuthology.orchestra.run.smithi032.stdout:Upgrading: 2024-04-25T00:20:13.413 INFO:teuthology.orchestra.run.smithi032.stdout: librados2 x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 3.4 M 2024-04-25T00:20:13.413 INFO:teuthology.orchestra.run.smithi032.stdout: librbd1 x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 3.1 M 2024-04-25T00:20:13.413 INFO:teuthology.orchestra.run.smithi032.stdout: openssl x86_64 1:3.2.1-1.el9 baseos 1.3 M 2024-04-25T00:20:13.413 INFO:teuthology.orchestra.run.smithi032.stdout: openssl-libs x86_64 1:3.2.1-1.el9 baseos 2.4 M 2024-04-25T00:20:13.413 INFO:teuthology.orchestra.run.smithi032.stdout:Installing dependencies: 2024-04-25T00:20:13.413 INFO:teuthology.orchestra.run.smithi032.stdout: abseil-cpp x86_64 20211102.0-3.el9 epel 548 k 2024-04-25T00:20:13.413 INFO:teuthology.orchestra.run.smithi032.stdout: automake noarch 1.16.2-8.el9 appstream 689 k 2024-04-25T00:20:13.413 INFO:teuthology.orchestra.run.smithi032.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-04-25T00:20:13.413 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-common x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 21 M 2024-04-25T00:20:13.413 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-grafana-dashboards noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 32 k 2024-04-25T00:20:13.413 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mds x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 2.4 M 2024-04-25T00:20:13.414 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mgr-modules-core noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 249 k 2024-04-25T00:20:13.414 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mon x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 4.7 M 2024-04-25T00:20:13.414 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-osd x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 17 M 2024-04-25T00:20:13.414 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-prometheus-alerts noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 15 k 2024-04-25T00:20:13.414 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-selinux x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 25 k 2024-04-25T00:20:13.414 INFO:teuthology.orchestra.run.smithi032.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-04-25T00:20:13.414 INFO:teuthology.orchestra.run.smithi032.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-04-25T00:20:13.414 INFO:teuthology.orchestra.run.smithi032.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-04-25T00:20:13.414 INFO:teuthology.orchestra.run.smithi032.stdout: fuse x86_64 2.9.9-15.el9 baseos 80 k 2024-04-25T00:20:13.414 INFO:teuthology.orchestra.run.smithi032.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-25T00:20:13.414 INFO:teuthology.orchestra.run.smithi032.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2024-04-25T00:20:13.414 INFO:teuthology.orchestra.run.smithi032.stdout: libarrow x86_64 9.0.0-10.el9 epel 4.4 M 2024-04-25T00:20:13.414 INFO:teuthology.orchestra.run.smithi032.stdout: libarrow-doc noarch 9.0.0-10.el9 epel 26 k 2024-04-25T00:20:13.415 INFO:teuthology.orchestra.run.smithi032.stdout: libcephsqlite x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 163 k 2024-04-25T00:20:13.415 INFO:teuthology.orchestra.run.smithi032.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-25T00:20:13.415 INFO:teuthology.orchestra.run.smithi032.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-25T00:20:13.415 INFO:teuthology.orchestra.run.smithi032.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-04-25T00:20:13.415 INFO:teuthology.orchestra.run.smithi032.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-25T00:20:13.415 INFO:teuthology.orchestra.run.smithi032.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-04-25T00:20:13.415 INFO:teuthology.orchestra.run.smithi032.stdout: libradosstriper1 x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 501 k 2024-04-25T00:20:13.415 INFO:teuthology.orchestra.run.smithi032.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-04-25T00:20:13.415 INFO:teuthology.orchestra.run.smithi032.stdout: librgw2 x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 5.3 M 2024-04-25T00:20:13.415 INFO:teuthology.orchestra.run.smithi032.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-25T00:20:13.415 INFO:teuthology.orchestra.run.smithi032.stdout: libzstd-devel x86_64 1.5.1-2.el9 appstream 47 k 2024-04-25T00:20:13.415 INFO:teuthology.orchestra.run.smithi032.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-04-25T00:20:13.415 INFO:teuthology.orchestra.run.smithi032.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2024-04-25T00:20:13.416 INFO:teuthology.orchestra.run.smithi032.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-04-25T00:20:13.416 INFO:teuthology.orchestra.run.smithi032.stdout: luarocks noarch 3.9.2-1.el9 epel 151 k 2024-04-25T00:20:13.416 INFO:teuthology.orchestra.run.smithi032.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-04-25T00:20:13.416 INFO:teuthology.orchestra.run.smithi032.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-04-25T00:20:13.416 INFO:teuthology.orchestra.run.smithi032.stdout: parquet-libs x86_64 9.0.0-10.el9 epel 838 k 2024-04-25T00:20:13.416 INFO:teuthology.orchestra.run.smithi032.stdout: postgresql-private-libs x86_64 13.14-1.el9 appstream 133 k 2024-04-25T00:20:13.416 INFO:teuthology.orchestra.run.smithi032.stdout: protobuf x86_64 3.14.0-13.el9 appstream 1.0 M 2024-04-25T00:20:13.416 INFO:teuthology.orchestra.run.smithi032.stdout: protobuf-compiler x86_64 3.14.0-13.el9 crb 863 k 2024-04-25T00:20:13.416 INFO:teuthology.orchestra.run.smithi032.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-25T00:20:13.416 INFO:teuthology.orchestra.run.smithi032.stdout: python3-attrs noarch 20.3.0-7.el9 appstream 87 k 2024-04-25T00:20:13.416 INFO:teuthology.orchestra.run.smithi032.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-04-25T00:20:13.416 INFO:teuthology.orchestra.run.smithi032.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-25T00:20:13.416 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-25T00:20:13.416 INFO:teuthology.orchestra.run.smithi032.stdout: python3-ceph-argparse x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 45 k 2024-04-25T00:20:13.417 INFO:teuthology.orchestra.run.smithi032.stdout: python3-ceph-common x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 133 k 2024-04-25T00:20:13.417 INFO:teuthology.orchestra.run.smithi032.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-25T00:20:13.417 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-04-25T00:20:13.417 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-25T00:20:13.417 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-25T00:20:13.417 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cmd2 noarch 2.4.2-2.el9 epel 303 k 2024-04-25T00:20:13.417 INFO:teuthology.orchestra.run.smithi032.stdout: python3-colorama noarch 0.4.4-12.el9 epel 35 k 2024-04-25T00:20:13.417 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-04-25T00:20:13.417 INFO:teuthology.orchestra.run.smithi032.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-04-25T00:20:13.417 INFO:teuthology.orchestra.run.smithi032.stdout: python3-google-auth noarch 1:2.29.0-1.el9 epel 206 k 2024-04-25T00:20:13.417 INFO:teuthology.orchestra.run.smithi032.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2024-04-25T00:20:13.417 INFO:teuthology.orchestra.run.smithi032.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2024-04-25T00:20:13.417 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-25T00:20:13.418 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-25T00:20:13.418 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-04-25T00:20:13.418 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-25T00:20:13.418 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-25T00:20:13.418 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-25T00:20:13.418 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-04-25T00:20:13.418 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-04-25T00:20:13.418 INFO:teuthology.orchestra.run.smithi032.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-25T00:20:13.418 INFO:teuthology.orchestra.run.smithi032.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-25T00:20:13.418 INFO:teuthology.orchestra.run.smithi032.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-04-25T00:20:13.418 INFO:teuthology.orchestra.run.smithi032.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-04-25T00:20:13.418 INFO:teuthology.orchestra.run.smithi032.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-25T00:20:13.418 INFO:teuthology.orchestra.run.smithi032.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-25T00:20:13.418 INFO:teuthology.orchestra.run.smithi032.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-04-25T00:20:13.419 INFO:teuthology.orchestra.run.smithi032.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-04-25T00:20:13.419 INFO:teuthology.orchestra.run.smithi032.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-04-25T00:20:13.419 INFO:teuthology.orchestra.run.smithi032.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-04-25T00:20:13.419 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-25T00:20:13.419 INFO:teuthology.orchestra.run.smithi032.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-04-25T00:20:13.419 INFO:teuthology.orchestra.run.smithi032.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-25T00:20:13.419 INFO:teuthology.orchestra.run.smithi032.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-04-25T00:20:13.419 INFO:teuthology.orchestra.run.smithi032.stdout: python3-protobuf noarch 3.14.0-13.el9 appstream 269 k 2024-04-25T00:20:13.419 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-25T00:20:13.419 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-04-25T00:20:13.419 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-04-25T00:20:13.419 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-04-25T00:20:13.419 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pyperclip noarch 1.8.2-2.el9 epel 28 k 2024-04-25T00:20:13.420 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-04-25T00:20:13.420 INFO:teuthology.orchestra.run.smithi032.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-25T00:20:13.420 INFO:teuthology.orchestra.run.smithi032.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-25T00:20:13.420 INFO:teuthology.orchestra.run.smithi032.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-04-25T00:20:13.420 INFO:teuthology.orchestra.run.smithi032.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-25T00:20:13.420 INFO:teuthology.orchestra.run.smithi032.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-25T00:20:13.420 INFO:teuthology.orchestra.run.smithi032.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-04-25T00:20:13.420 INFO:teuthology.orchestra.run.smithi032.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-25T00:20:13.420 INFO:teuthology.orchestra.run.smithi032.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-04-25T00:20:13.420 INFO:teuthology.orchestra.run.smithi032.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-25T00:20:13.420 INFO:teuthology.orchestra.run.smithi032.stdout: python3-urllib3 noarch 1.26.5-5.el9 baseos 215 k 2024-04-25T00:20:13.420 INFO:teuthology.orchestra.run.smithi032.stdout: python3-wcwidth noarch 0.2.5-8.el9 appstream 44 k 2024-04-25T00:20:13.420 INFO:teuthology.orchestra.run.smithi032.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-25T00:20:13.421 INFO:teuthology.orchestra.run.smithi032.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-25T00:20:13.421 INFO:teuthology.orchestra.run.smithi032.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-25T00:20:13.421 INFO:teuthology.orchestra.run.smithi032.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-25T00:20:13.421 INFO:teuthology.orchestra.run.smithi032.stdout: qatlib x86_64 24.02.0-1.el9 appstream 221 k 2024-04-25T00:20:13.421 INFO:teuthology.orchestra.run.smithi032.stdout: qatzip-libs x86_64 1.2.0-1.el9 appstream 47 k 2024-04-25T00:20:13.421 INFO:teuthology.orchestra.run.smithi032.stdout: re2 x86_64 1:20211101-3.el9 epel 192 k 2024-04-25T00:20:13.421 INFO:teuthology.orchestra.run.smithi032.stdout: socat x86_64 1.7.4.1-5.el9 appstream 305 k 2024-04-25T00:20:13.421 INFO:teuthology.orchestra.run.smithi032.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-25T00:20:13.421 INFO:teuthology.orchestra.run.smithi032.stdout: uuid x86_64 1.6.2-55.el9 appstream 58 k 2024-04-25T00:20:13.421 INFO:teuthology.orchestra.run.smithi032.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-04-25T00:20:13.421 INFO:teuthology.orchestra.run.smithi032.stdout: zlib-devel x86_64 1.2.11-41.el9 appstream 45 k 2024-04-25T00:20:13.421 INFO:teuthology.orchestra.run.smithi032.stdout:Installing weak dependencies: 2024-04-25T00:20:13.421 INFO:teuthology.orchestra.run.smithi032.stdout: qatlib-service x86_64 24.02.0-1.el9 appstream 36 k 2024-04-25T00:20:13.421 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:20:13.422 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction Summary 2024-04-25T00:20:13.422 INFO:teuthology.orchestra.run.smithi032.stdout:======================================================================================= 2024-04-25T00:20:13.422 INFO:teuthology.orchestra.run.smithi032.stdout:Install 148 Packages 2024-04-25T00:20:13.422 INFO:teuthology.orchestra.run.smithi032.stdout:Upgrade 4 Packages 2024-04-25T00:20:13.422 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:20:13.422 INFO:teuthology.orchestra.run.smithi032.stdout:Total download size: 220 M 2024-04-25T00:20:13.423 INFO:teuthology.orchestra.run.smithi032.stdout:Downloading Packages: 2024-04-25T00:20:13.493 INFO:teuthology.orchestra.run.smithi130.stdout:(3/152): ceph-immutable-object-cache-19.0.0-244 1.5 MB/s | 145 kB 00:00 2024-04-25T00:20:13.877 INFO:teuthology.orchestra.run.smithi130.stdout:(4/152): ceph-base-19.0.0-2445.gd2c45d97.el9.x8 5.2 MB/s | 5.4 MB 00:01 2024-04-25T00:20:14.027 INFO:teuthology.orchestra.run.smithi130.stdout:(5/152): ceph-mds-19.0.0-2445.gd2c45d97.el9.x86 4.4 MB/s | 2.4 MB 00:00 2024-04-25T00:20:14.127 INFO:teuthology.orchestra.run.smithi130.stdout:(6/152): ceph-mgr-19.0.0-2445.gd2c45d97.el9.x86 6.0 MB/s | 1.5 MB 00:00 2024-04-25T00:20:14.389 INFO:teuthology.orchestra.run.smithi032.stdout:(1/152): ceph-19.0.0-2445.gd2c45d97.el9.x86_64. 31 kB/s | 6.6 kB 00:00 2024-04-25T00:20:14.578 INFO:teuthology.orchestra.run.smithi130.stdout:(7/152): ceph-common-19.0.0-2445.gd2c45d97.el9. 12 MB/s | 21 MB 00:01 2024-04-25T00:20:14.648 INFO:teuthology.orchestra.run.smithi032.stdout:(2/152): ceph-fuse-19.0.0-2445.gd2c45d97.el9.x8 3.4 MB/s | 890 kB 00:00 2024-04-25T00:20:14.719 INFO:teuthology.orchestra.run.smithi130.stdout:(8/152): ceph-mon-19.0.0-2445.gd2c45d97.el9.x86 6.8 MB/s | 4.7 MB 00:00 2024-04-25T00:20:14.731 INFO:teuthology.orchestra.run.smithi032.stdout:(3/152): ceph-immutable-object-cache-19.0.0-244 1.7 MB/s | 145 kB 00:00 2024-04-25T00:20:14.828 INFO:teuthology.orchestra.run.smithi130.stdout:(9/152): ceph-selinux-19.0.0-2445.gd2c45d97.el9 232 kB/s | 25 kB 00:00 2024-04-25T00:20:15.132 INFO:teuthology.orchestra.run.smithi032.stdout:(4/152): ceph-base-19.0.0-2445.gd2c45d97.el9.x8 5.6 MB/s | 5.4 MB 00:00 2024-04-25T00:20:15.433 INFO:teuthology.orchestra.run.smithi032.stdout:(5/152): ceph-mds-19.0.0-2445.gd2c45d97.el9.x86 3.4 MB/s | 2.4 MB 00:00 2024-04-25T00:20:15.474 INFO:teuthology.orchestra.run.smithi032.stdout:(6/152): ceph-mgr-19.0.0-2445.gd2c45d97.el9.x86 4.4 MB/s | 1.5 MB 00:00 2024-04-25T00:20:16.270 INFO:teuthology.orchestra.run.smithi130.stdout:(10/152): ceph-radosgw-19.0.0-2445.gd2c45d97.el 6.0 MB/s | 10 MB 00:01 2024-04-25T00:20:16.383 INFO:teuthology.orchestra.run.smithi032.stdout:(7/152): ceph-mon-19.0.0-2445.gd2c45d97.el9.x86 4.9 MB/s | 4.7 MB 00:00 2024-04-25T00:20:16.394 INFO:teuthology.orchestra.run.smithi130.stdout:(11/152): cephfs-mirror-19.0.0-2445.gd2c45d97.e 1.7 MB/s | 222 kB 00:00 2024-04-25T00:20:16.486 INFO:teuthology.orchestra.run.smithi130.stdout:(12/152): cephfs-shell-19.0.0-2445.gd2c45d97.el 295 kB/s | 27 kB 00:00 2024-04-25T00:20:16.579 INFO:teuthology.orchestra.run.smithi130.stdout:(13/152): libcephfs-devel-19.0.0-2445.gd2c45d97 345 kB/s | 32 kB 00:00 2024-04-25T00:20:16.787 INFO:teuthology.orchestra.run.smithi130.stdout:(14/152): libcephfs2-19.0.0-2445.gd2c45d97.el9. 3.5 MB/s | 745 kB 00:00 2024-04-25T00:20:16.871 INFO:teuthology.orchestra.run.smithi130.stdout:(15/152): libcephsqlite-19.0.0-2445.gd2c45d97.e 1.9 MB/s | 163 kB 00:00 2024-04-25T00:20:17.047 INFO:teuthology.orchestra.run.smithi130.stdout:(16/152): ceph-osd-19.0.0-2445.gd2c45d97.el9.x8 5.8 MB/s | 17 MB 00:02 2024-04-25T00:20:17.072 INFO:teuthology.orchestra.run.smithi130.stdout:(17/152): librados-devel-19.0.0-2445.gd2c45d97. 632 kB/s | 127 kB 00:00 2024-04-25T00:20:17.230 INFO:teuthology.orchestra.run.smithi130.stdout:(18/152): libradosstriper1-19.0.0-2445.gd2c45d9 2.7 MB/s | 501 kB 00:00 2024-04-25T00:20:17.339 INFO:teuthology.orchestra.run.smithi130.stdout:(19/152): python3-ceph-argparse-19.0.0-2445.gd2 416 kB/s | 45 kB 00:00 2024-04-25T00:20:17.456 INFO:teuthology.orchestra.run.smithi130.stdout:(20/152): python3-ceph-common-19.0.0-2445.gd2c4 1.1 MB/s | 133 kB 00:00 2024-04-25T00:20:17.576 INFO:teuthology.orchestra.run.smithi121.stdout:CentOS Stream 9 - CRB 5.2 MB/s | 6.2 MB 00:01 2024-04-25T00:20:17.581 INFO:teuthology.orchestra.run.smithi130.stdout:(21/152): python3-cephfs-19.0.0-2445.gd2c45d97. 1.3 MB/s | 162 kB 00:00 2024-04-25T00:20:17.706 INFO:teuthology.orchestra.run.smithi130.stdout:(22/152): python3-rados-19.0.0-2445.gd2c45d97.e 2.5 MB/s | 321 kB 00:00 2024-04-25T00:20:17.840 INFO:teuthology.orchestra.run.smithi130.stdout:(23/152): python3-rbd-19.0.0-2445.gd2c45d97.el9 2.2 MB/s | 299 kB 00:00 2024-04-25T00:20:17.915 INFO:teuthology.orchestra.run.smithi130.stdout:(24/152): librgw2-19.0.0-2445.gd2c45d97.el9.x86 6.3 MB/s | 5.3 MB 00:00 2024-04-25T00:20:17.932 INFO:teuthology.orchestra.run.smithi130.stdout:(25/152): python3-rgw-19.0.0-2445.gd2c45d97.el9 1.1 MB/s | 99 kB 00:00 2024-04-25T00:20:17.966 INFO:teuthology.orchestra.run.smithi130.stdout:(26/152): rbd-fuse-19.0.0-2445.gd2c45d97.el9.x8 1.7 MB/s | 85 kB 00:00 2024-04-25T00:20:18.000 INFO:teuthology.orchestra.run.smithi032.stdout:(8/152): ceph-osd-19.0.0-2445.gd2c45d97.el9.x86 6.7 MB/s | 17 MB 00:02 2024-04-25T00:20:18.083 INFO:teuthology.orchestra.run.smithi130.stdout:(27/152): rbd-nbd-19.0.0-2445.gd2c45d97.el9.x86 1.4 MB/s | 171 kB 00:00 2024-04-25T00:20:18.133 INFO:teuthology.orchestra.run.smithi130.stdout:(28/152): ceph-grafana-dashboards-19.0.0-2445.g 640 kB/s | 32 kB 00:00 2024-04-25T00:20:18.158 INFO:teuthology.orchestra.run.smithi032.stdout:(9/152): ceph-radosgw-19.0.0-2445.gd2c45d97.el9 5.8 MB/s | 10 MB 00:01 2024-04-25T00:20:18.175 INFO:teuthology.orchestra.run.smithi032.stdout:(10/152): ceph-selinux-19.0.0-2445.gd2c45d97.el 144 kB/s | 25 kB 00:00 2024-04-25T00:20:18.192 INFO:teuthology.orchestra.run.smithi130.stdout:(29/152): ceph-mgr-cephadm-19.0.0-2445.gd2c45d9 2.4 MB/s | 141 kB 00:00 2024-04-25T00:20:18.233 INFO:teuthology.orchestra.run.smithi032.stdout:(11/152): cephfs-mirror-19.0.0-2445.gd2c45d97.e 3.7 MB/s | 222 kB 00:00 2024-04-25T00:20:18.301 INFO:teuthology.orchestra.run.smithi032.stdout:(12/152): cephfs-shell-19.0.0-2445.gd2c45d97.el 406 kB/s | 27 kB 00:00 2024-04-25T00:20:18.376 INFO:teuthology.orchestra.run.smithi032.stdout:(13/152): libcephfs-devel-19.0.0-2445.gd2c45d97 423 kB/s | 32 kB 00:00 2024-04-25T00:20:18.576 INFO:teuthology.orchestra.run.smithi130.stdout:(30/152): rbd-mirror-19.0.0-2445.gd2c45d97.el9. 4.8 MB/s | 3.1 MB 00:00 2024-04-25T00:20:18.593 INFO:teuthology.orchestra.run.smithi032.stdout:(14/152): libcephfs2-19.0.0-2445.gd2c45d97.el9. 3.4 MB/s | 745 kB 00:00 2024-04-25T00:20:18.667 INFO:teuthology.orchestra.run.smithi130.stdout:(31/152): ceph-mgr-dashboard-19.0.0-2445.gd2c45 7.4 MB/s | 3.5 MB 00:00 2024-04-25T00:20:18.676 INFO:teuthology.orchestra.run.smithi032.stdout:(15/152): libcephsqlite-19.0.0-2445.gd2c45d97.e 1.9 MB/s | 163 kB 00:00 2024-04-25T00:20:18.734 INFO:teuthology.orchestra.run.smithi130.stdout:(32/152): ceph-mgr-modules-core-19.0.0-2445.gd2 3.6 MB/s | 249 kB 00:00 2024-04-25T00:20:18.760 INFO:teuthology.orchestra.run.smithi032.stdout:(16/152): librados-devel-19.0.0-2445.gd2c45d97. 1.5 MB/s | 127 kB 00:00 2024-04-25T00:20:18.801 INFO:teuthology.orchestra.run.smithi130.stdout:(33/152): ceph-mgr-rook-19.0.0-2445.gd2c45d97.e 742 kB/s | 49 kB 00:00 2024-04-25T00:20:18.869 INFO:teuthology.orchestra.run.smithi130.stdout:(34/152): ceph-prometheus-alerts-19.0.0-2445.gd 228 kB/s | 15 kB 00:00 2024-04-25T00:20:18.894 INFO:teuthology.orchestra.run.smithi032.stdout:(17/152): libradosstriper1-19.0.0-2445.gd2c45d9 3.7 MB/s | 501 kB 00:00 2024-04-25T00:20:18.944 INFO:teuthology.orchestra.run.smithi130.stdout:(35/152): ceph-volume-19.0.0-2445.gd2c45d97.el9 3.4 MB/s | 264 kB 00:00 2024-04-25T00:20:19.111 INFO:teuthology.orchestra.run.smithi130.stdout:(36/152): cephadm-19.0.0-2445.gd2c45d97.el9.noa 4.5 MB/s | 762 kB 00:00 2024-04-25T00:20:19.203 INFO:teuthology.orchestra.run.smithi130.stdout:(37/152): cephfs-top-19.0.0-2445.gd2c45d97.el9. 199 kB/s | 18 kB 00:00 2024-04-25T00:20:19.320 INFO:teuthology.orchestra.run.smithi130.stdout:(38/152): fuse-2.9.9-15.el9.x86_64.rpm 689 kB/s | 80 kB 00:00 2024-04-25T00:20:19.445 INFO:teuthology.orchestra.run.smithi130.stdout:(39/152): libgfortran-11.4.1-3.el9.x86_64.rpm 6.3 MB/s | 803 kB 00:00 2024-04-25T00:20:19.495 INFO:teuthology.orchestra.run.smithi130.stdout:(40/152): libquadmath-11.4.1-3.el9.x86_64.rpm 3.8 MB/s | 195 kB 00:00 2024-04-25T00:20:19.527 INFO:teuthology.orchestra.run.smithi121.stdout:CentOS Stream 9 - Extras packages 20 kB/s | 16 kB 00:00 2024-04-25T00:20:19.546 INFO:teuthology.orchestra.run.smithi130.stdout:(41/152): python3-cffi-1.14.5-5.el9.x86_64.rpm 4.9 MB/s | 253 kB 00:00 2024-04-25T00:20:19.671 INFO:teuthology.orchestra.run.smithi130.stdout:(42/152): python3-cryptography-36.0.1-4.el9.x86 9.8 MB/s | 1.2 MB 00:00 2024-04-25T00:20:19.713 INFO:teuthology.orchestra.run.smithi130.stdout:(43/152): python3-ply-3.11-14.el9.noarch.rpm 2.5 MB/s | 106 kB 00:00 2024-04-25T00:20:19.764 INFO:teuthology.orchestra.run.smithi130.stdout:(44/152): python3-pycparser-2.20-6.el9.noarch.r 2.6 MB/s | 135 kB 00:00 2024-04-25T00:20:19.814 INFO:teuthology.orchestra.run.smithi130.stdout:(45/152): python3-requests-2.25.1-8.el9.noarch. 2.4 MB/s | 125 kB 00:00 2024-04-25T00:20:19.864 INFO:teuthology.orchestra.run.smithi130.stdout:(46/152): python3-urllib3-1.26.5-5.el9.noarch.r 4.2 MB/s | 215 kB 00:00 2024-04-25T00:20:19.915 INFO:teuthology.orchestra.run.smithi032.stdout:(18/152): librgw2-19.0.0-2445.gd2c45d97.el9.x86 5.2 MB/s | 5.3 MB 00:01 2024-04-25T00:20:20.053 INFO:teuthology.orchestra.run.smithi032.stdout:(19/152): python3-ceph-argparse-19.0.0-2445.gd2 328 kB/s | 45 kB 00:00 2024-04-25T00:20:20.081 INFO:teuthology.orchestra.run.smithi130.stdout:(47/152): ceph-mgr-diskprediction-local-19.0.0- 4.9 MB/s | 7.4 MB 00:01 2024-04-25T00:20:20.111 INFO:teuthology.orchestra.run.smithi032.stdout:(20/152): python3-ceph-common-19.0.0-2445.gd2c4 2.2 MB/s | 133 kB 00:00 2024-04-25T00:20:20.170 INFO:teuthology.orchestra.run.smithi032.stdout:(21/152): python3-cephfs-19.0.0-2445.gd2c45d97. 2.7 MB/s | 162 kB 00:00 2024-04-25T00:20:20.240 INFO:teuthology.orchestra.run.smithi130.stdout:(48/152): automake-1.16.2-8.el9.noarch.rpm 1.8 MB/s | 689 kB 00:00 2024-04-25T00:20:20.262 INFO:teuthology.orchestra.run.smithi032.stdout:(22/152): python3-rados-19.0.0-2445.gd2c45d97.e 3.4 MB/s | 321 kB 00:00 2024-04-25T00:20:20.299 INFO:teuthology.orchestra.run.smithi130.stdout:(49/152): boost-program-options-1.75.0-8.el9.x8 1.8 MB/s | 107 kB 00:00 2024-04-25T00:20:20.329 INFO:teuthology.orchestra.run.smithi032.stdout:(23/152): python3-rbd-19.0.0-2445.gd2c45d97.el9 4.4 MB/s | 299 kB 00:00 2024-04-25T00:20:20.357 INFO:teuthology.orchestra.run.smithi130.stdout:(50/152): elfutils-libelf-devel-0.190-2.el9.x86 367 kB/s | 21 kB 00:00 2024-04-25T00:20:20.388 INFO:teuthology.orchestra.run.smithi032.stdout:(24/152): python3-rgw-19.0.0-2445.gd2c45d97.el9 1.7 MB/s | 99 kB 00:00 2024-04-25T00:20:20.424 INFO:teuthology.orchestra.run.smithi130.stdout:(51/152): bison-3.7.4-5.el9.x86_64.rpm 2.7 MB/s | 943 kB 00:00 2024-04-25T00:20:20.438 INFO:teuthology.orchestra.run.smithi032.stdout:(25/152): rbd-fuse-19.0.0-2445.gd2c45d97.el9.x8 1.7 MB/s | 85 kB 00:00 2024-04-25T00:20:20.449 INFO:teuthology.orchestra.run.smithi130.stdout:(52/152): flex-2.6.4-9.el9.x86_64.rpm 3.3 MB/s | 313 kB 00:00 2024-04-25T00:20:20.483 INFO:teuthology.orchestra.run.smithi130.stdout:(53/152): flexiblas-3.0.4-8.el9.x86_64.rpm 560 kB/s | 33 kB 00:00 2024-04-25T00:20:20.542 INFO:teuthology.orchestra.run.smithi130.stdout:(54/152): flexiblas-openblas-openmp-3.0.4-8.el9 311 kB/s | 18 kB 00:00 2024-04-25T00:20:20.600 INFO:teuthology.orchestra.run.smithi130.stdout:(55/152): libaio-devel-0.3.111-13.el9.x86_64.rp 191 kB/s | 11 kB 00:00 2024-04-25T00:20:20.659 INFO:teuthology.orchestra.run.smithi130.stdout:(56/152): libpmemobj-1.12.1-1.el9.x86_64.rpm 2.7 MB/s | 160 kB 00:00 2024-04-25T00:20:20.718 INFO:teuthology.orchestra.run.smithi130.stdout:(57/152): librabbitmq-0.11.0-7.el9.x86_64.rpm 778 kB/s | 45 kB 00:00 2024-04-25T00:20:20.859 INFO:teuthology.orchestra.run.smithi130.stdout:(58/152): librdkafka-1.6.1-102.el9.x86_64.rpm 4.6 MB/s | 662 kB 00:00 2024-04-25T00:20:20.914 INFO:teuthology.orchestra.run.smithi032.stdout:(26/152): rbd-mirror-19.0.0-2445.gd2c45d97.el9. 6.5 MB/s | 3.1 MB 00:00 2024-04-25T00:20:20.943 INFO:teuthology.orchestra.run.smithi130.stdout:(59/152): flexiblas-netlib-3.0.4-8.el9.x86_64.r 6.1 MB/s | 3.0 MB 00:00 2024-04-25T00:20:20.972 INFO:teuthology.orchestra.run.smithi032.stdout:(27/152): rbd-nbd-19.0.0-2445.gd2c45d97.el9.x86 2.9 MB/s | 171 kB 00:00 2024-04-25T00:20:21.010 INFO:teuthology.orchestra.run.smithi130.stdout:(60/152): libtool-2.4.6-45.el9.x86_64.rpm 3.8 MB/s | 582 kB 00:00 2024-04-25T00:20:21.022 INFO:teuthology.orchestra.run.smithi032.stdout:(28/152): ceph-grafana-dashboards-19.0.0-2445.g 636 kB/s | 32 kB 00:00 2024-04-25T00:20:21.027 INFO:teuthology.orchestra.run.smithi130.stdout:(61/152): libuuid-devel-2.37.4-18.el9.x86_64.rp 304 kB/s | 25 kB 00:00 2024-04-25T00:20:21.069 INFO:teuthology.orchestra.run.smithi130.stdout:(62/152): libzstd-devel-1.5.1-2.el9.x86_64.rpm 809 kB/s | 47 kB 00:00 2024-04-25T00:20:21.089 INFO:teuthology.orchestra.run.smithi032.stdout:(29/152): ceph-mgr-cephadm-19.0.0-2445.gd2c45d9 2.1 MB/s | 141 kB 00:00 2024-04-25T00:20:21.094 INFO:teuthology.orchestra.run.smithi130.stdout:(63/152): lttng-ust-2.12.0-6.el9.x86_64.rpm 4.3 MB/s | 292 kB 00:00 2024-04-25T00:20:21.128 INFO:teuthology.orchestra.run.smithi130.stdout:(64/152): lua-5.4.4-4.el9.x86_64.rpm 3.2 MB/s | 188 kB 00:00 2024-04-25T00:20:21.145 INFO:teuthology.orchestra.run.smithi130.stdout:(65/152): openblas-0.3.21-2.el9.x86_64.rpm 700 kB/s | 35 kB 00:00 2024-04-25T00:20:21.239 INFO:teuthology.orchestra.run.smithi121.stdout:Extra Packages for Enterprise Linux 13 MB/s | 21 MB 00:01 2024-04-25T00:20:21.537 INFO:teuthology.orchestra.run.smithi130.stdout:(66/152): openblas-openmp-0.3.21-2.el9.x86_64.r 12 MB/s | 4.7 MB 00:00 2024-04-25T00:20:21.607 INFO:teuthology.orchestra.run.smithi032.stdout:(30/152): ceph-mgr-dashboard-19.0.0-2445.gd2c45 6.8 MB/s | 3.5 MB 00:00 2024-04-25T00:20:21.987 INFO:teuthology.orchestra.run.smithi130.stdout:(67/152): ceph-test-19.0.0-2445.gd2c45d97.el9.x 6.7 MB/s | 48 MB 00:07 2024-04-25T00:20:22.054 INFO:teuthology.orchestra.run.smithi130.stdout:(68/152): openssl-devel-3.2.1-1.el9.x86_64.rpm 4.9 MB/s | 4.4 MB 00:00 2024-04-25T00:20:22.096 INFO:teuthology.orchestra.run.smithi130.stdout:(69/152): postgresql-13.14-1.el9.x86_64.rpm 2.8 MB/s | 1.6 MB 00:00 2024-04-25T00:20:22.138 INFO:teuthology.orchestra.run.smithi130.stdout:(70/152): postgresql-private-libs-13.14-1.el9.x 1.6 MB/s | 133 kB 00:00 2024-04-25T00:20:22.280 INFO:teuthology.orchestra.run.smithi130.stdout:(71/152): protobuf-3.14.0-13.el9.x86_64.rpm 7.1 MB/s | 1.0 MB 00:00 2024-04-25T00:20:22.338 INFO:teuthology.orchestra.run.smithi130.stdout:(72/152): python3-attrs-20.3.0-7.el9.noarch.rpm 1.5 MB/s | 87 kB 00:00 2024-04-25T00:20:22.373 INFO:teuthology.orchestra.run.smithi032.stdout:(31/152): ceph-mgr-diskprediction-local-19.0.0- 9.6 MB/s | 7.4 MB 00:00 2024-04-25T00:20:22.547 INFO:teuthology.orchestra.run.smithi130.stdout:(73/152): postgresql-contrib-13.14-1.el9.x86_64 1.6 MB/s | 911 kB 00:00 2024-04-25T00:20:22.566 INFO:teuthology.orchestra.run.smithi032.stdout:(32/152): ceph-common-19.0.0-2445.gd2c45d97.el9 2.5 MB/s | 21 MB 00:08 2024-04-25T00:20:22.591 INFO:teuthology.orchestra.run.smithi032.stdout:(33/152): ceph-mgr-modules-core-19.0.0-2445.gd2 1.1 MB/s | 249 kB 00:00 2024-04-25T00:20:22.624 INFO:teuthology.orchestra.run.smithi032.stdout:(34/152): ceph-mgr-rook-19.0.0-2445.gd2c45d97.e 846 kB/s | 49 kB 00:00 2024-04-25T00:20:22.649 INFO:teuthology.orchestra.run.smithi032.stdout:(35/152): ceph-prometheus-alerts-19.0.0-2445.gd 262 kB/s | 15 kB 00:00 2024-04-25T00:20:22.683 INFO:teuthology.orchestra.run.smithi032.stdout:(36/152): ceph-volume-19.0.0-2445.gd2c45d97.el9 4.4 MB/s | 264 kB 00:00 2024-04-25T00:20:22.697 INFO:teuthology.orchestra.run.smithi130.stdout:(74/152): postgresql-server-13.14-1.el9.x86_64. 9.6 MB/s | 5.8 MB 00:00 2024-04-25T00:20:22.723 INFO:teuthology.orchestra.run.smithi130.stdout:(75/152): python3-devel-3.9.18-3.el9.x86_64.rpm 1.4 MB/s | 244 kB 00:00 2024-04-25T00:20:22.725 INFO:teuthology.orchestra.run.smithi032.stdout:(37/152): cephfs-top-19.0.0-2445.gd2c45d97.el9. 439 kB/s | 18 kB 00:00 2024-04-25T00:20:22.757 INFO:teuthology.orchestra.run.smithi130.stdout:(76/152): python3-jinja2-2.11.3-5.el9.noarch.rp 4.1 MB/s | 248 kB 00:00 2024-04-25T00:20:22.781 INFO:teuthology.orchestra.run.smithi130.stdout:(77/152): python3-jsonpatch-1.21-16.el9.noarch. 438 kB/s | 26 kB 00:00 2024-04-25T00:20:22.815 INFO:teuthology.orchestra.run.smithi130.stdout:(78/152): python3-jsonpointer-2.0-4.el9.noarch. 327 kB/s | 19 kB 00:00 2024-04-25T00:20:22.825 INFO:teuthology.orchestra.run.smithi032.stdout:(38/152): fuse-2.9.9-15.el9.x86_64.rpm 804 kB/s | 80 kB 00:00 2024-04-25T00:20:22.859 INFO:teuthology.orchestra.run.smithi032.stdout:(39/152): cephadm-19.0.0-2445.gd2c45d97.el9.noa 3.6 MB/s | 762 kB 00:00 2024-04-25T00:20:22.874 INFO:teuthology.orchestra.run.smithi130.stdout:(79/152): python3-markupsafe-1.1.1-12.el9.x86_6 596 kB/s | 35 kB 00:00 2024-04-25T00:20:22.899 INFO:teuthology.orchestra.run.smithi130.stdout:(80/152): python3-mako-1.1.4-6.el9.noarch.rpm 1.4 MB/s | 172 kB 00:00 2024-04-25T00:20:22.934 INFO:teuthology.orchestra.run.smithi032.stdout:(40/152): libgfortran-11.4.1-3.el9.x86_64.rpm 7.2 MB/s | 803 kB 00:00 2024-04-25T00:20:22.974 INFO:teuthology.orchestra.run.smithi130.stdout:(81/152): python3-babel-2.9.1-2.el9.noarch.rpm 9.4 MB/s | 6.0 MB 00:00 2024-04-25T00:20:23.025 INFO:teuthology.orchestra.run.smithi130.stdout:(82/152): python3-numpy-f2py-1.20.1-5.el9.x86_6 2.0 MB/s | 250 kB 00:00 2024-04-25T00:20:23.050 INFO:teuthology.orchestra.run.smithi130.stdout:(83/152): python3-oauthlib-3.1.1-5.el9.noarch.r 2.9 MB/s | 222 kB 00:00 2024-04-25T00:20:23.084 INFO:teuthology.orchestra.run.smithi130.stdout:(84/152): python3-packaging-20.9-5.el9.noarch.r 1.3 MB/s | 77 kB 00:00 2024-04-25T00:20:23.109 INFO:teuthology.orchestra.run.smithi130.stdout:(85/152): python3-prettytable-0.7.2-27.el9.noar 723 kB/s | 42 kB 00:00 2024-04-25T00:20:23.168 INFO:teuthology.orchestra.run.smithi130.stdout:(86/152): python3-pyasn1-0.4.8-6.el9.noarch.rpm 2.7 MB/s | 159 kB 00:00 2024-04-25T00:20:23.193 INFO:teuthology.orchestra.run.smithi130.stdout:(87/152): python3-protobuf-3.14.0-13.el9.noarch 2.4 MB/s | 269 kB 00:00 2024-04-25T00:20:23.227 INFO:teuthology.orchestra.run.smithi130.stdout:(88/152): python3-pyasn1-modules-0.4.8-6.el9.no 4.7 MB/s | 279 kB 00:00 2024-04-25T00:20:23.252 INFO:teuthology.orchestra.run.smithi130.stdout:(89/152): python3-pytz-2021.1-5.el9.noarch.rpm 871 kB/s | 51 kB 00:00 2024-04-25T00:20:23.327 INFO:teuthology.orchestra.run.smithi130.stdout:(90/152): python3-numpy-1.20.1-5.el9.x86_64.rpm 11 MB/s | 5.1 MB 00:00 2024-04-25T00:20:23.352 INFO:teuthology.orchestra.run.smithi130.stdout:(91/152): python3-requests-oauthlib-1.3.0-12.el 427 kB/s | 54 kB 00:00 2024-04-25T00:20:23.360 INFO:teuthology.orchestra.run.smithi032.stdout:(41/152): ceph-test-19.0.0-2445.gd2c45d97.el9.x 9.3 MB/s | 48 MB 00:05 2024-04-25T00:20:23.385 INFO:teuthology.orchestra.run.smithi032.stdout:(42/152): libquadmath-11.4.1-3.el9.x86_64.rpm 371 kB/s | 195 kB 00:00 2024-04-25T00:20:23.386 INFO:teuthology.orchestra.run.smithi130.stdout:(92/152): python3-toml-0.10.2-6.el9.noarch.rpm 714 kB/s | 42 kB 00:00 2024-04-25T00:20:23.410 INFO:teuthology.orchestra.run.smithi032.stdout:(43/152): python3-cffi-1.14.5-5.el9.x86_64.rpm 532 kB/s | 253 kB 00:00 2024-04-25T00:20:23.411 INFO:teuthology.orchestra.run.smithi130.stdout:(93/152): python3-wcwidth-0.2.5-8.el9.noarch.rp 753 kB/s | 44 kB 00:00 2024-04-25T00:20:23.445 INFO:teuthology.orchestra.run.smithi130.stdout:(94/152): qatlib-24.02.0-1.el9.x86_64.rpm 3.7 MB/s | 221 kB 00:00 2024-04-25T00:20:23.452 INFO:teuthology.orchestra.run.smithi032.stdout:(44/152): python3-ply-3.11-14.el9.noarch.rpm 1.6 MB/s | 106 kB 00:00 2024-04-25T00:20:23.469 INFO:teuthology.orchestra.run.smithi032.stdout:(45/152): python3-pycparser-2.20-6.el9.noarch.r 2.3 MB/s | 135 kB 00:00 2024-04-25T00:20:23.470 INFO:teuthology.orchestra.run.smithi130.stdout:(95/152): qatlib-service-24.02.0-1.el9.x86_64.r 614 kB/s | 36 kB 00:00 2024-04-25T00:20:23.503 INFO:teuthology.orchestra.run.smithi032.stdout:(46/152): python3-requests-2.25.1-8.el9.noarch. 2.4 MB/s | 125 kB 00:00 2024-04-25T00:20:23.503 INFO:teuthology.orchestra.run.smithi130.stdout:(96/152): qatzip-libs-1.2.0-1.el9.x86_64.rpm 812 kB/s | 47 kB 00:00 2024-04-25T00:20:23.528 INFO:teuthology.orchestra.run.smithi032.stdout:(47/152): python3-urllib3-1.26.5-5.el9.noarch.r 3.6 MB/s | 215 kB 00:00 2024-04-25T00:20:23.531 INFO:teuthology.orchestra.run.smithi130.stdout:(97/152): socat-1.7.4.1-5.el9.x86_64.rpm 4.9 MB/s | 305 kB 00:00 2024-04-25T00:20:23.565 INFO:teuthology.orchestra.run.smithi130.stdout:(98/152): uuid-1.6.2-55.el9.x86_64.rpm 951 kB/s | 58 kB 00:00 2024-04-25T00:20:23.587 INFO:teuthology.orchestra.run.smithi032.stdout:(48/152): python3-cryptography-36.0.1-4.el9.x86 5.4 MB/s | 1.2 MB 00:00 2024-04-25T00:20:23.590 INFO:teuthology.orchestra.run.smithi130.stdout:(99/152): xfsprogs-devel-6.3.0-1.el9.x86_64.rpm 1.6 MB/s | 94 kB 00:00 2024-04-25T00:20:23.615 INFO:teuthology.orchestra.run.smithi130.stdout:(100/152): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.2 MB/s | 64 kB 00:00 2024-04-25T00:20:23.640 INFO:teuthology.orchestra.run.smithi130.stdout:(101/152): zlib-devel-1.2.11-41.el9.x86_64.rpm 892 kB/s | 45 kB 00:00 2024-04-25T00:20:23.691 INFO:teuthology.orchestra.run.smithi130.stdout:(102/152): lua-devel-5.4.4-4.el9.x86_64.rpm 297 kB/s | 22 kB 00:00 2024-04-25T00:20:23.791 INFO:teuthology.orchestra.run.smithi130.stdout:(103/152): abseil-cpp-20211102.0-3.el9.x86_64.r 5.4 MB/s | 548 kB 00:00 2024-04-25T00:20:23.825 INFO:teuthology.orchestra.run.smithi130.stdout:(104/152): protobuf-compiler-3.14.0-13.el9.x86_ 4.6 MB/s | 863 kB 00:00 2024-04-25T00:20:23.858 INFO:teuthology.orchestra.run.smithi130.stdout:(105/152): gperftools-libs-2.9.1-2.el9.x86_64.r 4.5 MB/s | 309 kB 00:00 2024-04-25T00:20:23.900 INFO:teuthology.orchestra.run.smithi130.stdout:(106/152): grpc-data-1.46.7-10.el9.noarch.rpm 258 kB/s | 19 kB 00:00 2024-04-25T00:20:23.976 INFO:teuthology.orchestra.run.smithi130.stdout:(107/152): libarrow-9.0.0-10.el9.x86_64.rpm 38 MB/s | 4.4 MB 00:00 2024-04-25T00:20:24.001 INFO:teuthology.orchestra.run.smithi130.stdout:(108/152): libarrow-doc-9.0.0-10.el9.noarch.rpm 258 kB/s | 26 kB 00:00 2024-04-25T00:20:24.012 INFO:teuthology.orchestra.run.smithi032.stdout:(49/152): boost-program-options-1.75.0-8.el9.x8 251 kB/s | 107 kB 00:00 2024-04-25T00:20:24.026 INFO:teuthology.orchestra.run.smithi130.stdout:(109/152): liboath-2.6.7-2.el9.x86_64.rpm 971 kB/s | 49 kB 00:00 2024-04-25T00:20:24.052 INFO:teuthology.orchestra.run.smithi130.stdout:(110/152): libunwind-1.6.2-1.el9.x86_64.rpm 1.3 MB/s | 67 kB 00:00 2024-04-25T00:20:24.077 INFO:teuthology.orchestra.run.smithi130.stdout:(111/152): luarocks-3.9.2-1.el9.noarch.rpm 2.9 MB/s | 151 kB 00:00 2024-04-25T00:20:24.119 INFO:teuthology.orchestra.run.smithi130.stdout:(112/152): python3-asyncssh-2.13.2-5.el9.noarch 13 MB/s | 548 kB 00:00 2024-04-25T00:20:24.152 INFO:teuthology.orchestra.run.smithi130.stdout:(113/152): python3-bcrypt-3.2.2-1.el9.x86_64.rp 1.3 MB/s | 43 kB 00:00 2024-04-25T00:20:24.186 INFO:teuthology.orchestra.run.smithi130.stdout:(114/152): parquet-libs-9.0.0-10.el9.x86_64.rpm 6.1 MB/s | 838 kB 00:00 2024-04-25T00:20:24.211 INFO:teuthology.orchestra.run.smithi130.stdout:(115/152): python3-cachetools-4.2.4-1.el9.noarc 547 kB/s | 32 kB 00:00 2024-04-25T00:20:24.245 INFO:teuthology.orchestra.run.smithi130.stdout:(116/152): python3-certifi-2021.10.8-2.el9.noar 256 kB/s | 15 kB 00:00 2024-04-25T00:20:24.270 INFO:teuthology.orchestra.run.smithi032.stdout:(50/152): elfutils-libelf-devel-0.190-2.el9.x86 83 kB/s | 21 kB 00:00 2024-04-25T00:20:24.278 INFO:teuthology.orchestra.run.smithi130.stdout:(117/152): python3-cheroot-8.6.0-4.el9.noarch.r 2.5 MB/s | 172 kB 00:00 2024-04-25T00:20:24.312 INFO:teuthology.orchestra.run.smithi130.stdout:(118/152): python3-cherrypy-18.6.1-2.el9.noarch 5.2 MB/s | 358 kB 00:00 2024-04-25T00:20:24.346 INFO:teuthology.orchestra.run.smithi130.stdout:(119/152): python3-cmd2-2.4.2-2.el9.noarch.rpm 4.4 MB/s | 303 kB 00:00 2024-04-25T00:20:24.380 INFO:teuthology.orchestra.run.smithi130.stdout:(120/152): python3-colorama-0.4.4-12.el9.noarch 528 kB/s | 35 kB 00:00 2024-04-25T00:20:24.414 INFO:teuthology.orchestra.run.smithi130.stdout:(121/152): python3-google-auth-2.29.0-1.el9.noa 3.0 MB/s | 206 kB 00:00 2024-04-25T00:20:24.480 INFO:teuthology.orchestra.run.smithi130.stdout:(122/152): python3-grpcio-tools-1.46.7-10.el9.x 2.1 MB/s | 144 kB 00:00 2024-04-25T00:20:24.547 INFO:teuthology.orchestra.run.smithi130.stdout:(123/152): python3-jaraco-8.2.1-3.el9.noarch.rp 160 kB/s | 11 kB 00:00 2024-04-25T00:20:24.614 INFO:teuthology.orchestra.run.smithi130.stdout:(124/152): python3-grpcio-1.46.7-10.el9.x86_64. 8.7 MB/s | 2.0 MB 00:00 2024-04-25T00:20:24.640 INFO:teuthology.orchestra.run.smithi130.stdout:(125/152): python3-jaraco-classes-3.2.1-5.el9.n 191 kB/s | 18 kB 00:00 2024-04-25T00:20:24.673 INFO:teuthology.orchestra.run.smithi130.stdout:(126/152): python3-jaraco-collections-3.0.0-8.e 394 kB/s | 23 kB 00:00 2024-04-25T00:20:24.698 INFO:teuthology.orchestra.run.smithi130.stdout:(127/152): python3-jaraco-functools-3.5.0-2.el9 332 kB/s | 19 kB 00:00 2024-04-25T00:20:24.715 INFO:teuthology.orchestra.run.smithi130.stdout:(128/152): python3-jaraco-text-3.2.0-6.el9.noar 469 kB/s | 20 kB 00:00 2024-04-25T00:20:24.757 INFO:teuthology.orchestra.run.smithi130.stdout:(129/152): python3-kubernetes-26.1.0-2.el9.noar 17 MB/s | 1.0 MB 00:00 2024-04-25T00:20:24.783 INFO:teuthology.orchestra.run.smithi130.stdout:(130/152): python3-logutils-0.3.5-21.el9.noarch 691 kB/s | 46 kB 00:00 2024-04-25T00:20:24.808 INFO:teuthology.orchestra.run.smithi130.stdout:(131/152): python3-more-itertools-8.12.0-2.el9. 1.5 MB/s | 79 kB 00:00 2024-04-25T00:20:24.833 INFO:teuthology.orchestra.run.smithi130.stdout:(132/152): python3-natsort-7.1.1-5.el9.noarch.r 1.1 MB/s | 58 kB 00:00 2024-04-25T00:20:24.858 INFO:teuthology.orchestra.run.smithi130.stdout:(133/152): python3-pecan-1.4.2-3.el9.noarch.rpm 5.3 MB/s | 272 kB 00:00 2024-04-25T00:20:24.884 INFO:teuthology.orchestra.run.smithi130.stdout:(134/152): python3-portend-3.1.0-2.el9.noarch.r 326 kB/s | 16 kB 00:00 2024-04-25T00:20:24.887 INFO:teuthology.orchestra.run.smithi032.stdout:(51/152): automake-1.16.2-8.el9.noarch.rpm 498 kB/s | 689 kB 00:01 2024-04-25T00:20:24.909 INFO:teuthology.orchestra.run.smithi130.stdout:(135/152): python3-pyOpenSSL-21.0.0-1.el9.noarc 1.7 MB/s | 90 kB 00:00 2024-04-25T00:20:24.934 INFO:teuthology.orchestra.run.smithi130.stdout:(136/152): python3-pyperclip-1.8.2-2.el9.noarch 563 kB/s | 28 kB 00:00 2024-04-25T00:20:24.960 INFO:teuthology.orchestra.run.smithi130.stdout:(137/152): python3-repoze-lru-0.7-16.el9.noarch 612 kB/s | 31 kB 00:00 2024-04-25T00:20:24.985 INFO:teuthology.orchestra.run.smithi130.stdout:(138/152): python3-routes-2.5.1-5.el9.noarch.rp 3.7 MB/s | 188 kB 00:00 2024-04-25T00:20:25.011 INFO:teuthology.orchestra.run.smithi130.stdout:(139/152): python3-rsa-4.9-2.el9.noarch.rpm 1.1 MB/s | 59 kB 00:00 2024-04-25T00:20:25.012 INFO:teuthology.orchestra.run.smithi032.stdout:(52/152): flex-2.6.4-9.el9.x86_64.rpm 423 kB/s | 313 kB 00:00 2024-04-25T00:20:25.035 INFO:teuthology.orchestra.run.smithi130.stdout:(140/152): python3-tempora-5.0.0-2.el9.noarch.r 714 kB/s | 36 kB 00:00 2024-04-25T00:20:25.061 INFO:teuthology.orchestra.run.smithi130.stdout:(141/152): python3-typing-extensions-4.4.0-2.el 1.0 MB/s | 51 kB 00:00 2024-04-25T00:20:25.086 INFO:teuthology.orchestra.run.smithi130.stdout:(142/152): python3-webob-1.8.7-6.el9.noarch.rpm 4.5 MB/s | 230 kB 00:00 2024-04-25T00:20:25.111 INFO:teuthology.orchestra.run.smithi130.stdout:(143/152): python3-websocket-client-1.2.3-2.el9 1.7 MB/s | 90 kB 00:00 2024-04-25T00:20:25.137 INFO:teuthology.orchestra.run.smithi130.stdout:(144/152): python3-werkzeug-2.0.3-3.el9.1.noarc 8.3 MB/s | 427 kB 00:00 2024-04-25T00:20:25.155 INFO:teuthology.orchestra.run.smithi032.stdout:(53/152): flexiblas-3.0.4-8.el9.x86_64.rpm 122 kB/s | 33 kB 00:00 2024-04-25T00:20:25.162 INFO:teuthology.orchestra.run.smithi130.stdout:(145/152): python3-zc-lockfile-2.0-10.el9.noarc 397 kB/s | 20 kB 00:00 2024-04-25T00:20:25.187 INFO:teuthology.orchestra.run.smithi130.stdout:(146/152): re2-20211101-3.el9.x86_64.rpm 3.7 MB/s | 192 kB 00:00 2024-04-25T00:20:25.238 INFO:teuthology.orchestra.run.smithi130.stdout:(147/152): thrift-0.15.0-2.el9.x86_64.rpm 21 MB/s | 1.6 MB 00:00 2024-04-25T00:20:25.413 INFO:teuthology.orchestra.run.smithi130.stdout:(148/152): python3-scipy-1.6.2-8.el9.x86_64.rpm 7.5 MB/s | 16 MB 00:02 2024-04-25T00:20:25.414 INFO:teuthology.orchestra.run.smithi032.stdout:(54/152): flexiblas-openblas-openmp-3.0.4-8.el9 70 kB/s | 18 kB 00:00 2024-04-25T00:20:25.589 INFO:teuthology.orchestra.run.smithi032.stdout:(55/152): libaio-devel-0.3.111-13.el9.x86_64.rp 64 kB/s | 11 kB 00:00 2024-04-25T00:20:25.663 INFO:teuthology.orchestra.run.smithi130.stdout:(149/152): openssl-3.2.1-1.el9.x86_64.rpm 5.4 MB/s | 1.3 MB 00:00 2024-04-25T00:20:25.755 INFO:teuthology.orchestra.run.smithi130.stdout:(150/152): librados2-19.0.0-2445.gd2c45d97.el9. 6.0 MB/s | 3.4 MB 00:00 2024-04-25T00:20:25.864 INFO:teuthology.orchestra.run.smithi130.stdout:(151/152): librbd1-19.0.0-2445.gd2c45d97.el9.x8 5.0 MB/s | 3.1 MB 00:00 2024-04-25T00:20:25.906 INFO:teuthology.orchestra.run.smithi130.stdout:(152/152): openssl-libs-3.2.1-1.el9.x86_64.rpm 9.9 MB/s | 2.4 MB 00:00 2024-04-25T00:20:25.908 INFO:teuthology.orchestra.run.smithi130.stdout:-------------------------------------------------------------------------------- 2024-04-25T00:20:25.908 INFO:teuthology.orchestra.run.smithi130.stdout:Total 16 MB/s | 220 MB 00:13 2024-04-25T00:20:26.089 INFO:teuthology.orchestra.run.smithi032.stdout:(56/152): bison-3.7.4-5.el9.x86_64.rpm 368 kB/s | 943 kB 00:02 2024-04-25T00:20:26.114 INFO:teuthology.orchestra.run.smithi032.stdout:(57/152): libpmemobj-1.12.1-1.el9.x86_64.rpm 305 kB/s | 160 kB 00:00 2024-04-25T00:20:26.203 INFO:teuthology.orchestra.run.smithi121.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-04-25T00:20:26.431 INFO:teuthology.orchestra.run.smithi032.stdout:(58/152): librabbitmq-0.11.0-7.el9.x86_64.rpm 132 kB/s | 45 kB 00:00 2024-04-25T00:20:27.033 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction check 2024-04-25T00:20:27.173 INFO:teuthology.orchestra.run.smithi032.stdout:(59/152): librdkafka-1.6.1-102.el9.x86_64.rpm 626 kB/s | 662 kB 00:01 2024-04-25T00:20:27.175 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction check succeeded. 2024-04-25T00:20:27.176 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction test 2024-04-25T00:20:27.448 INFO:teuthology.orchestra.run.smithi032.stdout:(60/152): libuuid-devel-2.37.4-18.el9.x86_64.rp 92 kB/s | 25 kB 00:00 2024-04-25T00:20:27.632 INFO:teuthology.orchestra.run.smithi032.stdout:(61/152): libtool-2.4.6-45.el9.x86_64.rpm 485 kB/s | 582 kB 00:01 2024-04-25T00:20:27.791 INFO:teuthology.orchestra.run.smithi032.stdout:(62/152): libzstd-devel-1.5.1-2.el9.x86_64.rpm 139 kB/s | 47 kB 00:00 2024-04-25T00:20:28.074 INFO:teuthology.orchestra.run.smithi032.stdout:(63/152): flexiblas-netlib-3.0.4-8.el9.x86_64.r 1.0 MB/s | 3.0 MB 00:03 2024-04-25T00:20:28.208 INFO:teuthology.orchestra.run.smithi032.stdout:(64/152): lttng-ust-2.12.0-6.el9.x86_64.rpm 508 kB/s | 292 kB 00:00 2024-04-25T00:20:28.308 INFO:teuthology.orchestra.run.smithi032.stdout:(65/152): lua-5.4.4-4.el9.x86_64.rpm 364 kB/s | 188 kB 00:00 2024-04-25T00:20:28.350 INFO:teuthology.orchestra.run.smithi032.stdout:(66/152): openblas-0.3.21-2.el9.x86_64.rpm 128 kB/s | 35 kB 00:00 2024-04-25T00:20:28.354 INFO:teuthology.orchestra.run.smithi121.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-04-25T00:20:28.355 INFO:teuthology.orchestra.run.smithi121.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-04-25T00:20:28.365 INFO:teuthology.orchestra.run.smithi121.stdout:Package NetworkManager-1:1.47.4-1.el9.x86_64 is already installed. 2024-04-25T00:20:28.366 INFO:teuthology.orchestra.run.smithi121.stdout:Package iproute-6.2.0-5.el9.x86_64 is already installed. 2024-04-25T00:20:28.367 INFO:teuthology.orchestra.run.smithi121.stdout:Package util-linux-2.37.4-18.el9.x86_64 is already installed. 2024-04-25T00:20:28.368 INFO:teuthology.orchestra.run.smithi121.stdout:Package libacl-devel-2.3.1-4.el9.x86_64 is already installed. 2024-04-25T00:20:28.369 INFO:teuthology.orchestra.run.smithi121.stdout:Package libattr-devel-2.5.1-3.el9.x86_64 is already installed. 2024-04-25T00:20:28.372 INFO:teuthology.orchestra.run.smithi121.stdout:Package xfsdump-3.1.12-4.el9.x86_64 is already installed. 2024-04-25T00:20:28.373 INFO:teuthology.orchestra.run.smithi121.stdout:Package xfsprogs-6.3.0-1.el9.x86_64 is already installed. 2024-04-25T00:20:28.433 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2024-04-25T00:20:28.447 INFO:teuthology.orchestra.run.smithi121.stdout:======================================================================================= 2024-04-25T00:20:28.447 INFO:teuthology.orchestra.run.smithi121.stdout: Package Arch Version Repository Size 2024-04-25T00:20:28.447 INFO:teuthology.orchestra.run.smithi121.stdout:======================================================================================= 2024-04-25T00:20:28.447 INFO:teuthology.orchestra.run.smithi121.stdout:Installing: 2024-04-25T00:20:28.447 INFO:teuthology.orchestra.run.smithi121.stdout: bison x86_64 3.7.4-5.el9 appstream 943 k 2024-04-25T00:20:28.447 INFO:teuthology.orchestra.run.smithi121.stdout: ceph x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 6.6 k 2024-04-25T00:20:28.447 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-base x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 5.4 M 2024-04-25T00:20:28.447 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-fuse x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 890 k 2024-04-25T00:20:28.447 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-immutable-object-cache x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 145 k 2024-04-25T00:20:28.447 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mgr x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 1.5 M 2024-04-25T00:20:28.448 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mgr-cephadm noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 141 k 2024-04-25T00:20:28.448 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mgr-dashboard noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 3.5 M 2024-04-25T00:20:28.448 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mgr-diskprediction-local noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 7.4 M 2024-04-25T00:20:28.448 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mgr-rook noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 49 k 2024-04-25T00:20:28.448 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-radosgw x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 10 M 2024-04-25T00:20:28.448 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-test x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 48 M 2024-04-25T00:20:28.448 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-volume noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 264 k 2024-04-25T00:20:28.448 INFO:teuthology.orchestra.run.smithi121.stdout: cephadm noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 762 k 2024-04-25T00:20:28.448 INFO:teuthology.orchestra.run.smithi121.stdout: cephfs-mirror x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 222 k 2024-04-25T00:20:28.448 INFO:teuthology.orchestra.run.smithi121.stdout: cephfs-shell x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 27 k 2024-04-25T00:20:28.448 INFO:teuthology.orchestra.run.smithi121.stdout: cephfs-top noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 18 k 2024-04-25T00:20:28.448 INFO:teuthology.orchestra.run.smithi121.stdout: elfutils-libelf-devel x86_64 0.190-2.el9 appstream 21 k 2024-04-25T00:20:28.448 INFO:teuthology.orchestra.run.smithi121.stdout: flex x86_64 2.6.4-9.el9 appstream 313 k 2024-04-25T00:20:28.449 INFO:teuthology.orchestra.run.smithi121.stdout: libaio-devel x86_64 0.3.111-13.el9 appstream 11 k 2024-04-25T00:20:28.449 INFO:teuthology.orchestra.run.smithi121.stdout: libcephfs-devel x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 32 k 2024-04-25T00:20:28.449 INFO:teuthology.orchestra.run.smithi121.stdout: libcephfs2 x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 745 k 2024-04-25T00:20:28.449 INFO:teuthology.orchestra.run.smithi121.stdout: librados-devel x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 127 k 2024-04-25T00:20:28.449 INFO:teuthology.orchestra.run.smithi121.stdout: libtool x86_64 2.4.6-45.el9 appstream 582 k 2024-04-25T00:20:28.449 INFO:teuthology.orchestra.run.smithi121.stdout: libuuid-devel x86_64 2.37.4-18.el9 appstream 25 k 2024-04-25T00:20:28.449 INFO:teuthology.orchestra.run.smithi121.stdout: openssl-devel x86_64 1:3.2.1-1.el9 appstream 4.4 M 2024-04-25T00:20:28.449 INFO:teuthology.orchestra.run.smithi121.stdout: postgresql x86_64 13.14-1.el9 appstream 1.6 M 2024-04-25T00:20:28.449 INFO:teuthology.orchestra.run.smithi121.stdout: postgresql-contrib x86_64 13.14-1.el9 appstream 911 k 2024-04-25T00:20:28.449 INFO:teuthology.orchestra.run.smithi121.stdout: postgresql-server x86_64 13.14-1.el9 appstream 5.8 M 2024-04-25T00:20:28.449 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cephfs x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 162 k 2024-04-25T00:20:28.449 INFO:teuthology.orchestra.run.smithi121.stdout: python3-rados x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 321 k 2024-04-25T00:20:28.449 INFO:teuthology.orchestra.run.smithi121.stdout: python3-rbd x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 299 k 2024-04-25T00:20:28.450 INFO:teuthology.orchestra.run.smithi121.stdout: python3-rgw x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 99 k 2024-04-25T00:20:28.450 INFO:teuthology.orchestra.run.smithi121.stdout: rbd-fuse x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 85 k 2024-04-25T00:20:28.450 INFO:teuthology.orchestra.run.smithi121.stdout: rbd-mirror x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 3.1 M 2024-04-25T00:20:28.450 INFO:teuthology.orchestra.run.smithi121.stdout: rbd-nbd x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 171 k 2024-04-25T00:20:28.450 INFO:teuthology.orchestra.run.smithi121.stdout: xfsprogs-devel x86_64 6.3.0-1.el9 appstream 94 k 2024-04-25T00:20:28.450 INFO:teuthology.orchestra.run.smithi121.stdout:Upgrading: 2024-04-25T00:20:28.450 INFO:teuthology.orchestra.run.smithi121.stdout: librados2 x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 3.4 M 2024-04-25T00:20:28.450 INFO:teuthology.orchestra.run.smithi121.stdout: librbd1 x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 3.1 M 2024-04-25T00:20:28.450 INFO:teuthology.orchestra.run.smithi121.stdout: openssl x86_64 1:3.2.1-1.el9 baseos 1.3 M 2024-04-25T00:20:28.450 INFO:teuthology.orchestra.run.smithi121.stdout: openssl-libs x86_64 1:3.2.1-1.el9 baseos 2.4 M 2024-04-25T00:20:28.450 INFO:teuthology.orchestra.run.smithi121.stdout:Installing dependencies: 2024-04-25T00:20:28.450 INFO:teuthology.orchestra.run.smithi121.stdout: abseil-cpp x86_64 20211102.0-3.el9 epel 548 k 2024-04-25T00:20:28.451 INFO:teuthology.orchestra.run.smithi121.stdout: automake noarch 1.16.2-8.el9 appstream 689 k 2024-04-25T00:20:28.451 INFO:teuthology.orchestra.run.smithi121.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-04-25T00:20:28.451 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-common x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 21 M 2024-04-25T00:20:28.451 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-grafana-dashboards noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 32 k 2024-04-25T00:20:28.451 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mds x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 2.4 M 2024-04-25T00:20:28.451 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mgr-modules-core noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 249 k 2024-04-25T00:20:28.451 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mon x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 4.7 M 2024-04-25T00:20:28.451 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-osd x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 17 M 2024-04-25T00:20:28.451 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-prometheus-alerts noarch 2:19.0.0-2445.gd2c45d97.el9 ceph-noarch 15 k 2024-04-25T00:20:28.451 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-selinux x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 25 k 2024-04-25T00:20:28.451 INFO:teuthology.orchestra.run.smithi121.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-04-25T00:20:28.451 INFO:teuthology.orchestra.run.smithi121.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-04-25T00:20:28.451 INFO:teuthology.orchestra.run.smithi121.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-04-25T00:20:28.451 INFO:teuthology.orchestra.run.smithi121.stdout: fuse x86_64 2.9.9-15.el9 baseos 80 k 2024-04-25T00:20:28.452 INFO:teuthology.orchestra.run.smithi121.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-25T00:20:28.452 INFO:teuthology.orchestra.run.smithi121.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2024-04-25T00:20:28.452 INFO:teuthology.orchestra.run.smithi121.stdout: libarrow x86_64 9.0.0-10.el9 epel 4.4 M 2024-04-25T00:20:28.452 INFO:teuthology.orchestra.run.smithi121.stdout: libarrow-doc noarch 9.0.0-10.el9 epel 26 k 2024-04-25T00:20:28.452 INFO:teuthology.orchestra.run.smithi121.stdout: libcephsqlite x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 163 k 2024-04-25T00:20:28.452 INFO:teuthology.orchestra.run.smithi121.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-25T00:20:28.452 INFO:teuthology.orchestra.run.smithi121.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-25T00:20:28.452 INFO:teuthology.orchestra.run.smithi121.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-04-25T00:20:28.452 INFO:teuthology.orchestra.run.smithi121.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-25T00:20:28.452 INFO:teuthology.orchestra.run.smithi121.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-04-25T00:20:28.452 INFO:teuthology.orchestra.run.smithi121.stdout: libradosstriper1 x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 501 k 2024-04-25T00:20:28.452 INFO:teuthology.orchestra.run.smithi121.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-04-25T00:20:28.452 INFO:teuthology.orchestra.run.smithi121.stdout: librgw2 x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 5.3 M 2024-04-25T00:20:28.452 INFO:teuthology.orchestra.run.smithi121.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-25T00:20:28.453 INFO:teuthology.orchestra.run.smithi121.stdout: libzstd-devel x86_64 1.5.1-2.el9 appstream 47 k 2024-04-25T00:20:28.453 INFO:teuthology.orchestra.run.smithi121.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-04-25T00:20:28.453 INFO:teuthology.orchestra.run.smithi121.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2024-04-25T00:20:28.453 INFO:teuthology.orchestra.run.smithi121.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-04-25T00:20:28.453 INFO:teuthology.orchestra.run.smithi121.stdout: luarocks noarch 3.9.2-1.el9 epel 151 k 2024-04-25T00:20:28.453 INFO:teuthology.orchestra.run.smithi121.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-04-25T00:20:28.453 INFO:teuthology.orchestra.run.smithi121.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-04-25T00:20:28.453 INFO:teuthology.orchestra.run.smithi121.stdout: parquet-libs x86_64 9.0.0-10.el9 epel 838 k 2024-04-25T00:20:28.453 INFO:teuthology.orchestra.run.smithi121.stdout: postgresql-private-libs x86_64 13.14-1.el9 appstream 133 k 2024-04-25T00:20:28.453 INFO:teuthology.orchestra.run.smithi121.stdout: protobuf x86_64 3.14.0-13.el9 appstream 1.0 M 2024-04-25T00:20:28.453 INFO:teuthology.orchestra.run.smithi121.stdout: protobuf-compiler x86_64 3.14.0-13.el9 crb 863 k 2024-04-25T00:20:28.453 INFO:teuthology.orchestra.run.smithi121.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-25T00:20:28.453 INFO:teuthology.orchestra.run.smithi121.stdout: python3-attrs noarch 20.3.0-7.el9 appstream 87 k 2024-04-25T00:20:28.454 INFO:teuthology.orchestra.run.smithi121.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-04-25T00:20:28.454 INFO:teuthology.orchestra.run.smithi121.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-25T00:20:28.454 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-25T00:20:28.454 INFO:teuthology.orchestra.run.smithi121.stdout: python3-ceph-argparse x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 45 k 2024-04-25T00:20:28.454 INFO:teuthology.orchestra.run.smithi121.stdout: python3-ceph-common x86_64 2:19.0.0-2445.gd2c45d97.el9 ceph 133 k 2024-04-25T00:20:28.454 INFO:teuthology.orchestra.run.smithi121.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-25T00:20:28.454 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-04-25T00:20:28.454 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-25T00:20:28.454 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-25T00:20:28.454 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cmd2 noarch 2.4.2-2.el9 epel 303 k 2024-04-25T00:20:28.454 INFO:teuthology.orchestra.run.smithi121.stdout: python3-colorama noarch 0.4.4-12.el9 epel 35 k 2024-04-25T00:20:28.454 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-04-25T00:20:28.454 INFO:teuthology.orchestra.run.smithi121.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-04-25T00:20:28.455 INFO:teuthology.orchestra.run.smithi121.stdout: python3-google-auth noarch 1:2.29.0-1.el9 epel 206 k 2024-04-25T00:20:28.455 INFO:teuthology.orchestra.run.smithi121.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2024-04-25T00:20:28.455 INFO:teuthology.orchestra.run.smithi121.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2024-04-25T00:20:28.455 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-25T00:20:28.455 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-25T00:20:28.455 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-04-25T00:20:28.455 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-25T00:20:28.455 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-25T00:20:28.455 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-25T00:20:28.455 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-04-25T00:20:28.455 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-04-25T00:20:28.455 INFO:teuthology.orchestra.run.smithi121.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-25T00:20:28.455 INFO:teuthology.orchestra.run.smithi121.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-25T00:20:28.455 INFO:teuthology.orchestra.run.smithi121.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-04-25T00:20:28.455 INFO:teuthology.orchestra.run.smithi121.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-04-25T00:20:28.456 INFO:teuthology.orchestra.run.smithi121.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-25T00:20:28.456 INFO:teuthology.orchestra.run.smithi121.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-25T00:20:28.456 INFO:teuthology.orchestra.run.smithi121.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-04-25T00:20:28.456 INFO:teuthology.orchestra.run.smithi121.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-04-25T00:20:28.456 INFO:teuthology.orchestra.run.smithi121.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-04-25T00:20:28.456 INFO:teuthology.orchestra.run.smithi121.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-04-25T00:20:28.456 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-25T00:20:28.456 INFO:teuthology.orchestra.run.smithi121.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-04-25T00:20:28.456 INFO:teuthology.orchestra.run.smithi121.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-25T00:20:28.456 INFO:teuthology.orchestra.run.smithi121.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-04-25T00:20:28.456 INFO:teuthology.orchestra.run.smithi121.stdout: python3-protobuf noarch 3.14.0-13.el9 appstream 269 k 2024-04-25T00:20:28.456 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-25T00:20:28.456 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-04-25T00:20:28.456 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-04-25T00:20:28.456 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-04-25T00:20:28.457 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pyperclip noarch 1.8.2-2.el9 epel 28 k 2024-04-25T00:20:28.457 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-04-25T00:20:28.457 INFO:teuthology.orchestra.run.smithi121.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-25T00:20:28.457 INFO:teuthology.orchestra.run.smithi121.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-25T00:20:28.457 INFO:teuthology.orchestra.run.smithi121.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-04-25T00:20:28.457 INFO:teuthology.orchestra.run.smithi121.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-25T00:20:28.457 INFO:teuthology.orchestra.run.smithi121.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-25T00:20:28.457 INFO:teuthology.orchestra.run.smithi121.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-04-25T00:20:28.457 INFO:teuthology.orchestra.run.smithi121.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-25T00:20:28.457 INFO:teuthology.orchestra.run.smithi121.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-04-25T00:20:28.457 INFO:teuthology.orchestra.run.smithi121.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-25T00:20:28.457 INFO:teuthology.orchestra.run.smithi121.stdout: python3-urllib3 noarch 1.26.5-5.el9 baseos 215 k 2024-04-25T00:20:28.457 INFO:teuthology.orchestra.run.smithi121.stdout: python3-wcwidth noarch 0.2.5-8.el9 appstream 44 k 2024-04-25T00:20:28.457 INFO:teuthology.orchestra.run.smithi121.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-25T00:20:28.457 INFO:teuthology.orchestra.run.smithi121.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-25T00:20:28.458 INFO:teuthology.orchestra.run.smithi121.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-25T00:20:28.458 INFO:teuthology.orchestra.run.smithi121.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-25T00:20:28.458 INFO:teuthology.orchestra.run.smithi121.stdout: qatlib x86_64 24.02.0-1.el9 appstream 221 k 2024-04-25T00:20:28.458 INFO:teuthology.orchestra.run.smithi121.stdout: qatzip-libs x86_64 1.2.0-1.el9 appstream 47 k 2024-04-25T00:20:28.458 INFO:teuthology.orchestra.run.smithi121.stdout: re2 x86_64 1:20211101-3.el9 epel 192 k 2024-04-25T00:20:28.458 INFO:teuthology.orchestra.run.smithi121.stdout: socat x86_64 1.7.4.1-5.el9 appstream 305 k 2024-04-25T00:20:28.458 INFO:teuthology.orchestra.run.smithi121.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-25T00:20:28.458 INFO:teuthology.orchestra.run.smithi121.stdout: uuid x86_64 1.6.2-55.el9 appstream 58 k 2024-04-25T00:20:28.458 INFO:teuthology.orchestra.run.smithi121.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-04-25T00:20:28.458 INFO:teuthology.orchestra.run.smithi121.stdout: zlib-devel x86_64 1.2.11-41.el9 appstream 45 k 2024-04-25T00:20:28.458 INFO:teuthology.orchestra.run.smithi121.stdout:Installing weak dependencies: 2024-04-25T00:20:28.458 INFO:teuthology.orchestra.run.smithi121.stdout: qatlib-service x86_64 24.02.0-1.el9 appstream 36 k 2024-04-25T00:20:28.458 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-25T00:20:28.459 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction Summary 2024-04-25T00:20:28.459 INFO:teuthology.orchestra.run.smithi121.stdout:======================================================================================= 2024-04-25T00:20:28.459 INFO:teuthology.orchestra.run.smithi121.stdout:Install 148 Packages 2024-04-25T00:20:28.459 INFO:teuthology.orchestra.run.smithi121.stdout:Upgrade 4 Packages 2024-04-25T00:20:28.459 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-25T00:20:28.459 INFO:teuthology.orchestra.run.smithi121.stdout:Total download size: 220 M 2024-04-25T00:20:28.460 INFO:teuthology.orchestra.run.smithi121.stdout:Downloading Packages: 2024-04-25T00:20:28.934 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction test succeeded. 2024-04-25T00:20:28.934 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction 2024-04-25T00:20:29.037 INFO:teuthology.orchestra.run.smithi121.stdout:(1/152): ceph-19.0.0-2445.gd2c45d97.el9.x86_64. 39 kB/s | 6.6 kB 00:00 2024-04-25T00:20:29.246 INFO:teuthology.orchestra.run.smithi121.stdout:(2/152): ceph-fuse-19.0.0-2445.gd2c45d97.el9.x8 4.2 MB/s | 890 kB 00:00 2024-04-25T00:20:29.305 INFO:teuthology.orchestra.run.smithi121.stdout:(3/152): ceph-immutable-object-cache-19.0.0-244 2.4 MB/s | 145 kB 00:00 2024-04-25T00:20:29.580 INFO:teuthology.orchestra.run.smithi121.stdout:(4/152): ceph-base-19.0.0-2445.gd2c45d97.el9.x8 7.6 MB/s | 5.4 MB 00:00 2024-04-25T00:20:29.625 INFO:teuthology.orchestra.run.smithi032.stdout:(67/152): postgresql-13.14-1.el9.x86_64.rpm 1.2 MB/s | 1.6 MB 00:01 2024-04-25T00:20:29.630 INFO:teuthology.orchestra.run.smithi121.stdout:(5/152): ceph-mds-19.0.0-2445.gd2c45d97.el9.x86 7.3 MB/s | 2.4 MB 00:00 2024-04-25T00:20:29.739 INFO:teuthology.orchestra.run.smithi121.stdout:(6/152): ceph-mgr-19.0.0-2445.gd2c45d97.el9.x86 9.4 MB/s | 1.5 MB 00:00 2024-04-25T00:20:30.042 INFO:teuthology.orchestra.run.smithi032.stdout:(68/152): openblas-openmp-0.3.21-2.el9.x86_64.r 2.6 MB/s | 4.7 MB 00:01 2024-04-25T00:20:30.214 INFO:teuthology.orchestra.run.smithi121.stdout:(7/152): ceph-mon-19.0.0-2445.gd2c45d97.el9.x86 8.0 MB/s | 4.7 MB 00:00 2024-04-25T00:20:30.551 INFO:teuthology.orchestra.run.smithi032.stdout:(69/152): postgresql-private-libs-13.14-1.el9.x 262 kB/s | 133 kB 00:00 2024-04-25T00:20:30.901 INFO:teuthology.orchestra.run.smithi032.stdout:(70/152): postgresql-contrib-13.14-1.el9.x86_64 714 kB/s | 911 kB 00:01 2024-04-25T00:20:31.331 INFO:teuthology.orchestra.run.smithi121.stdout:(8/152): ceph-osd-19.0.0-2445.gd2c45d97.el9.x86 11 MB/s | 17 MB 00:01 2024-04-25T00:20:31.435 INFO:teuthology.orchestra.run.smithi032.stdout:(71/152): openssl-devel-3.2.1-1.el9.x86_64.rpm 1.4 MB/s | 4.4 MB 00:03 2024-04-25T00:20:31.447 INFO:teuthology.orchestra.run.smithi121.stdout:(9/152): ceph-radosgw-19.0.0-2445.gd2c45d97.el9 8.3 MB/s | 10 MB 00:01 2024-04-25T00:20:31.473 INFO:teuthology.orchestra.run.smithi121.stdout:(10/152): ceph-selinux-19.0.0-2445.gd2c45d97.el 177 kB/s | 25 kB 00:00 2024-04-25T00:20:31.532 INFO:teuthology.orchestra.run.smithi121.stdout:(11/152): cephfs-mirror-19.0.0-2445.gd2c45d97.e 3.7 MB/s | 222 kB 00:00 2024-04-25T00:20:31.590 INFO:teuthology.orchestra.run.smithi121.stdout:(12/152): cephfs-shell-19.0.0-2445.gd2c45d97.el 465 kB/s | 27 kB 00:00 2024-04-25T00:20:31.657 INFO:teuthology.orchestra.run.smithi121.stdout:(13/152): libcephfs-devel-19.0.0-2445.gd2c45d97 476 kB/s | 32 kB 00:00 2024-04-25T00:20:31.768 INFO:teuthology.orchestra.run.smithi032.stdout:(72/152): python3-attrs-20.3.0-7.el9.noarch.rpm 263 kB/s | 87 kB 00:00 2024-04-25T00:20:31.899 INFO:teuthology.orchestra.run.smithi121.stdout:(14/152): ceph-common-19.0.0-2445.gd2c45d97.el9 6.9 MB/s | 21 MB 00:03 2024-04-25T00:20:31.933 INFO:teuthology.orchestra.run.smithi121.stdout:(15/152): libcephfs2-19.0.0-2445.gd2c45d97.el9. 2.6 MB/s | 745 kB 00:00 2024-04-25T00:20:31.949 INFO:teuthology.orchestra.run.smithi121.stdout:(16/152): libcephsqlite-19.0.0-2445.gd2c45d97.e 3.2 MB/s | 163 kB 00:00 2024-04-25T00:20:31.960 INFO:teuthology.orchestra.run.smithi032.stdout:(73/152): protobuf-3.14.0-13.el9.x86_64.rpm 976 kB/s | 1.0 MB 00:01 2024-04-25T00:20:31.983 INFO:teuthology.orchestra.run.smithi121.stdout:(17/152): librados-devel-19.0.0-2445.gd2c45d97. 2.5 MB/s | 127 kB 00:00 2024-04-25T00:20:32.033 INFO:teuthology.orchestra.run.smithi121.stdout:(18/152): libradosstriper1-19.0.0-2445.gd2c45d9 5.9 MB/s | 501 kB 00:00 2024-04-25T00:20:32.125 INFO:teuthology.orchestra.run.smithi121.stdout:(19/152): python3-ceph-argparse-19.0.0-2445.gd2 492 kB/s | 45 kB 00:00 2024-04-25T00:20:32.217 INFO:teuthology.orchestra.run.smithi121.stdout:(20/152): python3-ceph-common-19.0.0-2445.gd2c4 1.4 MB/s | 133 kB 00:00 2024-04-25T00:20:32.318 INFO:teuthology.orchestra.run.smithi121.stdout:(21/152): python3-cephfs-19.0.0-2445.gd2c45d97. 1.6 MB/s | 162 kB 00:00 2024-04-25T00:20:32.410 INFO:teuthology.orchestra.run.smithi121.stdout:(22/152): librgw2-19.0.0-2445.gd2c45d97.el9.x86 12 MB/s | 5.3 MB 00:00 2024-04-25T00:20:32.435 INFO:teuthology.orchestra.run.smithi121.stdout:(23/152): python3-rados-19.0.0-2445.gd2c45d97.e 2.7 MB/s | 321 kB 00:00 2024-04-25T00:20:32.477 INFO:teuthology.orchestra.run.smithi121.stdout:(24/152): python3-rbd-19.0.0-2445.gd2c45d97.el9 4.4 MB/s | 299 kB 00:00 2024-04-25T00:20:32.511 INFO:teuthology.orchestra.run.smithi121.stdout:(25/152): python3-rgw-19.0.0-2445.gd2c45d97.el9 1.3 MB/s | 99 kB 00:00 2024-04-25T00:20:32.535 INFO:teuthology.orchestra.run.smithi032.stdout:(74/152): python3-devel-3.9.18-3.el9.x86_64.rpm 425 kB/s | 244 kB 00:00 2024-04-25T00:20:32.561 INFO:teuthology.orchestra.run.smithi121.stdout:(26/152): rbd-fuse-19.0.0-2445.gd2c45d97.el9.x8 1.0 MB/s | 85 kB 00:00 2024-04-25T00:20:32.628 INFO:teuthology.orchestra.run.smithi121.stdout:(27/152): rbd-nbd-19.0.0-2445.gd2c45d97.el9.x86 2.5 MB/s | 171 kB 00:00 2024-04-25T00:20:32.687 INFO:teuthology.orchestra.run.smithi121.stdout:(28/152): ceph-grafana-dashboards-19.0.0-2445.g 540 kB/s | 32 kB 00:00 2024-04-25T00:20:32.762 INFO:teuthology.orchestra.run.smithi121.stdout:(29/152): ceph-mgr-cephadm-19.0.0-2445.gd2c45d9 1.8 MB/s | 141 kB 00:00 2024-04-25T00:20:32.879 INFO:teuthology.orchestra.run.smithi121.stdout:(30/152): rbd-mirror-19.0.0-2445.gd2c45d97.el9. 8.4 MB/s | 3.1 MB 00:00 2024-04-25T00:20:33.052 INFO:teuthology.orchestra.run.smithi032.stdout:(75/152): python3-jinja2-2.11.3-5.el9.noarch.rp 480 kB/s | 248 kB 00:00 2024-04-25T00:20:33.129 INFO:teuthology.orchestra.run.smithi121.stdout:(31/152): ceph-mgr-dashboard-19.0.0-2445.gd2c45 9.6 MB/s | 3.5 MB 00:00 2024-04-25T00:20:33.213 INFO:teuthology.orchestra.run.smithi121.stdout:(32/152): ceph-mgr-modules-core-19.0.0-2445.gd2 2.9 MB/s | 249 kB 00:00 2024-04-25T00:20:33.272 INFO:teuthology.orchestra.run.smithi121.stdout:(33/152): ceph-mgr-rook-19.0.0-2445.gd2c45d97.e 850 kB/s | 49 kB 00:00 2024-04-25T00:20:33.319 INFO:teuthology.orchestra.run.smithi032.stdout:(76/152): python3-jsonpatch-1.21-16.el9.noarch. 96 kB/s | 26 kB 00:00 2024-04-25T00:20:33.338 INFO:teuthology.orchestra.run.smithi121.stdout:(34/152): ceph-prometheus-alerts-19.0.0-2445.gd 230 kB/s | 15 kB 00:00 2024-04-25T00:20:33.430 INFO:teuthology.orchestra.run.smithi121.stdout:(35/152): ceph-volume-19.0.0-2445.gd2c45d97.el9 2.8 MB/s | 264 kB 00:00 2024-04-25T00:20:33.531 INFO:teuthology.orchestra.run.smithi121.stdout:(36/152): cephadm-19.0.0-2445.gd2c45d97.el9.noa 7.5 MB/s | 762 kB 00:00 2024-04-25T00:20:33.586 INFO:teuthology.orchestra.run.smithi032.stdout:(77/152): python3-jsonpointer-2.0-4.el9.noarch. 72 kB/s | 19 kB 00:00 2024-04-25T00:20:33.606 INFO:teuthology.orchestra.run.smithi121.stdout:(37/152): cephfs-top-19.0.0-2445.gd2c45d97.el9. 243 kB/s | 18 kB 00:00 2024-04-25T00:20:33.731 INFO:teuthology.orchestra.run.smithi121.stdout:(38/152): ceph-mgr-diskprediction-local-19.0.0- 8.7 MB/s | 7.4 MB 00:00 2024-04-25T00:20:33.782 INFO:teuthology.orchestra.run.smithi121.stdout:(39/152): fuse-2.9.9-15.el9.x86_64.rpm 459 kB/s | 80 kB 00:00 2024-04-25T00:20:33.786 INFO:teuthology.orchestra.run.smithi032.stdout:(78/152): postgresql-server-13.14-1.el9.x86_64. 1.8 MB/s | 5.8 MB 00:03 2024-04-25T00:20:33.873 INFO:teuthology.orchestra.run.smithi121.stdout:(40/152): libquadmath-11.4.1-3.el9.x86_64.rpm 2.1 MB/s | 195 kB 00:00 2024-04-25T00:20:33.940 INFO:teuthology.orchestra.run.smithi121.stdout:(41/152): python3-cffi-1.14.5-5.el9.x86_64.rpm 3.7 MB/s | 253 kB 00:00 2024-04-25T00:20:34.053 INFO:teuthology.orchestra.run.smithi032.stdout:(79/152): python3-markupsafe-1.1.1-12.el9.x86_6 130 kB/s | 35 kB 00:00 2024-04-25T00:20:34.074 INFO:teuthology.orchestra.run.smithi121.stdout:(42/152): libgfortran-11.4.1-3.el9.x86_64.rpm 2.3 MB/s | 803 kB 00:00 2024-04-25T00:20:34.087 INFO:teuthology.orchestra.run.smithi032.stdout:(80/152): python3-mako-1.1.4-6.el9.noarch.rpm 344 kB/s | 172 kB 00:00 2024-04-25T00:20:34.141 INFO:teuthology.orchestra.run.smithi121.stdout:(43/152): python3-ply-3.11-14.el9.noarch.rpm 1.6 MB/s | 106 kB 00:00 2024-04-25T00:20:34.174 INFO:teuthology.orchestra.run.smithi121.stdout:(44/152): python3-cryptography-36.0.1-4.el9.x86 5.2 MB/s | 1.2 MB 00:00 2024-04-25T00:20:34.208 INFO:teuthology.orchestra.run.smithi121.stdout:(45/152): python3-pycparser-2.20-6.el9.noarch.r 2.0 MB/s | 135 kB 00:00 2024-04-25T00:20:34.242 INFO:teuthology.orchestra.run.smithi121.stdout:(46/152): python3-requests-2.25.1-8.el9.noarch. 1.8 MB/s | 125 kB 00:00 2024-04-25T00:20:34.275 INFO:teuthology.orchestra.run.smithi121.stdout:(47/152): python3-urllib3-1.26.5-5.el9.noarch.r 3.1 MB/s | 215 kB 00:00 2024-04-25T00:20:34.604 INFO:teuthology.orchestra.run.smithi032.stdout:(81/152): python3-numpy-f2py-1.20.1-5.el9.x86_6 485 kB/s | 250 kB 00:00 2024-04-25T00:20:34.743 INFO:teuthology.orchestra.run.smithi121.stdout:(48/152): ceph-test-19.0.0-2445.gd2c45d97.el9.x 15 MB/s | 48 MB 00:03 2024-04-25T00:20:34.943 INFO:teuthology.orchestra.run.smithi121.stdout:(49/152): boost-program-options-1.75.0-8.el9.x8 535 kB/s | 107 kB 00:00 2024-04-25T00:20:34.965 INFO:teuthology.orchestra.run.smithi130.stdout: Preparing : 1/1 2024-04-25T00:20:34.968 INFO:teuthology.orchestra.run.smithi121.stdout:(50/152): automake-1.16.2-8.el9.noarch.rpm 949 kB/s | 689 kB 00:00 2024-04-25T00:20:34.993 INFO:teuthology.orchestra.run.smithi121.stdout:(51/152): elfutils-libelf-devel-0.190-2.el9.x86 429 kB/s | 21 kB 00:00 2024-04-25T00:20:35.035 INFO:teuthology.orchestra.run.smithi121.stdout:(52/152): bison-3.7.4-5.el9.x86_64.rpm 1.2 MB/s | 943 kB 00:00 2024-04-25T00:20:35.051 INFO:teuthology.orchestra.run.smithi130.stdout: Upgrading : openssl-libs-1:3.2.1-1.el9.x86_64 1/156 2024-04-25T00:20:35.052 INFO:teuthology.orchestra.run.smithi121.stdout:(53/152): flexiblas-3.0.4-8.el9.x86_64.rpm 559 kB/s | 33 kB 00:00 2024-04-25T00:20:35.077 INFO:teuthology.orchestra.run.smithi121.stdout:(54/152): flex-2.6.4-9.el9.x86_64.rpm 2.8 MB/s | 313 kB 00:00 2024-04-25T00:20:35.087 INFO:teuthology.orchestra.run.smithi032.stdout:(82/152): python3-babel-2.9.1-2.el9.noarch.rpm 1.8 MB/s | 6.0 MB 00:03 2024-04-25T00:20:35.089 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 2/156 2024-04-25T00:20:35.103 INFO:teuthology.orchestra.run.smithi121.stdout:(55/152): flexiblas-openblas-openmp-3.0.4-8.el9 364 kB/s | 18 kB 00:00 2024-04-25T00:20:35.112 INFO:teuthology.orchestra.run.smithi032.stdout:(83/152): python3-oauthlib-3.1.1-5.el9.noarch.r 437 kB/s | 222 kB 00:00 2024-04-25T00:20:35.128 INFO:teuthology.orchestra.run.smithi121.stdout:(56/152): libaio-devel-0.3.111-13.el9.x86_64.rp 221 kB/s | 11 kB 00:00 2024-04-25T00:20:35.178 INFO:teuthology.orchestra.run.smithi121.stdout:(57/152): librabbitmq-0.11.0-7.el9.x86_64.rpm 909 kB/s | 45 kB 00:00 2024-04-25T00:20:35.203 INFO:teuthology.orchestra.run.smithi121.stdout:(58/152): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.6 MB/s | 160 kB 00:00 2024-04-25T00:20:35.254 INFO:teuthology.orchestra.run.smithi121.stdout:(59/152): flexiblas-netlib-3.0.4-8.el9.x86_64.r 14 MB/s | 3.0 MB 00:00 2024-04-25T00:20:35.277 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 3/156 2024-04-25T00:20:35.287 INFO:teuthology.orchestra.run.smithi121.stdout:(60/152): librdkafka-1.6.1-102.el9.x86_64.rpm 5.9 MB/s | 662 kB 00:00 2024-04-25T00:20:35.313 INFO:teuthology.orchestra.run.smithi121.stdout:(61/152): libuuid-devel-2.37.4-18.el9.x86_64.rp 434 kB/s | 25 kB 00:00 2024-04-25T00:20:35.319 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/156 2024-04-25T00:20:35.346 INFO:teuthology.orchestra.run.smithi121.stdout:(62/152): libtool-2.4.6-45.el9.x86_64.rpm 4.0 MB/s | 582 kB 00:00 2024-04-25T00:20:35.353 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 5/156 2024-04-25T00:20:35.371 INFO:teuthology.orchestra.run.smithi121.stdout:(63/152): libzstd-devel-1.5.1-2.el9.x86_64.rpm 566 kB/s | 47 kB 00:00 2024-04-25T00:20:35.396 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 6/156 2024-04-25T00:20:35.397 INFO:teuthology.orchestra.run.smithi121.stdout:(64/152): lttng-ust-2.12.0-6.el9.x86_64.rpm 3.4 MB/s | 292 kB 00:00 2024-04-25T00:20:35.422 INFO:teuthology.orchestra.run.smithi121.stdout:(65/152): lua-5.4.4-4.el9.x86_64.rpm 2.4 MB/s | 188 kB 00:00 2024-04-25T00:20:35.447 INFO:teuthology.orchestra.run.smithi121.stdout:(66/152): openblas-0.3.21-2.el9.x86_64.rpm 467 kB/s | 35 kB 00:00 2024-04-25T00:20:35.471 INFO:teuthology.orchestra.run.smithi032.stdout:(84/152): python3-prettytable-0.7.2-27.el9.noar 118 kB/s | 42 kB 00:00 2024-04-25T00:20:35.499 INFO:teuthology.orchestra.run.smithi032.stdout:(85/152): python3-packaging-20.9-5.el9.noarch.r 188 kB/s | 77 kB 00:00 2024-04-25T00:20:35.581 INFO:teuthology.orchestra.run.smithi121.stdout:(67/152): postgresql-13.14-1.el9.x86_64.rpm 12 MB/s | 1.6 MB 00:00 2024-04-25T00:20:35.681 INFO:teuthology.orchestra.run.smithi121.stdout:(68/152): openblas-openmp-0.3.21-2.el9.x86_64.r 17 MB/s | 4.7 MB 00:00 2024-04-25T00:20:35.725 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 7/156 2024-04-25T00:20:35.756 INFO:teuthology.orchestra.run.smithi121.stdout:(69/152): openssl-devel-3.2.1-1.el9.x86_64.rpm 13 MB/s | 4.4 MB 00:00 2024-04-25T00:20:35.781 INFO:teuthology.orchestra.run.smithi121.stdout:(70/152): postgresql-contrib-13.14-1.el9.x86_64 4.4 MB/s | 911 kB 00:00 2024-04-25T00:20:35.815 INFO:teuthology.orchestra.run.smithi121.stdout:(71/152): postgresql-private-libs-13.14-1.el9.x 997 kB/s | 133 kB 00:00 2024-04-25T00:20:35.829 INFO:teuthology.orchestra.run.smithi130.stdout: Upgrading : librados2-2:19.0.0-2445.gd2c45d97.el9.x86_64 8/156 2024-04-25T00:20:35.882 INFO:teuthology.orchestra.run.smithi121.stdout:(72/152): python3-attrs-20.3.0-7.el9.noarch.rpm 1.3 MB/s | 87 kB 00:00 2024-04-25T00:20:35.916 INFO:teuthology.orchestra.run.smithi121.stdout:(73/152): protobuf-3.14.0-13.el9.x86_64.rpm 7.5 MB/s | 1.0 MB 00:00 2024-04-25T00:20:35.942 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: librados2-2:19.0.0-2445.gd2c45d97.el9.x86_64 8/156 2024-04-25T00:20:35.972 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libcephfs2-2:19.0.0-2445.gd2c45d97.el9.x86_64 9/156 2024-04-25T00:20:35.982 INFO:teuthology.orchestra.run.smithi121.stdout:(74/152): python3-devel-3.9.18-3.el9.x86_64.rpm 3.6 MB/s | 244 kB 00:00 2024-04-25T00:20:35.991 INFO:teuthology.orchestra.run.smithi032.stdout:(86/152): python3-protobuf-3.14.0-13.el9.noarch 519 kB/s | 269 kB 00:00 2024-04-25T00:20:36.016 INFO:teuthology.orchestra.run.smithi032.stdout:(87/152): python3-pyasn1-0.4.8-6.el9.noarch.rpm 307 kB/s | 159 kB 00:00 2024-04-25T00:20:36.050 INFO:teuthology.orchestra.run.smithi121.stdout:(75/152): python3-jinja2-2.11.3-5.el9.noarch.rp 3.6 MB/s | 248 kB 00:00 2024-04-25T00:20:36.063 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: libcephfs2-2:19.0.0-2445.gd2c45d97.el9.x86_64 9/156 2024-04-25T00:20:36.083 INFO:teuthology.orchestra.run.smithi032.stdout:(88/152): python3-numpy-1.20.1-5.el9.x86_64.rpm 2.5 MB/s | 5.1 MB 00:02 2024-04-25T00:20:36.101 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-rados-2:19.0.0-2445.gd2c45d97.el9.x86_64 10/156 2024-04-25T00:20:36.125 INFO:teuthology.orchestra.run.smithi121.stdout:(76/152): postgresql-server-13.14-1.el9.x86_64. 16 MB/s | 5.8 MB 00:00 2024-04-25T00:20:36.150 INFO:teuthology.orchestra.run.smithi121.stdout:(77/152): python3-jsonpatch-1.21-16.el9.noarch. 256 kB/s | 26 kB 00:00 2024-04-25T00:20:36.184 INFO:teuthology.orchestra.run.smithi121.stdout:(78/152): python3-jsonpointer-2.0-4.el9.noarch. 326 kB/s | 19 kB 00:00 2024-04-25T00:20:36.209 INFO:teuthology.orchestra.run.smithi121.stdout:(79/152): python3-mako-1.1.4-6.el9.noarch.rpm 2.9 MB/s | 172 kB 00:00 2024-04-25T00:20:36.248 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : postgresql-private-libs-13.14-1.el9.x86_64 11/156 2024-04-25T00:20:36.276 INFO:teuthology.orchestra.run.smithi121.stdout:(80/152): python3-babel-2.9.1-2.el9.noarch.rpm 15 MB/s | 6.0 MB 00:00 2024-04-25T00:20:36.298 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : postgresql-13.14-1.el9.x86_64 12/156 2024-04-25T00:20:36.301 INFO:teuthology.orchestra.run.smithi121.stdout:(81/152): python3-markupsafe-1.1.1-12.el9.x86_6 296 kB/s | 35 kB 00:00 2024-04-25T00:20:36.326 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libcephsqlite-2:19.0.0-2445.gd2c45d97.el9.x86_64 13/156 2024-04-25T00:20:36.343 INFO:teuthology.orchestra.run.smithi121.stdout:(82/152): python3-numpy-f2py-1.20.1-5.el9.x86_6 3.7 MB/s | 250 kB 00:00 2024-04-25T00:20:36.358 INFO:teuthology.orchestra.run.smithi032.stdout:(89/152): python3-pytz-2021.1-5.el9.noarch.rpm 149 kB/s | 51 kB 00:00 2024-04-25T00:20:36.369 INFO:teuthology.orchestra.run.smithi121.stdout:(83/152): python3-oauthlib-3.1.1-5.el9.noarch.r 3.2 MB/s | 222 kB 00:00 2024-04-25T00:20:36.403 INFO:teuthology.orchestra.run.smithi121.stdout:(84/152): python3-packaging-20.9-5.el9.noarch.r 1.3 MB/s | 77 kB 00:00 2024-04-25T00:20:36.419 INFO:teuthology.orchestra.run.smithi121.stdout:(85/152): python3-prettytable-0.7.2-27.el9.noar 841 kB/s | 42 kB 00:00 2024-04-25T00:20:36.425 INFO:teuthology.orchestra.run.smithi032.stdout:(90/152): python3-requests-oauthlib-1.3.0-12.el 157 kB/s | 54 kB 00:00 2024-04-25T00:20:36.429 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: libcephsqlite-2:19.0.0-2445.gd2c45d97.el9.x86_64 13/156 2024-04-25T00:20:36.452 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libradosstriper1-2:19.0.0-2445.gd2c45d97.el9.x86 14/156 2024-04-25T00:20:36.461 INFO:teuthology.orchestra.run.smithi121.stdout:(86/152): python3-protobuf-3.14.0-13.el9.noarch 4.5 MB/s | 269 kB 00:00 2024-04-25T00:20:36.487 INFO:teuthology.orchestra.run.smithi121.stdout:(87/152): python3-pyasn1-0.4.8-6.el9.noarch.rpm 2.3 MB/s | 159 kB 00:00 2024-04-25T00:20:36.515 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: libradosstriper1-2:19.0.0-2445.gd2c45d97.el9.x86 14/156 2024-04-25T00:20:36.528 INFO:teuthology.orchestra.run.smithi121.stdout:(88/152): python3-pyasn1-modules-0.4.8-6.el9.no 4.1 MB/s | 279 kB 00:00 2024-04-25T00:20:36.536 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : qatlib-service-24.02.0-1.el9.x86_64 15/156 2024-04-25T00:20:36.545 INFO:teuthology.orchestra.run.smithi121.stdout:(89/152): python3-pytz-2021.1-5.el9.noarch.rpm 875 kB/s | 51 kB 00:00 2024-04-25T00:20:36.561 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: qatlib-service-24.02.0-1.el9.x86_64 15/156 2024-04-25T00:20:36.612 INFO:teuthology.orchestra.run.smithi121.stdout:(90/152): python3-numpy-1.20.1-5.el9.x86_64.rpm 13 MB/s | 5.1 MB 00:00 2024-04-25T00:20:36.637 INFO:teuthology.orchestra.run.smithi121.stdout:(91/152): python3-requests-oauthlib-1.3.0-12.el 493 kB/s | 54 kB 00:00 2024-04-25T00:20:36.671 INFO:teuthology.orchestra.run.smithi121.stdout:(92/152): python3-toml-0.10.2-6.el9.noarch.rpm 712 kB/s | 42 kB 00:00 2024-04-25T00:20:36.683 INFO:teuthology.orchestra.run.smithi032.stdout:(91/152): python3-toml-0.10.2-6.el9.noarch.rpm 161 kB/s | 42 kB 00:00 2024-04-25T00:20:36.696 INFO:teuthology.orchestra.run.smithi121.stdout:(93/152): python3-wcwidth-0.2.5-8.el9.noarch.rp 753 kB/s | 44 kB 00:00 2024-04-25T00:20:36.730 INFO:teuthology.orchestra.run.smithi121.stdout:(94/152): qatlib-24.02.0-1.el9.x86_64.rpm 3.7 MB/s | 221 kB 00:00 2024-04-25T00:20:36.753 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: qatlib-24.02.0-1.el9.x86_64 16/156 2024-04-25T00:20:36.755 INFO:teuthology.orchestra.run.smithi121.stdout:(95/152): qatlib-service-24.02.0-1.el9.x86_64.r 613 kB/s | 36 kB 00:00 2024-04-25T00:20:36.786 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : qatlib-24.02.0-1.el9.x86_64 16/156 2024-04-25T00:20:36.789 INFO:teuthology.orchestra.run.smithi121.stdout:(96/152): qatzip-libs-1.2.0-1.el9.x86_64.rpm 811 kB/s | 47 kB 00:00 2024-04-25T00:20:36.814 INFO:teuthology.orchestra.run.smithi121.stdout:(97/152): socat-1.7.4.1-5.el9.x86_64.rpm 5.1 MB/s | 305 kB 00:00 2024-04-25T00:20:36.834 INFO:teuthology.orchestra.run.smithi032.stdout:(92/152): python3-pyasn1-modules-0.4.8-6.el9.no 331 kB/s | 279 kB 00:00 2024-04-25T00:20:36.848 INFO:teuthology.orchestra.run.smithi121.stdout:(98/152): uuid-1.6.2-55.el9.x86_64.rpm 989 kB/s | 58 kB 00:00 2024-04-25T00:20:36.873 INFO:teuthology.orchestra.run.smithi121.stdout:(99/152): xfsprogs-devel-6.3.0-1.el9.x86_64.rpm 1.6 MB/s | 94 kB 00:00 2024-04-25T00:20:36.892 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : re2-1:20211101-3.el9.x86_64 17/156 2024-04-25T00:20:36.907 INFO:teuthology.orchestra.run.smithi121.stdout:(100/152): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.1 MB/s | 64 kB 00:00 2024-04-25T00:20:36.932 INFO:teuthology.orchestra.run.smithi121.stdout:(101/152): zlib-devel-1.2.11-41.el9.x86_64.rpm 764 kB/s | 45 kB 00:00 2024-04-25T00:20:36.974 INFO:teuthology.orchestra.run.smithi121.stdout:(102/152): lua-devel-5.4.4-4.el9.x86_64.rpm 333 kB/s | 22 kB 00:00 2024-04-25T00:20:36.988 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libarrow-9.0.0-10.el9.x86_64 18/156 2024-04-25T00:20:37.018 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 19/156 2024-04-25T00:20:37.026 INFO:teuthology.orchestra.run.smithi032.stdout:(93/152): python3-wcwidth-0.2.5-8.el9.noarch.rp 129 kB/s | 44 kB 00:00 2024-04-25T00:20:37.054 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : liboath-2.6.7-2.el9.x86_64 20/156 2024-04-25T00:20:37.119 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 21/156 2024-04-25T00:20:37.152 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 22/156 2024-04-25T00:20:37.196 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 23/156 2024-04-25T00:20:37.228 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : protobuf-3.14.0-13.el9.x86_64 24/156 2024-04-25T00:20:37.232 INFO:teuthology.orchestra.run.smithi121.stdout:(103/152): python3-scipy-1.6.2-8.el9.x86_64.rpm 24 MB/s | 16 MB 00:00 2024-04-25T00:20:37.257 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : lua-5.4.4-4.el9.x86_64 25/156 2024-04-25T00:20:37.274 INFO:teuthology.orchestra.run.smithi121.stdout:(104/152): protobuf-compiler-3.14.0-13.el9.x86_ 2.5 MB/s | 863 kB 00:00 2024-04-25T00:20:37.293 INFO:teuthology.orchestra.run.smithi032.stdout:(94/152): qatlib-service-24.02.0-1.el9.x86_64.r 135 kB/s | 36 kB 00:00 2024-04-25T00:20:37.325 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 26/156 2024-04-25T00:20:37.378 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 27/156 2024-04-25T00:20:37.410 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 28/156 2024-04-25T00:20:37.444 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 29/156 2024-04-25T00:20:37.500 INFO:teuthology.orchestra.run.smithi121.stdout:(105/152): grpc-data-1.46.7-10.el9.noarch.rpm 86 kB/s | 19 kB 00:00 2024-04-25T00:20:37.511 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 30/156 2024-04-25T00:20:37.547 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-ceph-common-2:19.0.0-2445.gd2c45d97.el9. 31/156 2024-04-25T00:20:37.559 INFO:teuthology.orchestra.run.smithi032.stdout:(95/152): qatlib-24.02.0-1.el9.x86_64.rpm 304 kB/s | 221 kB 00:00 2024-04-25T00:20:37.590 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-ceph-argparse-2:19.0.0-2445.gd2c45d97.el 32/156 2024-04-25T00:20:37.610 INFO:teuthology.orchestra.run.smithi121.stdout:(106/152): abseil-cpp-20211102.0-3.el9.x86_64.r 861 kB/s | 548 kB 00:00 2024-04-25T00:20:37.619 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-cephfs-2:19.0.0-2445.gd2c45d97.el9.x86_6 33/156 2024-04-25T00:20:37.635 INFO:teuthology.orchestra.run.smithi032.stdout:(96/152): qatzip-libs-1.2.0-1.el9.x86_64.rpm 139 kB/s | 47 kB 00:00 2024-04-25T00:20:37.690 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 34/156 2024-04-25T00:20:37.719 INFO:teuthology.orchestra.run.smithi121.stdout:(107/152): gperftools-libs-2.9.1-2.el9.x86_64.r 635 kB/s | 309 kB 00:00 2024-04-25T00:20:37.734 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : luarocks-3.9.2-1.el9.noarch 35/156 2024-04-25T00:20:37.795 INFO:teuthology.orchestra.run.smithi121.stdout:(108/152): libarrow-doc-9.0.0-10.el9.noarch.rpm 141 kB/s | 26 kB 00:00 2024-04-25T00:20:37.807 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : protobuf-compiler-3.14.0-13.el9.x86_64 36/156 2024-04-25T00:20:37.940 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 37/156 2024-04-25T00:20:37.953 INFO:teuthology.orchestra.run.smithi121.stdout:(109/152): liboath-2.6.7-2.el9.x86_64.rpm 208 kB/s | 49 kB 00:00 2024-04-25T00:20:37.977 INFO:teuthology.orchestra.run.smithi032.stdout:(97/152): uuid-1.6.2-55.el9.x86_64.rpm 169 kB/s | 58 kB 00:00 2024-04-25T00:20:37.995 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 38/156 2024-04-25T00:20:38.028 INFO:teuthology.orchestra.run.smithi121.stdout:(110/152): libunwind-1.6.2-1.el9.x86_64.rpm 288 kB/s | 67 kB 00:00 2024-04-25T00:20:38.282 INFO:teuthology.orchestra.run.smithi121.stdout:(111/152): luarocks-3.9.2-1.el9.noarch.rpm 464 kB/s | 151 kB 00:00 2024-04-25T00:20:38.402 INFO:teuthology.orchestra.run.smithi032.stdout:(98/152): xfsprogs-devel-6.3.0-1.el9.x86_64.rpm 221 kB/s | 94 kB 00:00 2024-04-25T00:20:38.461 INFO:teuthology.orchestra.run.smithi032.stdout:(99/152): socat-1.7.4.1-5.el9.x86_64.rpm 338 kB/s | 305 kB 00:00 2024-04-25T00:20:38.462 INFO:teuthology.orchestra.run.smithi121.stdout:(112/152): parquet-libs-9.0.0-10.el9.x86_64.rpm 1.9 MB/s | 838 kB 00:00 2024-04-25T00:20:38.522 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-rsa-4.9-2.el9.noarch 39/156 2024-04-25T00:20:38.598 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 40/156 2024-04-25T00:20:38.604 INFO:teuthology.orchestra.run.smithi121.stdout:(113/152): libarrow-9.0.0-10.el9.x86_64.rpm 4.0 MB/s | 4.4 MB 00:01 2024-04-25T00:20:38.628 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 41/156 2024-04-25T00:20:38.670 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : qatzip-libs-1.2.0-1.el9.x86_64 42/156 2024-04-25T00:20:38.709 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : librados-devel-2:19.0.0-2445.gd2c45d97.el9.x86_6 43/156 2024-04-25T00:20:38.744 INFO:teuthology.orchestra.run.smithi032.stdout:(100/152): xmlstarlet-1.6.1-20.el9.x86_64.rpm 186 kB/s | 64 kB 00:00 2024-04-25T00:20:38.751 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 44/156 2024-04-25T00:20:38.754 INFO:teuthology.orchestra.run.smithi121.stdout:(114/152): python3-bcrypt-3.2.2-1.el9.x86_64.rp 149 kB/s | 43 kB 00:00 2024-04-25T00:20:38.820 INFO:teuthology.orchestra.run.smithi032.stdout:(101/152): zlib-devel-1.2.11-41.el9.x86_64.rpm 125 kB/s | 45 kB 00:00 2024-04-25T00:20:38.838 INFO:teuthology.orchestra.run.smithi121.stdout:(115/152): python3-cachetools-4.2.4-1.el9.noarc 138 kB/s | 32 kB 00:00 2024-04-25T00:20:38.870 INFO:teuthology.orchestra.run.smithi032.stdout:(102/152): lua-devel-5.4.4-4.el9.x86_64.rpm 178 kB/s | 22 kB 00:00 2024-04-25T00:20:38.909 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 45/156 2024-04-25T00:20:38.970 INFO:teuthology.orchestra.run.smithi130.stdout: Upgrading : openssl-1:3.2.1-1.el9.x86_64 46/156 2024-04-25T00:20:39.005 INFO:teuthology.orchestra.run.smithi121.stdout:(116/152): python3-certifi-2021.10.8-2.el9.noar 60 kB/s | 15 kB 00:00 2024-04-25T00:20:39.024 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 47/156 2024-04-25T00:20:39.038 INFO:teuthology.orchestra.run.smithi121.stdout:(117/152): python3-asyncssh-2.13.2-5.el9.noarch 721 kB/s | 548 kB 00:00 2024-04-25T00:20:39.099 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 48/156 2024-04-25T00:20:39.137 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 49/156 2024-04-25T00:20:39.146 INFO:teuthology.orchestra.run.smithi032.stdout:(103/152): protobuf-compiler-3.14.0-13.el9.x86_ 2.6 MB/s | 863 kB 00:00 2024-04-25T00:20:39.174 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 50/156 2024-04-25T00:20:39.221 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 51/156 2024-04-25T00:20:39.257 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 52/156 2024-04-25T00:20:39.270 INFO:teuthology.orchestra.run.smithi032.stdout:(104/152): abseil-cpp-20211102.0-3.el9.x86_64.r 1.3 MB/s | 548 kB 00:00 2024-04-25T00:20:39.303 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-pyperclip-1.8.2-2.el9.noarch 53/156 2024-04-25T00:20:39.305 INFO:teuthology.orchestra.run.smithi121.stdout:(118/152): python3-cheroot-8.6.0-4.el9.noarch.r 368 kB/s | 172 kB 00:00 2024-04-25T00:20:39.342 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 54/156 2024-04-25T00:20:39.354 INFO:teuthology.orchestra.run.smithi032.stdout:(105/152): grpc-data-1.46.7-10.el9.noarch.rpm 233 kB/s | 19 kB 00:00 2024-04-25T00:20:39.439 INFO:teuthology.orchestra.run.smithi121.stdout:(119/152): python3-cherrypy-18.6.1-2.el9.noarch 826 kB/s | 358 kB 00:00 2024-04-25T00:20:39.477 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 55/156 2024-04-25T00:20:39.513 INFO:teuthology.orchestra.run.smithi032.stdout:(106/152): gperftools-libs-2.9.1-2.el9.x86_64.r 841 kB/s | 309 kB 00:00 2024-04-25T00:20:39.531 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 56/156 2024-04-25T00:20:39.561 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-colorama-0.4.4-12.el9.noarch 57/156 2024-04-25T00:20:39.581 INFO:teuthology.orchestra.run.smithi121.stdout:(120/152): python3-cmd2-2.4.2-2.el9.noarch.rpm 559 kB/s | 303 kB 00:00 2024-04-25T00:20:39.603 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 58/156 2024-04-25T00:20:39.605 INFO:teuthology.orchestra.run.smithi032.stdout:(107/152): libarrow-doc-9.0.0-10.el9.noarch.rpm 284 kB/s | 26 kB 00:00 2024-04-25T00:20:39.607 INFO:teuthology.orchestra.run.smithi121.stdout:(121/152): python3-colorama-0.4.4-12.el9.noarch 117 kB/s | 35 kB 00:00 2024-04-25T00:20:39.681 INFO:teuthology.orchestra.run.smithi032.stdout:(108/152): libarrow-9.0.0-10.el9.x86_64.rpm 14 MB/s | 4.4 MB 00:00 2024-04-25T00:20:39.695 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 59/156 2024-04-25T00:20:39.705 INFO:teuthology.orchestra.run.smithi032.stdout:(109/152): liboath-2.6.7-2.el9.x86_64.rpm 484 kB/s | 49 kB 00:00 2024-04-25T00:20:39.745 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-google-auth-1:2.29.0-1.el9.noarch 60/156 2024-04-25T00:20:39.772 INFO:teuthology.orchestra.run.smithi032.stdout:(110/152): libunwind-1.6.2-1.el9.x86_64.rpm 734 kB/s | 67 kB 00:00 2024-04-25T00:20:39.785 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 61/156 2024-04-25T00:20:39.798 INFO:teuthology.orchestra.run.smithi032.stdout:(111/152): luarocks-3.9.2-1.el9.noarch.rpm 1.6 MB/s | 151 kB 00:00 2024-04-25T00:20:39.810 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 62/156 2024-04-25T00:20:39.835 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libarrow-doc-9.0.0-10.el9.noarch 63/156 2024-04-25T00:20:39.882 INFO:teuthology.orchestra.run.smithi032.stdout:(112/152): parquet-libs-9.0.0-10.el9.x86_64.rpm 7.5 MB/s | 838 kB 00:00 2024-04-25T00:20:39.908 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 64/156 2024-04-25T00:20:39.957 INFO:teuthology.orchestra.run.smithi032.stdout:(113/152): python3-asyncssh-2.13.2-5.el9.noarch 3.4 MB/s | 548 kB 00:00 2024-04-25T00:20:39.964 INFO:teuthology.orchestra.run.smithi121.stdout:(122/152): python3-google-auth-2.29.0-1.el9.noa 391 kB/s | 206 kB 00:00 2024-04-25T00:20:39.982 INFO:teuthology.orchestra.run.smithi032.stdout:(114/152): python3-bcrypt-3.2.2-1.el9.x86_64.rp 433 kB/s | 43 kB 00:00 2024-04-25T00:20:40.031 INFO:teuthology.orchestra.run.smithi121.stdout:(123/152): python3-grpcio-tools-1.46.7-10.el9.x 340 kB/s | 144 kB 00:00 2024-04-25T00:20:40.041 INFO:teuthology.orchestra.run.smithi032.stdout:(115/152): python3-cachetools-4.2.4-1.el9.noarc 385 kB/s | 32 kB 00:00 2024-04-25T00:20:40.054 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : abseil-cpp-20211102.0-3.el9.x86_64 65/156 2024-04-25T00:20:40.074 INFO:teuthology.orchestra.run.smithi032.stdout:(116/152): python3-certifi-2021.10.8-2.el9.noar 163 kB/s | 15 kB 00:00 2024-04-25T00:20:40.084 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 66/156 2024-04-25T00:20:40.120 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : zlib-devel-1.2.11-41.el9.x86_64 67/156 2024-04-25T00:20:40.125 INFO:teuthology.orchestra.run.smithi032.stdout:(117/152): python3-cheroot-8.6.0-4.el9.noarch.r 2.0 MB/s | 172 kB 00:00 2024-04-25T00:20:40.148 INFO:teuthology.orchestra.run.smithi121.stdout:(124/152): python3-jaraco-8.2.1-3.el9.noarch.rp 58 kB/s | 11 kB 00:00 2024-04-25T00:20:40.155 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 68/156 2024-04-25T00:20:40.167 INFO:teuthology.orchestra.run.smithi032.stdout:(118/152): python3-cherrypy-18.6.1-2.el9.noarch 3.8 MB/s | 358 kB 00:00 2024-04-25T00:20:40.185 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : uuid-1.6.2-55.el9.x86_64 69/156 2024-04-25T00:20:40.217 INFO:teuthology.orchestra.run.smithi032.stdout:(119/152): python3-cmd2-2.4.2-2.el9.noarch.rpm 3.2 MB/s | 303 kB 00:00 2024-04-25T00:20:40.218 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-wcwidth-0.2.5-8.el9.noarch 70/156 2024-04-25T00:20:40.242 INFO:teuthology.orchestra.run.smithi032.stdout:(120/152): python3-colorama-0.4.4-12.el9.noarch 468 kB/s | 35 kB 00:00 2024-04-25T00:20:40.247 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 71/156 2024-04-25T00:20:40.282 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 72/156 2024-04-25T00:20:40.283 INFO:teuthology.orchestra.run.smithi121.stdout:(125/152): python3-jaraco-classes-3.2.1-5.el9.n 71 kB/s | 18 kB 00:00 2024-04-25T00:20:40.309 INFO:teuthology.orchestra.run.smithi032.stdout:(121/152): python3-google-auth-2.29.0-1.el9.noa 2.2 MB/s | 206 kB 00:00 2024-04-25T00:20:40.315 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 73/156 2024-04-25T00:20:40.356 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 74/156 2024-04-25T00:20:40.391 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 75/156 2024-04-25T00:20:40.401 INFO:teuthology.orchestra.run.smithi032.stdout:(122/152): python3-grpcio-1.46.7-10.el9.x86_64. 13 MB/s | 2.0 MB 00:00 2024-04-25T00:20:40.427 INFO:teuthology.orchestra.run.smithi032.stdout:(123/152): python3-grpcio-tools-1.46.7-10.el9.x 1.2 MB/s | 144 kB 00:00 2024-04-25T00:20:40.457 INFO:teuthology.orchestra.run.smithi121.stdout:(126/152): python3-jaraco-collections-3.0.0-8.e 75 kB/s | 23 kB 00:00 2024-04-25T00:20:40.476 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 76/156 2024-04-25T00:20:40.477 INFO:teuthology.orchestra.run.smithi032.stdout:(124/152): python3-jaraco-8.2.1-3.el9.noarch.rp 142 kB/s | 11 kB 00:00 2024-04-25T00:20:40.483 INFO:teuthology.orchestra.run.smithi121.stdout:(127/152): python3-jaraco-functools-3.5.0-2.el9 97 kB/s | 19 kB 00:00 2024-04-25T00:20:40.494 INFO:teuthology.orchestra.run.smithi032.stdout:(125/152): python3-jaraco-classes-3.2.1-5.el9.n 263 kB/s | 18 kB 00:00 2024-04-25T00:20:40.526 INFO:teuthology.orchestra.run.smithi121.stdout:(128/152): python3-grpcio-1.46.7-10.el9.x86_64. 2.2 MB/s | 2.0 MB 00:00 2024-04-25T00:20:40.553 INFO:teuthology.orchestra.run.smithi032.stdout:(126/152): python3-jaraco-collections-3.0.0-8.e 307 kB/s | 23 kB 00:00 2024-04-25T00:20:40.578 INFO:teuthology.orchestra.run.smithi032.stdout:(127/152): python3-jaraco-functools-3.5.0-2.el9 232 kB/s | 19 kB 00:00 2024-04-25T00:20:40.645 INFO:teuthology.orchestra.run.smithi032.stdout:(128/152): python3-jaraco-text-3.2.0-6.el9.noar 213 kB/s | 20 kB 00:00 2024-04-25T00:20:40.675 INFO:teuthology.orchestra.run.smithi121.stdout:(129/152): python3-jaraco-text-3.2.0-6.el9.noar 90 kB/s | 20 kB 00:00 2024-04-25T00:20:40.707 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-protobuf-3.14.0-13.el9.noarch 77/156 2024-04-25T00:20:40.712 INFO:teuthology.orchestra.run.smithi032.stdout:(129/152): python3-logutils-0.3.5-21.el9.noarch 693 kB/s | 46 kB 00:00 2024-04-25T00:20:40.737 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 78/156 2024-04-25T00:20:40.745 INFO:teuthology.orchestra.run.smithi032.stdout:(130/152): python3-kubernetes-26.1.0-2.el9.noar 6.1 MB/s | 1.0 MB 00:00 2024-04-25T00:20:40.758 INFO:teuthology.orchestra.run.smithi121.stdout:(130/152): python3-logutils-0.3.5-21.el9.noarch 198 kB/s | 46 kB 00:00 2024-04-25T00:20:40.785 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 79/156 2024-04-25T00:20:40.787 INFO:teuthology.orchestra.run.smithi032.stdout:(131/152): python3-more-itertools-8.12.0-2.el9. 1.0 MB/s | 79 kB 00:00 2024-04-25T00:20:40.821 INFO:teuthology.orchestra.run.smithi032.stdout:(132/152): python3-natsort-7.1.1-5.el9.noarch.r 765 kB/s | 58 kB 00:00 2024-04-25T00:20:40.893 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-packaging-20.9-5.el9.noarch 80/156 2024-04-25T00:20:40.925 INFO:teuthology.orchestra.run.smithi121.stdout:(131/152): python3-more-itertools-8.12.0-2.el9. 315 kB/s | 79 kB 00:00 2024-04-25T00:20:40.947 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 81/156 2024-04-25T00:20:40.971 INFO:teuthology.orchestra.run.smithi032.stdout:(133/152): python3-pecan-1.4.2-3.el9.noarch.rpm 1.4 MB/s | 272 kB 00:00 2024-04-25T00:20:41.009 INFO:teuthology.orchestra.run.smithi121.stdout:(132/152): python3-kubernetes-26.1.0-2.el9.noar 1.9 MB/s | 1.0 MB 00:00 2024-04-25T00:20:41.034 INFO:teuthology.orchestra.run.smithi121.stdout:(133/152): python3-natsort-7.1.1-5.el9.noarch.r 209 kB/s | 58 kB 00:00 2024-04-25T00:20:41.080 INFO:teuthology.orchestra.run.smithi032.stdout:(134/152): python3-portend-3.1.0-2.el9.noarch.r 63 kB/s | 16 kB 00:00 2024-04-25T00:20:41.106 INFO:teuthology.orchestra.run.smithi032.stdout:(135/152): python3-pyOpenSSL-21.0.0-1.el9.noarc 672 kB/s | 90 kB 00:00 2024-04-25T00:20:41.181 INFO:teuthology.orchestra.run.smithi032.stdout:(136/152): python3-pyperclip-1.8.2-2.el9.noarch 281 kB/s | 28 kB 00:00 2024-04-25T00:20:41.184 INFO:teuthology.orchestra.run.smithi121.stdout:(134/152): python3-portend-3.1.0-2.el9.noarch.r 94 kB/s | 16 kB 00:00 2024-04-25T00:20:41.206 INFO:teuthology.orchestra.run.smithi032.stdout:(137/152): python3-repoze-lru-0.7-16.el9.noarch 307 kB/s | 31 kB 00:00 2024-04-25T00:20:41.282 INFO:teuthology.orchestra.run.smithi032.stdout:(138/152): python3-routes-2.5.1-5.el9.noarch.rp 1.8 MB/s | 188 kB 00:00 2024-04-25T00:20:41.307 INFO:teuthology.orchestra.run.smithi032.stdout:(139/152): python3-rsa-4.9-2.el9.noarch.rpm 587 kB/s | 59 kB 00:00 2024-04-25T00:20:41.318 INFO:teuthology.orchestra.run.smithi121.stdout:(135/152): python3-pyOpenSSL-21.0.0-1.el9.noarc 317 kB/s | 90 kB 00:00 2024-04-25T00:20:41.357 INFO:teuthology.orchestra.run.smithi032.stdout:(140/152): python3-tempora-5.0.0-2.el9.noarch.r 475 kB/s | 36 kB 00:00 2024-04-25T00:20:41.360 INFO:teuthology.orchestra.run.smithi121.stdout:(136/152): python3-pyperclip-1.8.2-2.el9.noarch 161 kB/s | 28 kB 00:00 2024-04-25T00:20:41.391 INFO:teuthology.orchestra.run.smithi032.stdout:(141/152): python3-typing-extensions-4.4.0-2.el 608 kB/s | 51 kB 00:00 2024-04-25T00:20:41.410 INFO:teuthology.orchestra.run.smithi121.stdout:(137/152): python3-pecan-1.4.2-3.el9.noarch.rpm 562 kB/s | 272 kB 00:00 2024-04-25T00:20:41.466 INFO:teuthology.orchestra.run.smithi032.stdout:(142/152): python3-webob-1.8.7-6.el9.noarch.rpm 2.1 MB/s | 230 kB 00:00 2024-04-25T00:20:41.492 INFO:teuthology.orchestra.run.smithi032.stdout:(143/152): python3-websocket-client-1.2.3-2.el9 895 kB/s | 90 kB 00:00 2024-04-25T00:20:41.542 INFO:teuthology.orchestra.run.smithi032.stdout:(144/152): python3-werkzeug-2.0.3-3.el9.1.noarc 5.6 MB/s | 427 kB 00:00 2024-04-25T00:20:41.575 INFO:teuthology.orchestra.run.smithi032.stdout:(145/152): python3-zc-lockfile-2.0-10.el9.noarc 239 kB/s | 20 kB 00:00 2024-04-25T00:20:41.594 INFO:teuthology.orchestra.run.smithi121.stdout:(138/152): python3-repoze-lru-0.7-16.el9.noarch 111 kB/s | 31 kB 00:00 2024-04-25T00:20:41.656 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 82/156 2024-04-25T00:20:41.659 INFO:teuthology.orchestra.run.smithi032.stdout:(146/152): re2-20211101-3.el9.x86_64.rpm 1.6 MB/s | 192 kB 00:00 2024-04-25T00:20:41.661 INFO:teuthology.orchestra.run.smithi121.stdout:(139/152): python3-rsa-4.9-2.el9.noarch.rpm 236 kB/s | 59 kB 00:00 2024-04-25T00:20:41.702 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 83/156 2024-04-25T00:20:41.719 INFO:teuthology.orchestra.run.smithi121.stdout:(140/152): python3-routes-2.5.1-5.el9.noarch.rp 525 kB/s | 188 kB 00:00 2024-04-25T00:20:41.735 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 84/156 2024-04-25T00:20:41.743 INFO:teuthology.orchestra.run.smithi032.stdout:(147/152): thrift-0.15.0-2.el9.x86_64.rpm 9.5 MB/s | 1.6 MB 00:00 2024-04-25T00:20:41.833 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 85/156 2024-04-25T00:20:41.836 INFO:teuthology.orchestra.run.smithi121.stdout:(141/152): python3-tempora-5.0.0-2.el9.noarch.r 148 kB/s | 36 kB 00:00 2024-04-25T00:20:41.885 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 86/156 2024-04-25T00:20:41.973 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 87/156 2024-04-25T00:20:41.987 INFO:teuthology.orchestra.run.smithi121.stdout:(142/152): python3-typing-extensions-4.4.0-2.el 157 kB/s | 51 kB 00:00 2024-04-25T00:20:42.088 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-cmd2-2.4.2-2.el9.noarch 88/156 2024-04-25T00:20:42.170 INFO:teuthology.orchestra.run.smithi121.stdout:(143/152): python3-websocket-client-1.2.3-2.el9 269 kB/s | 90 kB 00:00 2024-04-25T00:20:42.204 INFO:teuthology.orchestra.run.smithi121.stdout:(144/152): python3-webob-1.8.7-6.el9.noarch.rpm 474 kB/s | 230 kB 00:00 2024-04-25T00:20:42.264 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : openblas-0.3.21-2.el9.x86_64 89/156 2024-04-25T00:20:42.294 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 90/156 2024-04-25T00:20:42.371 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 91/156 2024-04-25T00:20:42.412 INFO:teuthology.orchestra.run.smithi121.stdout:(145/152): python3-zc-lockfile-2.0-10.el9.noarc 83 kB/s | 20 kB 00:00 2024-04-25T00:20:42.510 INFO:teuthology.orchestra.run.smithi032.stdout:(148/152): librados2-19.0.0-2445.gd2c45d97.el9. 4.0 MB/s | 3.4 MB 00:00 2024-04-25T00:20:42.638 INFO:teuthology.orchestra.run.smithi121.stdout:(146/152): python3-werkzeug-2.0.3-3.el9.1.noarc 656 kB/s | 427 kB 00:00 2024-04-25T00:20:42.671 INFO:teuthology.orchestra.run.smithi121.stdout:(147/152): re2-20211101-3.el9.x86_64.rpm 410 kB/s | 192 kB 00:00 2024-04-25T00:20:42.677 INFO:teuthology.orchestra.run.smithi032.stdout:(149/152): python3-scipy-1.6.2-8.el9.x86_64.rpm 2.6 MB/s | 16 MB 00:06 2024-04-25T00:20:42.735 INFO:teuthology.orchestra.run.smithi032.stdout:(150/152): librbd1-19.0.0-2445.gd2c45d97.el9.x8 3.2 MB/s | 3.1 MB 00:00 2024-04-25T00:20:42.919 INFO:teuthology.orchestra.run.smithi032.stdout:(151/152): openssl-3.2.1-1.el9.x86_64.rpm 3.3 MB/s | 1.3 MB 00:00 2024-04-25T00:20:42.986 INFO:teuthology.orchestra.run.smithi032.stdout:(152/152): openssl-libs-3.2.1-1.el9.x86_64.rpm 7.8 MB/s | 2.4 MB 00:00 2024-04-25T00:20:42.988 INFO:teuthology.orchestra.run.smithi032.stdout:-------------------------------------------------------------------------------- 2024-04-25T00:20:42.989 INFO:teuthology.orchestra.run.smithi032.stdout:Total 7.4 MB/s | 220 MB 00:29 2024-04-25T00:20:43.010 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 92/156 2024-04-25T00:20:43.125 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 93/156 2024-04-25T00:20:43.338 INFO:teuthology.orchestra.run.smithi121.stdout:(148/152): librbd1-19.0.0-2445.gd2c45d97.el9.x8 4.7 MB/s | 3.1 MB 00:00 2024-04-25T00:20:43.388 INFO:teuthology.orchestra.run.smithi121.stdout:(149/152): librados2-19.0.0-2445.gd2c45d97.el9. 4.5 MB/s | 3.4 MB 00:00 2024-04-25T00:20:43.630 INFO:teuthology.orchestra.run.smithi121.stdout:(150/152): thrift-0.15.0-2.el9.x86_64.rpm 1.3 MB/s | 1.6 MB 00:01 2024-04-25T00:20:43.672 INFO:teuthology.orchestra.run.smithi121.stdout:(151/152): openssl-3.2.1-1.el9.x86_64.rpm 4.0 MB/s | 1.3 MB 00:00 2024-04-25T00:20:43.723 INFO:teuthology.orchestra.run.smithi121.stdout:(152/152): openssl-libs-3.2.1-1.el9.x86_64.rpm 7.2 MB/s | 2.4 MB 00:00 2024-04-25T00:20:43.725 INFO:teuthology.orchestra.run.smithi121.stdout:-------------------------------------------------------------------------------- 2024-04-25T00:20:43.726 INFO:teuthology.orchestra.run.smithi121.stdout:Total 14 MB/s | 220 MB 00:15 2024-04-25T00:20:44.137 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction check 2024-04-25T00:20:44.296 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction check succeeded. 2024-04-25T00:20:44.297 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction test 2024-04-25T00:20:44.324 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 94/156 2024-04-25T00:20:44.399 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 95/156 2024-04-25T00:20:44.426 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libzstd-devel-1.5.1-2.el9.x86_64 96/156 2024-04-25T00:20:44.459 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libuuid-devel-2.37.4-18.el9.x86_64 97/156 2024-04-25T00:20:44.770 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 98/156 2024-04-25T00:20:44.790 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction check 2024-04-25T00:20:44.814 INFO:teuthology.orchestra.run.smithi130.stdout: Upgrading : librbd1-2:19.0.0-2445.gd2c45d97.el9.x86_64 99/156 2024-04-25T00:20:44.905 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: librbd1-2:19.0.0-2445.gd2c45d97.el9.x86_64 99/156 2024-04-25T00:20:44.932 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction check succeeded. 2024-04-25T00:20:44.932 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction test 2024-04-25T00:20:44.944 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-rbd-2:19.0.0-2445.gd2c45d97.el9.x86_64 100/156 2024-04-25T00:20:44.980 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 101/156 2024-04-25T00:20:45.449 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : parquet-libs-9.0.0-10.el9.x86_64 102/156 2024-04-25T00:20:45.476 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : librgw2-2:19.0.0-2445.gd2c45d97.el9.x86_64 103/156 2024-04-25T00:20:45.546 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: librgw2-2:19.0.0-2445.gd2c45d97.el9.x86_64 103/156 2024-04-25T00:20:45.569 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-rgw-2:19.0.0-2445.gd2c45d97.el9.x86_64 104/156 2024-04-25T00:20:46.165 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction test succeeded. 2024-04-25T00:20:46.165 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction 2024-04-25T00:20:46.774 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction test succeeded. 2024-04-25T00:20:46.775 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction 2024-04-25T00:20:47.888 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-common-2:19.0.0-2445.gd2c45d97.el9.x86_64 105/156 2024-04-25T00:20:47.916 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-common-2:19.0.0-2445.gd2c45d97.el9.x86_64 105/156 2024-04-25T00:20:48.454 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-common-2:19.0.0-2445.gd2c45d97.el9.x86_64 105/156 2024-04-25T00:20:48.494 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-base-2:19.0.0-2445.gd2c45d97.el9.x86_64 106/156 2024-04-25T00:20:48.580 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-base-2:19.0.0-2445.gd2c45d97.el9.x86_64 106/156 2024-04-25T00:20:48.580 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-25T00:20:48.580 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-25T00:20:48.580 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-25T00:20:48.612 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-selinux-2:19.0.0-2445.gd2c45d97.el9.x86_64 107/156 2024-04-25T00:20:52.308 INFO:teuthology.orchestra.run.smithi032.stdout: Preparing : 1/1 2024-04-25T00:20:52.398 INFO:teuthology.orchestra.run.smithi032.stdout: Upgrading : openssl-libs-1:3.2.1-1.el9.x86_64 1/156 2024-04-25T00:20:52.438 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 2/156 2024-04-25T00:20:52.491 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 3/156 2024-04-25T00:20:52.533 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/156 2024-04-25T00:20:52.566 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 5/156 2024-04-25T00:20:52.610 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 6/156 2024-04-25T00:20:52.880 INFO:teuthology.orchestra.run.smithi121.stdout: Preparing : 1/1 2024-04-25T00:20:52.957 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 7/156 2024-04-25T00:20:52.967 INFO:teuthology.orchestra.run.smithi121.stdout: Upgrading : openssl-libs-1:3.2.1-1.el9.x86_64 1/156 2024-04-25T00:20:53.005 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 2/156 2024-04-25T00:20:53.067 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 3/156 2024-04-25T00:20:53.076 INFO:teuthology.orchestra.run.smithi032.stdout: Upgrading : librados2-2:19.0.0-2445.gd2c45d97.el9.x86_64 8/156 2024-04-25T00:20:53.101 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/156 2024-04-25T00:20:53.127 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 5/156 2024-04-25T00:20:53.169 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 6/156 2024-04-25T00:20:53.200 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: librados2-2:19.0.0-2445.gd2c45d97.el9.x86_64 8/156 2024-04-25T00:20:53.227 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libcephfs2-2:19.0.0-2445.gd2c45d97.el9.x86_64 9/156 2024-04-25T00:20:53.319 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: libcephfs2-2:19.0.0-2445.gd2c45d97.el9.x86_64 9/156 2024-04-25T00:20:53.356 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-rados-2:19.0.0-2445.gd2c45d97.el9.x86_64 10/156 2024-04-25T00:20:53.520 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : postgresql-private-libs-13.14-1.el9.x86_64 11/156 2024-04-25T00:20:53.548 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 7/156 2024-04-25T00:20:53.570 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : postgresql-13.14-1.el9.x86_64 12/156 2024-04-25T00:20:53.598 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libcephsqlite-2:19.0.0-2445.gd2c45d97.el9.x86_64 13/156 2024-04-25T00:20:53.603 INFO:teuthology.orchestra.run.smithi121.stdout: Upgrading : librados2-2:19.0.0-2445.gd2c45d97.el9.x86_64 8/156 2024-04-25T00:20:53.703 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: libcephsqlite-2:19.0.0-2445.gd2c45d97.el9.x86_64 13/156 2024-04-25T00:20:53.723 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: librados2-2:19.0.0-2445.gd2c45d97.el9.x86_64 8/156 2024-04-25T00:20:53.732 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libradosstriper1-2:19.0.0-2445.gd2c45d97.el9.x86 14/156 2024-04-25T00:20:53.745 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : libcephfs2-2:19.0.0-2445.gd2c45d97.el9.x86_64 9/156 2024-04-25T00:20:53.795 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: libradosstriper1-2:19.0.0-2445.gd2c45d97.el9.x86 14/156 2024-04-25T00:20:53.816 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : qatlib-service-24.02.0-1.el9.x86_64 15/156 2024-04-25T00:20:53.820 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: libcephfs2-2:19.0.0-2445.gd2c45d97.el9.x86_64 9/156 2024-04-25T00:20:53.841 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: qatlib-service-24.02.0-1.el9.x86_64 15/156 2024-04-25T00:20:53.858 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-rados-2:19.0.0-2445.gd2c45d97.el9.x86_64 10/156 2024-04-25T00:20:54.003 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : postgresql-private-libs-13.14-1.el9.x86_64 11/156 2024-04-25T00:20:54.035 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: qatlib-24.02.0-1.el9.x86_64 16/156 2024-04-25T00:20:54.063 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : postgresql-13.14-1.el9.x86_64 12/156 2024-04-25T00:20:54.066 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : qatlib-24.02.0-1.el9.x86_64 16/156 2024-04-25T00:20:54.092 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : libcephsqlite-2:19.0.0-2445.gd2c45d97.el9.x86_64 13/156 2024-04-25T00:20:54.195 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: libcephsqlite-2:19.0.0-2445.gd2c45d97.el9.x86_64 13/156 2024-04-25T00:20:54.217 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : libradosstriper1-2:19.0.0-2445.gd2c45d97.el9.x86 14/156 2024-04-25T00:20:54.219 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : re2-1:20211101-3.el9.x86_64 17/156 2024-04-25T00:20:54.263 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: libradosstriper1-2:19.0.0-2445.gd2c45d97.el9.x86 14/156 2024-04-25T00:20:54.285 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : qatlib-service-24.02.0-1.el9.x86_64 15/156 2024-04-25T00:20:54.305 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: qatlib-service-24.02.0-1.el9.x86_64 15/156 2024-04-25T00:20:54.316 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libarrow-9.0.0-10.el9.x86_64 18/156 2024-04-25T00:20:54.349 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 19/156 2024-04-25T00:20:54.384 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : liboath-2.6.7-2.el9.x86_64 20/156 2024-04-25T00:20:54.461 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 21/156 2024-04-25T00:20:54.475 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: qatlib-24.02.0-1.el9.x86_64 16/156 2024-04-25T00:20:54.500 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 22/156 2024-04-25T00:20:54.509 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : qatlib-24.02.0-1.el9.x86_64 16/156 2024-04-25T00:20:54.552 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 23/156 2024-04-25T00:20:54.584 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : protobuf-3.14.0-13.el9.x86_64 24/156 2024-04-25T00:20:54.612 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : lua-5.4.4-4.el9.x86_64 25/156 2024-04-25T00:20:54.616 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : re2-1:20211101-3.el9.x86_64 17/156 2024-04-25T00:20:54.685 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 26/156 2024-04-25T00:20:54.711 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : libarrow-9.0.0-10.el9.x86_64 18/156 2024-04-25T00:20:54.742 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 19/156 2024-04-25T00:20:54.761 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 27/156 2024-04-25T00:20:54.776 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : liboath-2.6.7-2.el9.x86_64 20/156 2024-04-25T00:20:54.798 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 28/156 2024-04-25T00:20:54.840 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 21/156 2024-04-25T00:20:54.842 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 29/156 2024-04-25T00:20:54.875 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 22/156 2024-04-25T00:20:54.919 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 23/156 2024-04-25T00:20:54.920 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 30/156 2024-04-25T00:20:54.952 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : protobuf-3.14.0-13.el9.x86_64 24/156 2024-04-25T00:20:54.952 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-ceph-common-2:19.0.0-2445.gd2c45d97.el9. 31/156 2024-04-25T00:20:54.980 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : lua-5.4.4-4.el9.x86_64 25/156 2024-04-25T00:20:54.996 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-ceph-argparse-2:19.0.0-2445.gd2c45d97.el 32/156 2024-04-25T00:20:55.024 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-cephfs-2:19.0.0-2445.gd2c45d97.el9.x86_6 33/156 2024-04-25T00:20:55.048 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 26/156 2024-04-25T00:20:55.099 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 34/156 2024-04-25T00:20:55.101 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 27/156 2024-04-25T00:20:55.133 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 28/156 2024-04-25T00:20:55.148 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : luarocks-3.9.2-1.el9.noarch 35/156 2024-04-25T00:20:55.168 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 29/156 2024-04-25T00:20:55.235 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : protobuf-compiler-3.14.0-13.el9.x86_64 36/156 2024-04-25T00:20:55.243 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 30/156 2024-04-25T00:20:55.287 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-ceph-common-2:19.0.0-2445.gd2c45d97.el9. 31/156 2024-04-25T00:20:55.330 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-ceph-argparse-2:19.0.0-2445.gd2c45d97.el 32/156 2024-04-25T00:20:55.359 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-cephfs-2:19.0.0-2445.gd2c45d97.el9.x86_6 33/156 2024-04-25T00:20:55.383 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 37/156 2024-04-25T00:20:55.427 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 34/156 2024-04-25T00:20:55.444 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 38/156 2024-04-25T00:20:55.474 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : luarocks-3.9.2-1.el9.noarch 35/156 2024-04-25T00:20:55.546 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : protobuf-compiler-3.14.0-13.el9.x86_64 36/156 2024-04-25T00:20:55.680 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 37/156 2024-04-25T00:20:55.733 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 38/156 2024-04-25T00:20:56.016 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-rsa-4.9-2.el9.noarch 39/156 2024-04-25T00:20:56.108 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 40/156 2024-04-25T00:20:56.150 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 41/156 2024-04-25T00:20:56.193 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : qatzip-libs-1.2.0-1.el9.x86_64 42/156 2024-04-25T00:20:56.231 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : librados-devel-2:19.0.0-2445.gd2c45d97.el9.x86_6 43/156 2024-04-25T00:20:56.293 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-rsa-4.9-2.el9.noarch 39/156 2024-04-25T00:20:56.293 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 44/156 2024-04-25T00:20:56.354 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 40/156 2024-04-25T00:20:56.384 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 41/156 2024-04-25T00:20:56.426 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : qatzip-libs-1.2.0-1.el9.x86_64 42/156 2024-04-25T00:20:56.439 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 45/156 2024-04-25T00:20:56.465 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : librados-devel-2:19.0.0-2445.gd2c45d97.el9.x86_6 43/156 2024-04-25T00:20:56.501 INFO:teuthology.orchestra.run.smithi032.stdout: Upgrading : openssl-1:3.2.1-1.el9.x86_64 46/156 2024-04-25T00:20:56.506 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 44/156 2024-04-25T00:20:56.557 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 47/156 2024-04-25T00:20:56.632 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 48/156 2024-04-25T00:20:56.668 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 49/156 2024-04-25T00:20:56.668 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 45/156 2024-04-25T00:20:56.696 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 50/156 2024-04-25T00:20:56.726 INFO:teuthology.orchestra.run.smithi121.stdout: Upgrading : openssl-1:3.2.1-1.el9.x86_64 46/156 2024-04-25T00:20:56.744 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 51/156 2024-04-25T00:20:56.779 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 52/156 2024-04-25T00:20:56.780 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 47/156 2024-04-25T00:20:56.822 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-pyperclip-1.8.2-2.el9.noarch 53/156 2024-04-25T00:20:56.853 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 48/156 2024-04-25T00:20:56.865 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 54/156 2024-04-25T00:20:56.884 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 49/156 2024-04-25T00:20:56.921 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 50/156 2024-04-25T00:20:56.968 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 51/156 2024-04-25T00:20:57.009 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 55/156 2024-04-25T00:20:57.012 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 52/156 2024-04-25T00:20:57.054 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-pyperclip-1.8.2-2.el9.noarch 53/156 2024-04-25T00:20:57.062 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 56/156 2024-04-25T00:20:57.092 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-colorama-0.4.4-12.el9.noarch 57/156 2024-04-25T00:20:57.097 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 54/156 2024-04-25T00:20:57.134 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 58/156 2024-04-25T00:20:57.232 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 55/156 2024-04-25T00:20:57.232 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 59/156 2024-04-25T00:20:57.275 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-google-auth-1:2.29.0-1.el9.noarch 60/156 2024-04-25T00:20:57.278 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 56/156 2024-04-25T00:20:57.308 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-colorama-0.4.4-12.el9.noarch 57/156 2024-04-25T00:20:57.315 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 61/156 2024-04-25T00:20:57.340 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 62/156 2024-04-25T00:20:57.349 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 58/156 2024-04-25T00:20:57.364 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libarrow-doc-9.0.0-10.el9.noarch 63/156 2024-04-25T00:20:57.443 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 59/156 2024-04-25T00:20:57.444 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 64/156 2024-04-25T00:20:57.475 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-google-auth-1:2.29.0-1.el9.noarch 60/156 2024-04-25T00:20:57.514 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 61/156 2024-04-25T00:20:57.542 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 62/156 2024-04-25T00:20:57.564 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : libarrow-doc-9.0.0-10.el9.noarch 63/156 2024-04-25T00:20:57.573 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : abseil-cpp-20211102.0-3.el9.x86_64 65/156 2024-04-25T00:20:57.606 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 66/156 2024-04-25T00:20:57.634 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : zlib-devel-1.2.11-41.el9.x86_64 67/156 2024-04-25T00:20:57.646 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 64/156 2024-04-25T00:20:57.668 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 68/156 2024-04-25T00:20:57.708 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : uuid-1.6.2-55.el9.x86_64 69/156 2024-04-25T00:20:57.748 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-wcwidth-0.2.5-8.el9.noarch 70/156 2024-04-25T00:20:57.781 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : abseil-cpp-20211102.0-3.el9.x86_64 65/156 2024-04-25T00:20:57.785 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 71/156 2024-04-25T00:20:57.813 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 66/156 2024-04-25T00:20:57.820 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 72/156 2024-04-25T00:20:57.841 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : zlib-devel-1.2.11-41.el9.x86_64 67/156 2024-04-25T00:20:57.853 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 73/156 2024-04-25T00:20:57.876 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 68/156 2024-04-25T00:20:57.895 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 74/156 2024-04-25T00:20:57.915 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : uuid-1.6.2-55.el9.x86_64 69/156 2024-04-25T00:20:57.938 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 75/156 2024-04-25T00:20:57.946 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-wcwidth-0.2.5-8.el9.noarch 70/156 2024-04-25T00:20:57.976 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 71/156 2024-04-25T00:20:58.003 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 72/156 2024-04-25T00:20:58.029 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 76/156 2024-04-25T00:20:58.036 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 73/156 2024-04-25T00:20:58.077 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-protobuf-3.14.0-13.el9.noarch 77/156 2024-04-25T00:20:58.078 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 74/156 2024-04-25T00:20:58.108 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 78/156 2024-04-25T00:20:58.111 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 75/156 2024-04-25T00:20:58.157 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 79/156 2024-04-25T00:20:58.198 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 76/156 2024-04-25T00:20:58.241 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-protobuf-3.14.0-13.el9.noarch 77/156 2024-04-25T00:20:58.259 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-packaging-20.9-5.el9.noarch 80/156 2024-04-25T00:20:58.273 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 78/156 2024-04-25T00:20:58.319 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 81/156 2024-04-25T00:20:58.321 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 79/156 2024-04-25T00:20:58.418 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-packaging-20.9-5.el9.noarch 80/156 2024-04-25T00:20:58.753 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 81/156 2024-04-25T00:20:59.106 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 82/156 2024-04-25T00:20:59.148 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 83/156 2024-04-25T00:20:59.182 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 84/156 2024-04-25T00:20:59.289 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 85/156 2024-04-25T00:20:59.350 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 86/156 2024-04-25T00:20:59.441 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 87/156 2024-04-25T00:20:59.476 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-cmd2-2.4.2-2.el9.noarch 88/156 2024-04-25T00:20:59.482 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 82/156 2024-04-25T00:20:59.522 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 83/156 2024-04-25T00:20:59.556 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 84/156 2024-04-25T00:20:59.653 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 85/156 2024-04-25T00:20:59.660 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : openblas-0.3.21-2.el9.x86_64 89/156 2024-04-25T00:20:59.690 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 90/156 2024-04-25T00:20:59.705 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 86/156 2024-04-25T00:20:59.770 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 91/156 2024-04-25T00:20:59.783 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 87/156 2024-04-25T00:20:59.833 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-cmd2-2.4.2-2.el9.noarch 88/156 2024-04-25T00:20:59.951 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-selinux-2:19.0.0-2445.gd2c45d97.el9.x86_64 107/156 2024-04-25T00:20:59.951 INFO:teuthology.orchestra.run.smithi130.stdout:skipping the directory /sys 2024-04-25T00:20:59.951 INFO:teuthology.orchestra.run.smithi130.stdout:skipping the directory /proc 2024-04-25T00:20:59.951 INFO:teuthology.orchestra.run.smithi130.stdout:skipping the directory /mnt 2024-04-25T00:20:59.951 INFO:teuthology.orchestra.run.smithi130.stdout:skipping the directory /var/tmp 2024-04-25T00:20:59.951 INFO:teuthology.orchestra.run.smithi130.stdout:skipping the directory /home 2024-04-25T00:20:59.951 INFO:teuthology.orchestra.run.smithi130.stdout:skipping the directory /root 2024-04-25T00:20:59.951 INFO:teuthology.orchestra.run.smithi130.stdout:skipping the directory /tmp 2024-04-25T00:20:59.951 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-25T00:21:00.010 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : openblas-0.3.21-2.el9.x86_64 89/156 2024-04-25T00:21:00.039 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 90/156 2024-04-25T00:21:00.124 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 91/156 2024-04-25T00:21:00.126 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-volume-2:19.0.0-2445.gd2c45d97.el9.noarch 108/156 2024-04-25T00:21:00.142 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-volume-2:19.0.0-2445.gd2c45d97.el9.noarch 108/156 2024-04-25T00:21:00.142 INFO:teuthology.orchestra.run.smithi130.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:00.142 INFO:teuthology.orchestra.run.smithi130.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2024-04-25T00:21:00.142 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-25T00:21:00.468 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 92/156 2024-04-25T00:21:00.585 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 93/156 2024-04-25T00:21:00.762 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 92/156 2024-04-25T00:21:00.878 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 93/156 2024-04-25T00:21:01.621 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-osd-2:19.0.0-2445.gd2c45d97.el9.x86_64 109/156 2024-04-25T00:21:01.646 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-osd-2:19.0.0-2445.gd2c45d97.el9.x86_64 109/156 2024-04-25T00:21:01.646 INFO:teuthology.orchestra.run.smithi130.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:01.647 INFO:teuthology.orchestra.run.smithi130.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-25T00:21:01.647 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-25T00:21:01.647 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-25T00:21:01.647 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-25T00:21:01.872 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-mds-2:19.0.0-2445.gd2c45d97.el9.x86_64 110/156 2024-04-25T00:21:01.894 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-mds-2:19.0.0-2445.gd2c45d97.el9.x86_64 110/156 2024-04-25T00:21:01.894 INFO:teuthology.orchestra.run.smithi130.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:01.894 INFO:teuthology.orchestra.run.smithi130.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-25T00:21:01.894 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-25T00:21:01.894 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-25T00:21:01.895 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-25T00:21:01.919 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 94/156 2024-04-25T00:21:02.011 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 95/156 2024-04-25T00:21:02.041 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libzstd-devel-1.5.1-2.el9.x86_64 96/156 2024-04-25T00:21:02.073 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libuuid-devel-2.37.4-18.el9.x86_64 97/156 2024-04-25T00:21:02.100 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 94/156 2024-04-25T00:21:02.193 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 95/156 2024-04-25T00:21:02.220 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : libzstd-devel-1.5.1-2.el9.x86_64 96/156 2024-04-25T00:21:02.254 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : libuuid-devel-2.37.4-18.el9.x86_64 97/156 2024-04-25T00:21:02.340 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-mon-2:19.0.0-2445.gd2c45d97.el9.x86_64 111/156 2024-04-25T00:21:02.362 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-mon-2:19.0.0-2445.gd2c45d97.el9.x86_64 111/156 2024-04-25T00:21:02.362 INFO:teuthology.orchestra.run.smithi130.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:02.362 INFO:teuthology.orchestra.run.smithi130.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-25T00:21:02.362 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-25T00:21:02.363 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-25T00:21:02.363 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-25T00:21:02.401 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 98/156 2024-04-25T00:21:02.435 INFO:teuthology.orchestra.run.smithi032.stdout: Upgrading : librbd1-2:19.0.0-2445.gd2c45d97.el9.x86_64 99/156 2024-04-25T00:21:02.469 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : automake-1.16.2-8.el9.noarch 112/156 2024-04-25T00:21:02.518 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-ply-3.11-14.el9.noarch 113/156 2024-04-25T00:21:02.527 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: librbd1-2:19.0.0-2445.gd2c45d97.el9.x86_64 99/156 2024-04-25T00:21:02.557 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-rbd-2:19.0.0-2445.gd2c45d97.el9.x86_64 100/156 2024-04-25T00:21:02.565 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 114/156 2024-04-25T00:21:02.566 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 98/156 2024-04-25T00:21:02.594 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 101/156 2024-04-25T00:21:02.608 INFO:teuthology.orchestra.run.smithi121.stdout: Upgrading : librbd1-2:19.0.0-2445.gd2c45d97.el9.x86_64 99/156 2024-04-25T00:21:02.699 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: librbd1-2:19.0.0-2445.gd2c45d97.el9.x86_64 99/156 2024-04-25T00:21:02.730 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-rbd-2:19.0.0-2445.gd2c45d97.el9.x86_64 100/156 2024-04-25T00:21:02.748 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 115/156 2024-04-25T00:21:02.766 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 101/156 2024-04-25T00:21:02.790 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 116/156 2024-04-25T00:21:02.867 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 117/156 2024-04-25T00:21:02.955 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 118/156 2024-04-25T00:21:03.090 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : parquet-libs-9.0.0-10.el9.x86_64 102/156 2024-04-25T00:21:03.102 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 119/156 2024-04-25T00:21:03.122 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : librgw2-2:19.0.0-2445.gd2c45d97.el9.x86_64 103/156 2024-04-25T00:21:03.140 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 120/156 2024-04-25T00:21:03.172 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 121/156 2024-04-25T00:21:03.176 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: librgw2-2:19.0.0-2445.gd2c45d97.el9.x86_64 103/156 2024-04-25T00:21:03.200 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : fuse-2.9.9-15.el9.x86_64 122/156 2024-04-25T00:21:03.206 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-rgw-2:19.0.0-2445.gd2c45d97.el9.x86_64 104/156 2024-04-25T00:21:03.237 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : parquet-libs-9.0.0-10.el9.x86_64 102/156 2024-04-25T00:21:03.269 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : librgw2-2:19.0.0-2445.gd2c45d97.el9.x86_64 103/156 2024-04-25T00:21:03.341 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: librgw2-2:19.0.0-2445.gd2c45d97.el9.x86_64 103/156 2024-04-25T00:21:03.372 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-rgw-2:19.0.0-2445.gd2c45d97.el9.x86_64 104/156 2024-04-25T00:21:03.540 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: cephadm-2:19.0.0-2445.gd2c45d97.el9.noarch 123/156 2024-04-25T00:21:03.573 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : cephadm-2:19.0.0-2445.gd2c45d97.el9.noarch 123/156 2024-04-25T00:21:03.613 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-prometheus-alerts-2:19.0.0-2445.gd2c45d97.e 124/156 2024-04-25T00:21:03.675 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-grafana-dashboards-2:19.0.0-2445.gd2c45d97. 125/156 2024-04-25T00:21:03.713 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-mgr-cephadm-2:19.0.0-2445.gd2c45d97.el9.noa 126/156 2024-04-25T00:21:04.745 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.0.0-2445.gd2c45d97.el9.noa 126/156 2024-04-25T00:21:04.782 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-mgr-dashboard-2:19.0.0-2445.gd2c45d97.el9.n 127/156 2024-04-25T00:21:05.660 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-common-2:19.0.0-2445.gd2c45d97.el9.x86_64 105/156 2024-04-25T00:21:05.714 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-common-2:19.0.0-2445.gd2c45d97.el9.x86_64 105/156 2024-04-25T00:21:05.745 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-common-2:19.0.0-2445.gd2c45d97.el9.x86_64 105/156 2024-04-25T00:21:05.750 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.0.0-2445.gd2c45d97.el9.n 127/156 2024-04-25T00:21:05.782 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-mgr-diskprediction-local-2:19.0.0-2445.gd2c 128/156 2024-04-25T00:21:05.882 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.0.0-2445.gd2c 128/156 2024-04-25T00:21:05.909 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-common-2:19.0.0-2445.gd2c45d97.el9.x86_64 105/156 2024-04-25T00:21:06.055 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-mgr-modules-core-2:19.0.0-2445.gd2c45d97.el 129/156 2024-04-25T00:21:06.085 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-mgr-2:19.0.0-2445.gd2c45d97.el9.x86_64 130/156 2024-04-25T00:21:06.107 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-mgr-2:19.0.0-2445.gd2c45d97.el9.x86_64 130/156 2024-04-25T00:21:06.107 INFO:teuthology.orchestra.run.smithi130.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:06.107 INFO:teuthology.orchestra.run.smithi130.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-25T00:21:06.108 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-25T00:21:06.108 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-25T00:21:06.108 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-25T00:21:06.152 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-mgr-rook-2:19.0.0-2445.gd2c45d97.el9.noarch 131/156 2024-04-25T00:21:06.162 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-mgr-rook-2:19.0.0-2445.gd2c45d97.el9.noarch 131/156 2024-04-25T00:21:06.260 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-2:19.0.0-2445.gd2c45d97.el9.x86_64 132/156 2024-04-25T00:21:06.308 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-fuse-2:19.0.0-2445.gd2c45d97.el9.x86_64 133/156 2024-04-25T00:21:06.316 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-common-2:19.0.0-2445.gd2c45d97.el9.x86_64 105/156 2024-04-25T00:21:06.366 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-base-2:19.0.0-2445.gd2c45d97.el9.x86_64 106/156 2024-04-25T00:21:06.450 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-common-2:19.0.0-2445.gd2c45d97.el9.x86_64 105/156 2024-04-25T00:21:06.459 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-base-2:19.0.0-2445.gd2c45d97.el9.x86_64 106/156 2024-04-25T00:21:06.459 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-25T00:21:06.459 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-25T00:21:06.459 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:21:06.488 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-base-2:19.0.0-2445.gd2c45d97.el9.x86_64 106/156 2024-04-25T00:21:06.495 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-selinux-2:19.0.0-2445.gd2c45d97.el9.x86_64 107/156 2024-04-25T00:21:06.571 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-base-2:19.0.0-2445.gd2c45d97.el9.x86_64 106/156 2024-04-25T00:21:06.571 INFO:teuthology.orchestra.run.smithi121.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-25T00:21:06.571 INFO:teuthology.orchestra.run.smithi121.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-25T00:21:06.572 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-25T00:21:06.632 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-selinux-2:19.0.0-2445.gd2c45d97.el9.x86_64 107/156 2024-04-25T00:21:07.193 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libtool-2.4.6-45.el9.x86_64 134/156 2024-04-25T00:21:07.221 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-radosgw-2:19.0.0-2445.gd2c45d97.el9.x86_64 135/156 2024-04-25T00:21:07.243 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-radosgw-2:19.0.0-2445.gd2c45d97.el9.x86_64 135/156 2024-04-25T00:21:07.244 INFO:teuthology.orchestra.run.smithi130.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:07.244 INFO:teuthology.orchestra.run.smithi130.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-25T00:21:07.244 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-25T00:21:07.244 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-25T00:21:07.244 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-25T00:21:07.288 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-immutable-object-cache-2:19.0.0-2445.gd2c45 136/156 2024-04-25T00:21:07.310 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.0.0-2445.gd2c45 136/156 2024-04-25T00:21:07.310 INFO:teuthology.orchestra.run.smithi130.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:07.310 INFO:teuthology.orchestra.run.smithi130.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-25T00:21:07.310 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-25T00:21:07.364 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : cephfs-mirror-2:19.0.0-2445.gd2c45d97.el9.x86_64 137/156 2024-04-25T00:21:07.386 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: cephfs-mirror-2:19.0.0-2445.gd2c45d97.el9.x86_64 137/156 2024-04-25T00:21:07.387 INFO:teuthology.orchestra.run.smithi130.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:07.387 INFO:teuthology.orchestra.run.smithi130.stdout:Invalid unit name "cephfs-mirror@*.service" escaped as "cephfs-mirror@\x2a.service". 2024-04-25T00:21:07.387 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-04-25T00:21:07.387 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/ceph.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-04-25T00:21:07.387 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-25T00:21:07.690 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : rbd-mirror-2:19.0.0-2445.gd2c45d97.el9.x86_64 138/156 2024-04-25T00:21:07.712 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: rbd-mirror-2:19.0.0-2445.gd2c45d97.el9.x86_64 138/156 2024-04-25T00:21:07.712 INFO:teuthology.orchestra.run.smithi130.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:07.713 INFO:teuthology.orchestra.run.smithi130.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-25T00:21:07.713 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-25T00:21:07.713 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-25T00:21:07.713 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-25T00:21:12.219 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-test-2:19.0.0-2445.gd2c45d97.el9.x86_64 139/156 2024-04-25T00:21:12.260 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : rbd-fuse-2:19.0.0-2445.gd2c45d97.el9.x86_64 140/156 2024-04-25T00:21:12.309 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : rbd-nbd-2:19.0.0-2445.gd2c45d97.el9.x86_64 141/156 2024-04-25T00:21:12.341 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : xfsprogs-devel-6.3.0-1.el9.x86_64 142/156 2024-04-25T00:21:12.369 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : elfutils-libelf-devel-0.190-2.el9.x86_64 143/156 2024-04-25T00:21:12.551 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : cephfs-shell-2:19.0.0-2445.gd2c45d97.el9.x86_64 144/156 2024-04-25T00:21:12.598 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : postgresql-contrib-13.14-1.el9.x86_64 145/156 2024-04-25T00:21:12.628 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libcephfs-devel-2:19.0.0-2445.gd2c45d97.el9.x86_ 146/156 2024-04-25T00:21:13.173 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: postgresql-server-13.14-1.el9.x86_64 147/156 2024-04-25T00:21:13.209 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : postgresql-server-13.14-1.el9.x86_64 147/156 2024-04-25T00:21:13.229 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: postgresql-server-13.14-1.el9.x86_64 147/156 2024-04-25T00:21:14.052 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : cephfs-top-2:19.0.0-2445.gd2c45d97.el9.noarch 148/156 2024-04-25T00:21:14.139 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : openssl-devel-1:3.2.1-1.el9.x86_64 149/156 2024-04-25T00:21:14.180 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libaio-devel-0.3.111-13.el9.x86_64 150/156 2024-04-25T00:21:14.265 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : flex-2.6.4-9.el9.x86_64 151/156 2024-04-25T00:21:14.294 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : bison-3.7.4-5.el9.x86_64 152/156 2024-04-25T00:21:14.294 INFO:teuthology.orchestra.run.smithi130.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 153/156 2024-04-25T00:21:14.378 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 153/156 2024-04-25T00:21:14.402 INFO:teuthology.orchestra.run.smithi130.stdout: Cleanup : openssl-1:3.0.7-27.el9.x86_64 154/156 2024-04-25T00:21:14.403 INFO:teuthology.orchestra.run.smithi130.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 155/156 2024-04-25T00:21:14.489 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 155/156 2024-04-25T00:21:14.559 INFO:teuthology.orchestra.run.smithi130.stdout: Cleanup : openssl-libs-1:3.0.7-27.el9.x86_64 156/156 2024-04-25T00:21:18.247 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-selinux-2:19.0.0-2445.gd2c45d97.el9.x86_64 107/156 2024-04-25T00:21:18.248 INFO:teuthology.orchestra.run.smithi121.stdout:skipping the directory /sys 2024-04-25T00:21:18.248 INFO:teuthology.orchestra.run.smithi121.stdout:skipping the directory /proc 2024-04-25T00:21:18.248 INFO:teuthology.orchestra.run.smithi121.stdout:skipping the directory /mnt 2024-04-25T00:21:18.248 INFO:teuthology.orchestra.run.smithi121.stdout:skipping the directory /var/tmp 2024-04-25T00:21:18.248 INFO:teuthology.orchestra.run.smithi121.stdout:skipping the directory /home 2024-04-25T00:21:18.248 INFO:teuthology.orchestra.run.smithi121.stdout:skipping the directory /root 2024-04-25T00:21:18.249 INFO:teuthology.orchestra.run.smithi121.stdout:skipping the directory /tmp 2024-04-25T00:21:18.249 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-25T00:21:18.410 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-volume-2:19.0.0-2445.gd2c45d97.el9.noarch 108/156 2024-04-25T00:21:18.424 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-volume-2:19.0.0-2445.gd2c45d97.el9.noarch 108/156 2024-04-25T00:21:18.424 INFO:teuthology.orchestra.run.smithi121.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:18.425 INFO:teuthology.orchestra.run.smithi121.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2024-04-25T00:21:18.425 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-25T00:21:18.480 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-selinux-2:19.0.0-2445.gd2c45d97.el9.x86_64 107/156 2024-04-25T00:21:18.480 INFO:teuthology.orchestra.run.smithi032.stdout:skipping the directory /sys 2024-04-25T00:21:18.480 INFO:teuthology.orchestra.run.smithi032.stdout:skipping the directory /proc 2024-04-25T00:21:18.480 INFO:teuthology.orchestra.run.smithi032.stdout:skipping the directory /mnt 2024-04-25T00:21:18.480 INFO:teuthology.orchestra.run.smithi032.stdout:skipping the directory /var/tmp 2024-04-25T00:21:18.481 INFO:teuthology.orchestra.run.smithi032.stdout:skipping the directory /home 2024-04-25T00:21:18.481 INFO:teuthology.orchestra.run.smithi032.stdout:skipping the directory /root 2024-04-25T00:21:18.481 INFO:teuthology.orchestra.run.smithi032.stdout:skipping the directory /tmp 2024-04-25T00:21:18.481 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:21:18.657 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-volume-2:19.0.0-2445.gd2c45d97.el9.noarch 108/156 2024-04-25T00:21:18.673 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-volume-2:19.0.0-2445.gd2c45d97.el9.noarch 108/156 2024-04-25T00:21:18.673 INFO:teuthology.orchestra.run.smithi032.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:18.673 INFO:teuthology.orchestra.run.smithi032.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2024-04-25T00:21:18.673 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:21:19.869 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-osd-2:19.0.0-2445.gd2c45d97.el9.x86_64 109/156 2024-04-25T00:21:19.893 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-osd-2:19.0.0-2445.gd2c45d97.el9.x86_64 109/156 2024-04-25T00:21:19.893 INFO:teuthology.orchestra.run.smithi121.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:19.893 INFO:teuthology.orchestra.run.smithi121.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-25T00:21:19.894 INFO:teuthology.orchestra.run.smithi121.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-25T00:21:19.894 INFO:teuthology.orchestra.run.smithi121.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-25T00:21:19.894 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-25T00:21:20.111 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-mds-2:19.0.0-2445.gd2c45d97.el9.x86_64 110/156 2024-04-25T00:21:20.133 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-mds-2:19.0.0-2445.gd2c45d97.el9.x86_64 110/156 2024-04-25T00:21:20.133 INFO:teuthology.orchestra.run.smithi121.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:20.133 INFO:teuthology.orchestra.run.smithi121.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-25T00:21:20.133 INFO:teuthology.orchestra.run.smithi121.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-25T00:21:20.133 INFO:teuthology.orchestra.run.smithi121.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-25T00:21:20.133 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-25T00:21:20.218 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-osd-2:19.0.0-2445.gd2c45d97.el9.x86_64 109/156 2024-04-25T00:21:20.245 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-osd-2:19.0.0-2445.gd2c45d97.el9.x86_64 109/156 2024-04-25T00:21:20.246 INFO:teuthology.orchestra.run.smithi032.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:20.246 INFO:teuthology.orchestra.run.smithi032.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-25T00:21:20.246 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-25T00:21:20.246 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-25T00:21:20.246 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:21:20.325 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: openssl-libs-1:3.0.7-27.el9.x86_64 156/156 2024-04-25T00:21:20.326 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-2:19.0.0-2445.gd2c45d97.el9.x86_64 1/156 2024-04-25T00:21:20.326 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-base-2:19.0.0-2445.gd2c45d97.el9.x86_64 2/156 2024-04-25T00:21:20.326 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-common-2:19.0.0-2445.gd2c45d97.el9.x86_64 3/156 2024-04-25T00:21:20.326 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-fuse-2:19.0.0-2445.gd2c45d97.el9.x86_64 4/156 2024-04-25T00:21:20.326 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-immutable-object-cache-2:19.0.0-2445.gd2c45 5/156 2024-04-25T00:21:20.326 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-mds-2:19.0.0-2445.gd2c45d97.el9.x86_64 6/156 2024-04-25T00:21:20.326 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-mgr-2:19.0.0-2445.gd2c45d97.el9.x86_64 7/156 2024-04-25T00:21:20.326 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-mon-2:19.0.0-2445.gd2c45d97.el9.x86_64 8/156 2024-04-25T00:21:20.326 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-osd-2:19.0.0-2445.gd2c45d97.el9.x86_64 9/156 2024-04-25T00:21:20.326 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-radosgw-2:19.0.0-2445.gd2c45d97.el9.x86_64 10/156 2024-04-25T00:21:20.326 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-selinux-2:19.0.0-2445.gd2c45d97.el9.x86_64 11/156 2024-04-25T00:21:20.326 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-test-2:19.0.0-2445.gd2c45d97.el9.x86_64 12/156 2024-04-25T00:21:20.327 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : cephfs-mirror-2:19.0.0-2445.gd2c45d97.el9.x86_64 13/156 2024-04-25T00:21:20.327 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : cephfs-shell-2:19.0.0-2445.gd2c45d97.el9.x86_64 14/156 2024-04-25T00:21:20.327 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libcephfs-devel-2:19.0.0-2445.gd2c45d97.el9.x86_ 15/156 2024-04-25T00:21:20.327 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libcephfs2-2:19.0.0-2445.gd2c45d97.el9.x86_64 16/156 2024-04-25T00:21:20.327 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libcephsqlite-2:19.0.0-2445.gd2c45d97.el9.x86_64 17/156 2024-04-25T00:21:20.327 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : librados-devel-2:19.0.0-2445.gd2c45d97.el9.x86_6 18/156 2024-04-25T00:21:20.327 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libradosstriper1-2:19.0.0-2445.gd2c45d97.el9.x86 19/156 2024-04-25T00:21:20.327 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : librgw2-2:19.0.0-2445.gd2c45d97.el9.x86_64 20/156 2024-04-25T00:21:20.327 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-ceph-argparse-2:19.0.0-2445.gd2c45d97.el 21/156 2024-04-25T00:21:20.327 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-ceph-common-2:19.0.0-2445.gd2c45d97.el9. 22/156 2024-04-25T00:21:20.327 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-cephfs-2:19.0.0-2445.gd2c45d97.el9.x86_6 23/156 2024-04-25T00:21:20.327 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-rados-2:19.0.0-2445.gd2c45d97.el9.x86_64 24/156 2024-04-25T00:21:20.327 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-rbd-2:19.0.0-2445.gd2c45d97.el9.x86_64 25/156 2024-04-25T00:21:20.340 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-rgw-2:19.0.0-2445.gd2c45d97.el9.x86_64 26/156 2024-04-25T00:21:20.340 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : rbd-fuse-2:19.0.0-2445.gd2c45d97.el9.x86_64 27/156 2024-04-25T00:21:20.340 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : rbd-mirror-2:19.0.0-2445.gd2c45d97.el9.x86_64 28/156 2024-04-25T00:21:20.340 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : rbd-nbd-2:19.0.0-2445.gd2c45d97.el9.x86_64 29/156 2024-04-25T00:21:20.340 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-grafana-dashboards-2:19.0.0-2445.gd2c45d97. 30/156 2024-04-25T00:21:20.340 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-mgr-cephadm-2:19.0.0-2445.gd2c45d97.el9.noa 31/156 2024-04-25T00:21:20.340 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-mgr-dashboard-2:19.0.0-2445.gd2c45d97.el9.n 32/156 2024-04-25T00:21:20.341 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.0.0-2445.gd2c 33/156 2024-04-25T00:21:20.341 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-mgr-modules-core-2:19.0.0-2445.gd2c45d97.el 34/156 2024-04-25T00:21:20.341 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-mgr-rook-2:19.0.0-2445.gd2c45d97.el9.noarch 35/156 2024-04-25T00:21:20.341 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-prometheus-alerts-2:19.0.0-2445.gd2c45d97.e 36/156 2024-04-25T00:21:20.341 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-volume-2:19.0.0-2445.gd2c45d97.el9.noarch 37/156 2024-04-25T00:21:20.341 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : cephadm-2:19.0.0-2445.gd2c45d97.el9.noarch 38/156 2024-04-25T00:21:20.341 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : cephfs-top-2:19.0.0-2445.gd2c45d97.el9.noarch 39/156 2024-04-25T00:21:20.341 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 40/156 2024-04-25T00:21:20.341 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 41/156 2024-04-25T00:21:20.341 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 42/156 2024-04-25T00:21:20.341 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 43/156 2024-04-25T00:21:20.341 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 44/156 2024-04-25T00:21:20.341 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-ply-3.11-14.el9.noarch 45/156 2024-04-25T00:21:20.341 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 46/156 2024-04-25T00:21:20.342 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 47/156 2024-04-25T00:21:20.342 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 48/156 2024-04-25T00:21:20.342 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : automake-1.16.2-8.el9.noarch 49/156 2024-04-25T00:21:20.342 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : bison-3.7.4-5.el9.x86_64 50/156 2024-04-25T00:21:20.342 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 51/156 2024-04-25T00:21:20.342 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : elfutils-libelf-devel-0.190-2.el9.x86_64 52/156 2024-04-25T00:21:20.342 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : flex-2.6.4-9.el9.x86_64 53/156 2024-04-25T00:21:20.342 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 54/156 2024-04-25T00:21:20.342 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 55/156 2024-04-25T00:21:20.342 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 56/156 2024-04-25T00:21:20.342 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libaio-devel-0.3.111-13.el9.x86_64 57/156 2024-04-25T00:21:20.342 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 58/156 2024-04-25T00:21:20.342 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 59/156 2024-04-25T00:21:20.343 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 60/156 2024-04-25T00:21:20.343 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libtool-2.4.6-45.el9.x86_64 61/156 2024-04-25T00:21:20.343 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libuuid-devel-2.37.4-18.el9.x86_64 62/156 2024-04-25T00:21:20.343 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libzstd-devel-1.5.1-2.el9.x86_64 63/156 2024-04-25T00:21:20.343 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 64/156 2024-04-25T00:21:20.343 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : lua-5.4.4-4.el9.x86_64 65/156 2024-04-25T00:21:20.343 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 66/156 2024-04-25T00:21:20.343 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 67/156 2024-04-25T00:21:20.343 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : openssl-devel-1:3.2.1-1.el9.x86_64 68/156 2024-04-25T00:21:20.343 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : postgresql-13.14-1.el9.x86_64 69/156 2024-04-25T00:21:20.343 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : postgresql-contrib-13.14-1.el9.x86_64 70/156 2024-04-25T00:21:20.343 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : postgresql-private-libs-13.14-1.el9.x86_64 71/156 2024-04-25T00:21:20.343 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : postgresql-server-13.14-1.el9.x86_64 72/156 2024-04-25T00:21:20.344 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : protobuf-3.14.0-13.el9.x86_64 73/156 2024-04-25T00:21:20.344 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 74/156 2024-04-25T00:21:20.344 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 75/156 2024-04-25T00:21:20.344 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 76/156 2024-04-25T00:21:20.344 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 77/156 2024-04-25T00:21:20.344 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 78/156 2024-04-25T00:21:20.344 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 79/156 2024-04-25T00:21:20.344 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 80/156 2024-04-25T00:21:20.344 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 81/156 2024-04-25T00:21:20.344 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 82/156 2024-04-25T00:21:20.344 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 83/156 2024-04-25T00:21:20.344 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 84/156 2024-04-25T00:21:20.344 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 85/156 2024-04-25T00:21:20.344 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 86/156 2024-04-25T00:21:20.345 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-protobuf-3.14.0-13.el9.noarch 87/156 2024-04-25T00:21:20.345 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 88/156 2024-04-25T00:21:20.345 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 89/156 2024-04-25T00:21:20.345 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 90/156 2024-04-25T00:21:20.345 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 91/156 2024-04-25T00:21:20.345 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 92/156 2024-04-25T00:21:20.345 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 93/156 2024-04-25T00:21:20.345 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-wcwidth-0.2.5-8.el9.noarch 94/156 2024-04-25T00:21:20.345 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : qatlib-24.02.0-1.el9.x86_64 95/156 2024-04-25T00:21:20.345 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : qatlib-service-24.02.0-1.el9.x86_64 96/156 2024-04-25T00:21:20.345 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : qatzip-libs-1.2.0-1.el9.x86_64 97/156 2024-04-25T00:21:20.345 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 98/156 2024-04-25T00:21:20.345 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : uuid-1.6.2-55.el9.x86_64 99/156 2024-04-25T00:21:20.345 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : xfsprogs-devel-6.3.0-1.el9.x86_64 100/156 2024-04-25T00:21:20.345 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 101/156 2024-04-25T00:21:20.346 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : zlib-devel-1.2.11-41.el9.x86_64 102/156 2024-04-25T00:21:20.346 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 103/156 2024-04-25T00:21:20.346 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : protobuf-compiler-3.14.0-13.el9.x86_64 104/156 2024-04-25T00:21:20.346 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : abseil-cpp-20211102.0-3.el9.x86_64 105/156 2024-04-25T00:21:20.346 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 106/156 2024-04-25T00:21:20.346 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 107/156 2024-04-25T00:21:20.346 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libarrow-9.0.0-10.el9.x86_64 108/156 2024-04-25T00:21:20.346 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libarrow-doc-9.0.0-10.el9.noarch 109/156 2024-04-25T00:21:20.346 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 110/156 2024-04-25T00:21:20.346 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 111/156 2024-04-25T00:21:20.346 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : luarocks-3.9.2-1.el9.noarch 112/156 2024-04-25T00:21:20.346 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : parquet-libs-9.0.0-10.el9.x86_64 113/156 2024-04-25T00:21:20.346 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 114/156 2024-04-25T00:21:20.346 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 115/156 2024-04-25T00:21:20.347 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 116/156 2024-04-25T00:21:20.347 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 117/156 2024-04-25T00:21:20.347 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 118/156 2024-04-25T00:21:20.347 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 119/156 2024-04-25T00:21:20.347 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-cmd2-2.4.2-2.el9.noarch 120/156 2024-04-25T00:21:20.347 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-colorama-0.4.4-12.el9.noarch 121/156 2024-04-25T00:21:20.347 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-google-auth-1:2.29.0-1.el9.noarch 122/156 2024-04-25T00:21:20.347 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 123/156 2024-04-25T00:21:20.347 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 124/156 2024-04-25T00:21:20.347 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 125/156 2024-04-25T00:21:20.347 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 126/156 2024-04-25T00:21:20.347 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 127/156 2024-04-25T00:21:20.347 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 128/156 2024-04-25T00:21:20.347 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 129/156 2024-04-25T00:21:20.347 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 130/156 2024-04-25T00:21:20.348 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 131/156 2024-04-25T00:21:20.348 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 132/156 2024-04-25T00:21:20.348 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 133/156 2024-04-25T00:21:20.348 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 134/156 2024-04-25T00:21:20.348 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 135/156 2024-04-25T00:21:20.348 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 136/156 2024-04-25T00:21:20.348 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-pyperclip-1.8.2-2.el9.noarch 137/156 2024-04-25T00:21:20.348 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 138/156 2024-04-25T00:21:20.348 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 139/156 2024-04-25T00:21:20.348 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 140/156 2024-04-25T00:21:20.348 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 141/156 2024-04-25T00:21:20.348 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 142/156 2024-04-25T00:21:20.348 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 143/156 2024-04-25T00:21:20.348 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 144/156 2024-04-25T00:21:20.348 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 145/156 2024-04-25T00:21:20.349 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 146/156 2024-04-25T00:21:20.349 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : re2-1:20211101-3.el9.x86_64 147/156 2024-04-25T00:21:20.349 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 148/156 2024-04-25T00:21:20.349 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : librados2-2:19.0.0-2445.gd2c45d97.el9.x86_64 149/156 2024-04-25T00:21:20.349 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 150/156 2024-04-25T00:21:20.349 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : librbd1-2:19.0.0-2445.gd2c45d97.el9.x86_64 151/156 2024-04-25T00:21:20.349 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 152/156 2024-04-25T00:21:20.349 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : openssl-1:3.2.1-1.el9.x86_64 153/156 2024-04-25T00:21:20.349 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : openssl-1:3.0.7-27.el9.x86_64 154/156 2024-04-25T00:21:20.349 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : openssl-libs-1:3.2.1-1.el9.x86_64 155/156 2024-04-25T00:21:20.477 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-mds-2:19.0.0-2445.gd2c45d97.el9.x86_64 110/156 2024-04-25T00:21:20.501 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-mds-2:19.0.0-2445.gd2c45d97.el9.x86_64 110/156 2024-04-25T00:21:20.501 INFO:teuthology.orchestra.run.smithi032.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:20.501 INFO:teuthology.orchestra.run.smithi032.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-25T00:21:20.501 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-25T00:21:20.501 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-25T00:21:20.502 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:21:20.578 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-mon-2:19.0.0-2445.gd2c45d97.el9.x86_64 111/156 2024-04-25T00:21:20.600 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-mon-2:19.0.0-2445.gd2c45d97.el9.x86_64 111/156 2024-04-25T00:21:20.600 INFO:teuthology.orchestra.run.smithi121.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:20.600 INFO:teuthology.orchestra.run.smithi121.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-25T00:21:20.601 INFO:teuthology.orchestra.run.smithi121.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-25T00:21:20.601 INFO:teuthology.orchestra.run.smithi121.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-25T00:21:20.601 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-25T00:21:20.708 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : automake-1.16.2-8.el9.noarch 112/156 2024-04-25T00:21:20.756 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-ply-3.11-14.el9.noarch 113/156 2024-04-25T00:21:20.811 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 114/156 2024-04-25T00:21:20.961 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-mon-2:19.0.0-2445.gd2c45d97.el9.x86_64 111/156 2024-04-25T00:21:20.985 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-mon-2:19.0.0-2445.gd2c45d97.el9.x86_64 111/156 2024-04-25T00:21:20.985 INFO:teuthology.orchestra.run.smithi032.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:20.985 INFO:teuthology.orchestra.run.smithi032.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-25T00:21:20.985 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-25T00:21:20.985 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-25T00:21:20.986 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:21:20.995 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 115/156 2024-04-25T00:21:21.046 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 116/156 2024-04-25T00:21:21.115 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 117/156 2024-04-25T00:21:21.117 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : automake-1.16.2-8.el9.noarch 112/156 2024-04-25T00:21:21.176 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-ply-3.11-14.el9.noarch 113/156 2024-04-25T00:21:21.202 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 118/156 2024-04-25T00:21:21.240 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 114/156 2024-04-25T00:21:21.334 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 119/156 2024-04-25T00:21:21.371 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 120/156 2024-04-25T00:21:21.403 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 121/156 2024-04-25T00:21:21.430 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : fuse-2.9.9-15.el9.x86_64 122/156 2024-04-25T00:21:21.436 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 115/156 2024-04-25T00:21:21.480 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 116/156 2024-04-25T00:21:21.553 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 117/156 2024-04-25T00:21:21.651 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 118/156 2024-04-25T00:21:21.795 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 119/156 2024-04-25T00:21:21.838 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 120/156 2024-04-25T00:21:21.870 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 121/156 2024-04-25T00:21:21.878 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: cephadm-2:19.0.0-2445.gd2c45d97.el9.noarch 123/156 2024-04-25T00:21:21.897 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : fuse-2.9.9-15.el9.x86_64 122/156 2024-04-25T00:21:21.904 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : cephadm-2:19.0.0-2445.gd2c45d97.el9.noarch 123/156 2024-04-25T00:21:21.935 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-prometheus-alerts-2:19.0.0-2445.gd2c45d97.e 124/156 2024-04-25T00:21:21.996 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-grafana-dashboards-2:19.0.0-2445.gd2c45d97. 125/156 2024-04-25T00:21:22.036 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-mgr-cephadm-2:19.0.0-2445.gd2c45d97.el9.noa 126/156 2024-04-25T00:21:22.282 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: cephadm-2:19.0.0-2445.gd2c45d97.el9.noarch 123/156 2024-04-25T00:21:22.313 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : cephadm-2:19.0.0-2445.gd2c45d97.el9.noarch 123/156 2024-04-25T00:21:22.345 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-prometheus-alerts-2:19.0.0-2445.gd2c45d97.e 124/156 2024-04-25T00:21:22.410 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-grafana-dashboards-2:19.0.0-2445.gd2c45d97. 125/156 2024-04-25T00:21:22.444 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-mgr-cephadm-2:19.0.0-2445.gd2c45d97.el9.noa 126/156 2024-04-25T00:21:22.470 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : openssl-libs-1:3.0.7-27.el9.x86_64 156/156 2024-04-25T00:21:22.470 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-25T00:21:22.471 INFO:teuthology.orchestra.run.smithi130.stdout:Upgraded: 2024-04-25T00:21:22.471 INFO:teuthology.orchestra.run.smithi130.stdout: librados2-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.471 INFO:teuthology.orchestra.run.smithi130.stdout: librbd1-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.471 INFO:teuthology.orchestra.run.smithi130.stdout: openssl-1:3.2.1-1.el9.x86_64 2024-04-25T00:21:22.471 INFO:teuthology.orchestra.run.smithi130.stdout: openssl-libs-1:3.2.1-1.el9.x86_64 2024-04-25T00:21:22.471 INFO:teuthology.orchestra.run.smithi130.stdout:Installed: 2024-04-25T00:21:22.471 INFO:teuthology.orchestra.run.smithi130.stdout: abseil-cpp-20211102.0-3.el9.x86_64 2024-04-25T00:21:22.471 INFO:teuthology.orchestra.run.smithi130.stdout: automake-1.16.2-8.el9.noarch 2024-04-25T00:21:22.471 INFO:teuthology.orchestra.run.smithi130.stdout: bison-3.7.4-5.el9.x86_64 2024-04-25T00:21:22.471 INFO:teuthology.orchestra.run.smithi130.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-04-25T00:21:22.471 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.471 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-base-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.471 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-common-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.471 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-fuse-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.472 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-grafana-dashboards-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:22.472 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-immutable-object-cache-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.472 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mds-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.472 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.472 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-cephadm-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:22.472 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-dashboard-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:22.472 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-diskprediction-local-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:22.472 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-modules-core-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:22.472 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-rook-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:22.472 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mon-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.472 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-osd-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.472 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-prometheus-alerts-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:22.472 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-radosgw-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.473 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-selinux-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.473 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-test-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.473 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-volume-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:22.473 INFO:teuthology.orchestra.run.smithi130.stdout: cephadm-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:22.473 INFO:teuthology.orchestra.run.smithi130.stdout: cephfs-mirror-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.473 INFO:teuthology.orchestra.run.smithi130.stdout: cephfs-shell-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.473 INFO:teuthology.orchestra.run.smithi130.stdout: cephfs-top-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:22.473 INFO:teuthology.orchestra.run.smithi130.stdout: elfutils-libelf-devel-0.190-2.el9.x86_64 2024-04-25T00:21:22.473 INFO:teuthology.orchestra.run.smithi130.stdout: flex-2.6.4-9.el9.x86_64 2024-04-25T00:21:22.473 INFO:teuthology.orchestra.run.smithi130.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-25T00:21:22.473 INFO:teuthology.orchestra.run.smithi130.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-25T00:21:22.473 INFO:teuthology.orchestra.run.smithi130.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-25T00:21:22.473 INFO:teuthology.orchestra.run.smithi130.stdout: fuse-2.9.9-15.el9.x86_64 2024-04-25T00:21:22.474 INFO:teuthology.orchestra.run.smithi130.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-25T00:21:22.474 INFO:teuthology.orchestra.run.smithi130.stdout: grpc-data-1.46.7-10.el9.noarch 2024-04-25T00:21:22.474 INFO:teuthology.orchestra.run.smithi130.stdout: libaio-devel-0.3.111-13.el9.x86_64 2024-04-25T00:21:22.474 INFO:teuthology.orchestra.run.smithi130.stdout: libarrow-9.0.0-10.el9.x86_64 2024-04-25T00:21:22.474 INFO:teuthology.orchestra.run.smithi130.stdout: libarrow-doc-9.0.0-10.el9.noarch 2024-04-25T00:21:22.474 INFO:teuthology.orchestra.run.smithi130.stdout: libcephfs-devel-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.474 INFO:teuthology.orchestra.run.smithi130.stdout: libcephfs2-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.474 INFO:teuthology.orchestra.run.smithi130.stdout: libcephsqlite-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.474 INFO:teuthology.orchestra.run.smithi130.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-25T00:21:22.474 INFO:teuthology.orchestra.run.smithi130.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-25T00:21:22.474 INFO:teuthology.orchestra.run.smithi130.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-25T00:21:22.474 INFO:teuthology.orchestra.run.smithi130.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-25T00:21:22.475 INFO:teuthology.orchestra.run.smithi130.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-25T00:21:22.475 INFO:teuthology.orchestra.run.smithi130.stdout: librados-devel-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.475 INFO:teuthology.orchestra.run.smithi130.stdout: libradosstriper1-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.475 INFO:teuthology.orchestra.run.smithi130.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-25T00:21:22.475 INFO:teuthology.orchestra.run.smithi130.stdout: librgw2-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.475 INFO:teuthology.orchestra.run.smithi130.stdout: libtool-2.4.6-45.el9.x86_64 2024-04-25T00:21:22.475 INFO:teuthology.orchestra.run.smithi130.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-25T00:21:22.475 INFO:teuthology.orchestra.run.smithi130.stdout: libuuid-devel-2.37.4-18.el9.x86_64 2024-04-25T00:21:22.475 INFO:teuthology.orchestra.run.smithi130.stdout: libzstd-devel-1.5.1-2.el9.x86_64 2024-04-25T00:21:22.475 INFO:teuthology.orchestra.run.smithi130.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-25T00:21:22.475 INFO:teuthology.orchestra.run.smithi130.stdout: lua-5.4.4-4.el9.x86_64 2024-04-25T00:21:22.475 INFO:teuthology.orchestra.run.smithi130.stdout: lua-devel-5.4.4-4.el9.x86_64 2024-04-25T00:21:22.475 INFO:teuthology.orchestra.run.smithi130.stdout: luarocks-3.9.2-1.el9.noarch 2024-04-25T00:21:22.476 INFO:teuthology.orchestra.run.smithi130.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-25T00:21:22.476 INFO:teuthology.orchestra.run.smithi130.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-25T00:21:22.476 INFO:teuthology.orchestra.run.smithi130.stdout: openssl-devel-1:3.2.1-1.el9.x86_64 2024-04-25T00:21:22.476 INFO:teuthology.orchestra.run.smithi130.stdout: parquet-libs-9.0.0-10.el9.x86_64 2024-04-25T00:21:22.476 INFO:teuthology.orchestra.run.smithi130.stdout: postgresql-13.14-1.el9.x86_64 2024-04-25T00:21:22.476 INFO:teuthology.orchestra.run.smithi130.stdout: postgresql-contrib-13.14-1.el9.x86_64 2024-04-25T00:21:22.476 INFO:teuthology.orchestra.run.smithi130.stdout: postgresql-private-libs-13.14-1.el9.x86_64 2024-04-25T00:21:22.476 INFO:teuthology.orchestra.run.smithi130.stdout: postgresql-server-13.14-1.el9.x86_64 2024-04-25T00:21:22.476 INFO:teuthology.orchestra.run.smithi130.stdout: protobuf-3.14.0-13.el9.x86_64 2024-04-25T00:21:22.476 INFO:teuthology.orchestra.run.smithi130.stdout: protobuf-compiler-3.14.0-13.el9.x86_64 2024-04-25T00:21:22.476 INFO:teuthology.orchestra.run.smithi130.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-25T00:21:22.476 INFO:teuthology.orchestra.run.smithi130.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-04-25T00:21:22.476 INFO:teuthology.orchestra.run.smithi130.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-25T00:21:22.476 INFO:teuthology.orchestra.run.smithi130.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-25T00:21:22.476 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-25T00:21:22.477 INFO:teuthology.orchestra.run.smithi130.stdout: python3-ceph-argparse-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.477 INFO:teuthology.orchestra.run.smithi130.stdout: python3-ceph-common-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.477 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cephfs-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.477 INFO:teuthology.orchestra.run.smithi130.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-25T00:21:22.477 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-25T00:21:22.477 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-25T00:21:22.477 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-25T00:21:22.477 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cmd2-2.4.2-2.el9.noarch 2024-04-25T00:21:22.477 INFO:teuthology.orchestra.run.smithi130.stdout: python3-colorama-0.4.4-12.el9.noarch 2024-04-25T00:21:22.477 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-25T00:21:22.477 INFO:teuthology.orchestra.run.smithi130.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-04-25T00:21:22.477 INFO:teuthology.orchestra.run.smithi130.stdout: python3-google-auth-1:2.29.0-1.el9.noarch 2024-04-25T00:21:22.477 INFO:teuthology.orchestra.run.smithi130.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2024-04-25T00:21:22.478 INFO:teuthology.orchestra.run.smithi130.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2024-04-25T00:21:22.478 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-25T00:21:22.478 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-25T00:21:22.478 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-25T00:21:22.478 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-25T00:21:22.478 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-25T00:21:22.478 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-25T00:21:22.478 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-25T00:21:22.478 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-25T00:21:22.479 INFO:teuthology.orchestra.run.smithi130.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-25T00:21:22.479 INFO:teuthology.orchestra.run.smithi130.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-25T00:21:22.479 INFO:teuthology.orchestra.run.smithi130.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-25T00:21:22.479 INFO:teuthology.orchestra.run.smithi130.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-25T00:21:22.479 INFO:teuthology.orchestra.run.smithi130.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-25T00:21:22.479 INFO:teuthology.orchestra.run.smithi130.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-25T00:21:22.479 INFO:teuthology.orchestra.run.smithi130.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-25T00:21:22.479 INFO:teuthology.orchestra.run.smithi130.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-25T00:21:22.479 INFO:teuthology.orchestra.run.smithi130.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-25T00:21:22.479 INFO:teuthology.orchestra.run.smithi130.stdout: python3-packaging-20.9-5.el9.noarch 2024-04-25T00:21:22.480 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-25T00:21:22.480 INFO:teuthology.orchestra.run.smithi130.stdout: python3-ply-3.11-14.el9.noarch 2024-04-25T00:21:22.480 INFO:teuthology.orchestra.run.smithi130.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-25T00:21:22.480 INFO:teuthology.orchestra.run.smithi130.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-25T00:21:22.480 INFO:teuthology.orchestra.run.smithi130.stdout: python3-protobuf-3.14.0-13.el9.noarch 2024-04-25T00:21:22.480 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-25T00:21:22.480 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-25T00:21:22.480 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-25T00:21:22.480 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-25T00:21:22.480 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pyperclip-1.8.2-2.el9.noarch 2024-04-25T00:21:22.480 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-25T00:21:22.481 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rados-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.481 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rbd-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.481 INFO:teuthology.orchestra.run.smithi130.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-25T00:21:22.481 INFO:teuthology.orchestra.run.smithi130.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-25T00:21:22.481 INFO:teuthology.orchestra.run.smithi130.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-25T00:21:22.481 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rgw-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.481 INFO:teuthology.orchestra.run.smithi130.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-25T00:21:22.481 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-25T00:21:22.481 INFO:teuthology.orchestra.run.smithi130.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-25T00:21:22.481 INFO:teuthology.orchestra.run.smithi130.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-25T00:21:22.482 INFO:teuthology.orchestra.run.smithi130.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-25T00:21:22.482 INFO:teuthology.orchestra.run.smithi130.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-25T00:21:22.482 INFO:teuthology.orchestra.run.smithi130.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-25T00:21:22.482 INFO:teuthology.orchestra.run.smithi130.stdout: python3-wcwidth-0.2.5-8.el9.noarch 2024-04-25T00:21:22.482 INFO:teuthology.orchestra.run.smithi130.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-25T00:21:22.482 INFO:teuthology.orchestra.run.smithi130.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-25T00:21:22.482 INFO:teuthology.orchestra.run.smithi130.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-25T00:21:22.482 INFO:teuthology.orchestra.run.smithi130.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-25T00:21:22.482 INFO:teuthology.orchestra.run.smithi130.stdout: qatlib-24.02.0-1.el9.x86_64 2024-04-25T00:21:22.483 INFO:teuthology.orchestra.run.smithi130.stdout: qatlib-service-24.02.0-1.el9.x86_64 2024-04-25T00:21:22.483 INFO:teuthology.orchestra.run.smithi130.stdout: qatzip-libs-1.2.0-1.el9.x86_64 2024-04-25T00:21:22.483 INFO:teuthology.orchestra.run.smithi130.stdout: rbd-fuse-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.483 INFO:teuthology.orchestra.run.smithi130.stdout: rbd-mirror-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.483 INFO:teuthology.orchestra.run.smithi130.stdout: rbd-nbd-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:22.483 INFO:teuthology.orchestra.run.smithi130.stdout: re2-1:20211101-3.el9.x86_64 2024-04-25T00:21:22.483 INFO:teuthology.orchestra.run.smithi130.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-25T00:21:22.483 INFO:teuthology.orchestra.run.smithi130.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-25T00:21:22.483 INFO:teuthology.orchestra.run.smithi130.stdout: uuid-1.6.2-55.el9.x86_64 2024-04-25T00:21:22.483 INFO:teuthology.orchestra.run.smithi130.stdout: xfsprogs-devel-6.3.0-1.el9.x86_64 2024-04-25T00:21:22.484 INFO:teuthology.orchestra.run.smithi130.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-25T00:21:22.484 INFO:teuthology.orchestra.run.smithi130.stdout: zlib-devel-1.2.11-41.el9.x86_64 2024-04-25T00:21:22.484 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-25T00:21:22.484 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-04-25T00:21:22.855 DEBUG:teuthology.parallel:result is None 2024-04-25T00:21:23.068 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.0.0-2445.gd2c45d97.el9.noa 126/156 2024-04-25T00:21:23.095 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-mgr-dashboard-2:19.0.0-2445.gd2c45d97.el9.n 127/156 2024-04-25T00:21:23.538 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.0.0-2445.gd2c45d97.el9.noa 126/156 2024-04-25T00:21:23.571 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-mgr-dashboard-2:19.0.0-2445.gd2c45d97.el9.n 127/156 2024-04-25T00:21:24.055 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.0.0-2445.gd2c45d97.el9.n 127/156 2024-04-25T00:21:24.087 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-mgr-diskprediction-local-2:19.0.0-2445.gd2c 128/156 2024-04-25T00:21:24.189 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.0.0-2445.gd2c 128/156 2024-04-25T00:21:24.350 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-mgr-modules-core-2:19.0.0-2445.gd2c45d97.el 129/156 2024-04-25T00:21:24.381 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-mgr-2:19.0.0-2445.gd2c45d97.el9.x86_64 130/156 2024-04-25T00:21:24.404 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-mgr-2:19.0.0-2445.gd2c45d97.el9.x86_64 130/156 2024-04-25T00:21:24.404 INFO:teuthology.orchestra.run.smithi121.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:24.404 INFO:teuthology.orchestra.run.smithi121.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-25T00:21:24.404 INFO:teuthology.orchestra.run.smithi121.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-25T00:21:24.404 INFO:teuthology.orchestra.run.smithi121.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-25T00:21:24.404 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-25T00:21:24.449 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-mgr-rook-2:19.0.0-2445.gd2c45d97.el9.noarch 131/156 2024-04-25T00:21:24.459 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-mgr-rook-2:19.0.0-2445.gd2c45d97.el9.noarch 131/156 2024-04-25T00:21:24.557 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-2:19.0.0-2445.gd2c45d97.el9.x86_64 132/156 2024-04-25T00:21:24.588 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.0.0-2445.gd2c45d97.el9.n 127/156 2024-04-25T00:21:24.605 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-fuse-2:19.0.0-2445.gd2c45d97.el9.x86_64 133/156 2024-04-25T00:21:24.622 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-mgr-diskprediction-local-2:19.0.0-2445.gd2c 128/156 2024-04-25T00:21:24.730 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.0.0-2445.gd2c 128/156 2024-04-25T00:21:24.910 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-mgr-modules-core-2:19.0.0-2445.gd2c45d97.el 129/156 2024-04-25T00:21:24.966 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-mgr-2:19.0.0-2445.gd2c45d97.el9.x86_64 130/156 2024-04-25T00:21:24.991 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-mgr-2:19.0.0-2445.gd2c45d97.el9.x86_64 130/156 2024-04-25T00:21:24.991 INFO:teuthology.orchestra.run.smithi032.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:24.991 INFO:teuthology.orchestra.run.smithi032.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-25T00:21:24.991 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-25T00:21:24.991 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-25T00:21:24.991 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:21:25.042 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-mgr-rook-2:19.0.0-2445.gd2c45d97.el9.noarch 131/156 2024-04-25T00:21:25.053 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-mgr-rook-2:19.0.0-2445.gd2c45d97.el9.noarch 131/156 2024-04-25T00:21:25.155 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-2:19.0.0-2445.gd2c45d97.el9.x86_64 132/156 2024-04-25T00:21:25.208 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-fuse-2:19.0.0-2445.gd2c45d97.el9.x86_64 133/156 2024-04-25T00:21:25.497 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : libtool-2.4.6-45.el9.x86_64 134/156 2024-04-25T00:21:25.534 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-radosgw-2:19.0.0-2445.gd2c45d97.el9.x86_64 135/156 2024-04-25T00:21:25.556 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-radosgw-2:19.0.0-2445.gd2c45d97.el9.x86_64 135/156 2024-04-25T00:21:25.556 INFO:teuthology.orchestra.run.smithi121.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:25.556 INFO:teuthology.orchestra.run.smithi121.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-25T00:21:25.557 INFO:teuthology.orchestra.run.smithi121.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-25T00:21:25.557 INFO:teuthology.orchestra.run.smithi121.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-25T00:21:25.557 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-25T00:21:25.602 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-immutable-object-cache-2:19.0.0-2445.gd2c45 136/156 2024-04-25T00:21:25.623 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.0.0-2445.gd2c45 136/156 2024-04-25T00:21:25.623 INFO:teuthology.orchestra.run.smithi121.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:25.623 INFO:teuthology.orchestra.run.smithi121.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-25T00:21:25.623 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-25T00:21:25.677 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : cephfs-mirror-2:19.0.0-2445.gd2c45d97.el9.x86_64 137/156 2024-04-25T00:21:25.699 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: cephfs-mirror-2:19.0.0-2445.gd2c45d97.el9.x86_64 137/156 2024-04-25T00:21:25.699 INFO:teuthology.orchestra.run.smithi121.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:25.699 INFO:teuthology.orchestra.run.smithi121.stdout:Invalid unit name "cephfs-mirror@*.service" escaped as "cephfs-mirror@\x2a.service". 2024-04-25T00:21:25.699 INFO:teuthology.orchestra.run.smithi121.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-04-25T00:21:25.699 INFO:teuthology.orchestra.run.smithi121.stdout:Created symlink /etc/systemd/system/ceph.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-04-25T00:21:25.699 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-25T00:21:26.003 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : rbd-mirror-2:19.0.0-2445.gd2c45d97.el9.x86_64 138/156 2024-04-25T00:21:26.024 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: rbd-mirror-2:19.0.0-2445.gd2c45d97.el9.x86_64 138/156 2024-04-25T00:21:26.025 INFO:teuthology.orchestra.run.smithi121.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:26.025 INFO:teuthology.orchestra.run.smithi121.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-25T00:21:26.025 INFO:teuthology.orchestra.run.smithi121.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-25T00:21:26.025 INFO:teuthology.orchestra.run.smithi121.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-25T00:21:26.025 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-25T00:21:26.153 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libtool-2.4.6-45.el9.x86_64 134/156 2024-04-25T00:21:26.194 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-radosgw-2:19.0.0-2445.gd2c45d97.el9.x86_64 135/156 2024-04-25T00:21:26.219 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-radosgw-2:19.0.0-2445.gd2c45d97.el9.x86_64 135/156 2024-04-25T00:21:26.219 INFO:teuthology.orchestra.run.smithi032.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:26.219 INFO:teuthology.orchestra.run.smithi032.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-25T00:21:26.220 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-25T00:21:26.220 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-25T00:21:26.220 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:21:26.262 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-immutable-object-cache-2:19.0.0-2445.gd2c45 136/156 2024-04-25T00:21:26.285 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.0.0-2445.gd2c45 136/156 2024-04-25T00:21:26.285 INFO:teuthology.orchestra.run.smithi032.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:26.285 INFO:teuthology.orchestra.run.smithi032.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-25T00:21:26.285 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:21:26.337 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : cephfs-mirror-2:19.0.0-2445.gd2c45d97.el9.x86_64 137/156 2024-04-25T00:21:26.361 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: cephfs-mirror-2:19.0.0-2445.gd2c45d97.el9.x86_64 137/156 2024-04-25T00:21:26.362 INFO:teuthology.orchestra.run.smithi032.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:26.362 INFO:teuthology.orchestra.run.smithi032.stdout:Invalid unit name "cephfs-mirror@*.service" escaped as "cephfs-mirror@\x2a.service". 2024-04-25T00:21:26.362 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-04-25T00:21:26.362 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/ceph.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-04-25T00:21:26.362 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:21:26.680 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : rbd-mirror-2:19.0.0-2445.gd2c45d97.el9.x86_64 138/156 2024-04-25T00:21:26.704 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: rbd-mirror-2:19.0.0-2445.gd2c45d97.el9.x86_64 138/156 2024-04-25T00:21:26.704 INFO:teuthology.orchestra.run.smithi032.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-25T00:21:26.704 INFO:teuthology.orchestra.run.smithi032.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-25T00:21:26.704 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-25T00:21:26.704 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-25T00:21:26.705 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:21:30.523 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-test-2:19.0.0-2445.gd2c45d97.el9.x86_64 139/156 2024-04-25T00:21:30.564 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : rbd-fuse-2:19.0.0-2445.gd2c45d97.el9.x86_64 140/156 2024-04-25T00:21:30.605 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : rbd-nbd-2:19.0.0-2445.gd2c45d97.el9.x86_64 141/156 2024-04-25T00:21:30.637 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : xfsprogs-devel-6.3.0-1.el9.x86_64 142/156 2024-04-25T00:21:30.665 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : elfutils-libelf-devel-0.190-2.el9.x86_64 143/156 2024-04-25T00:21:30.847 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : cephfs-shell-2:19.0.0-2445.gd2c45d97.el9.x86_64 144/156 2024-04-25T00:21:30.894 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : postgresql-contrib-13.14-1.el9.x86_64 145/156 2024-04-25T00:21:30.924 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : libcephfs-devel-2:19.0.0-2445.gd2c45d97.el9.x86_ 146/156 2024-04-25T00:21:31.467 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-test-2:19.0.0-2445.gd2c45d97.el9.x86_64 139/156 2024-04-25T00:21:31.510 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : rbd-fuse-2:19.0.0-2445.gd2c45d97.el9.x86_64 140/156 2024-04-25T00:21:31.542 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: postgresql-server-13.14-1.el9.x86_64 147/156 2024-04-25T00:21:31.551 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : rbd-nbd-2:19.0.0-2445.gd2c45d97.el9.x86_64 141/156 2024-04-25T00:21:31.588 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : postgresql-server-13.14-1.el9.x86_64 147/156 2024-04-25T00:21:31.604 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : xfsprogs-devel-6.3.0-1.el9.x86_64 142/156 2024-04-25T00:21:31.607 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: postgresql-server-13.14-1.el9.x86_64 147/156 2024-04-25T00:21:31.634 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : elfutils-libelf-devel-0.190-2.el9.x86_64 143/156 2024-04-25T00:21:31.835 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : cephfs-shell-2:19.0.0-2445.gd2c45d97.el9.x86_64 144/156 2024-04-25T00:21:31.881 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : postgresql-contrib-13.14-1.el9.x86_64 145/156 2024-04-25T00:21:31.910 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libcephfs-devel-2:19.0.0-2445.gd2c45d97.el9.x86_ 146/156 2024-04-25T00:21:32.432 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : cephfs-top-2:19.0.0-2445.gd2c45d97.el9.noarch 148/156 2024-04-25T00:21:32.515 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: postgresql-server-13.14-1.el9.x86_64 147/156 2024-04-25T00:21:32.518 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : openssl-devel-1:3.2.1-1.el9.x86_64 149/156 2024-04-25T00:21:32.558 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : postgresql-server-13.14-1.el9.x86_64 147/156 2024-04-25T00:21:32.559 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : libaio-devel-0.3.111-13.el9.x86_64 150/156 2024-04-25T00:21:32.579 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: postgresql-server-13.14-1.el9.x86_64 147/156 2024-04-25T00:21:32.640 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : flex-2.6.4-9.el9.x86_64 151/156 2024-04-25T00:21:32.672 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : bison-3.7.4-5.el9.x86_64 152/156 2024-04-25T00:21:32.672 INFO:teuthology.orchestra.run.smithi121.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 153/156 2024-04-25T00:21:32.756 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 153/156 2024-04-25T00:21:32.789 INFO:teuthology.orchestra.run.smithi121.stdout: Cleanup : openssl-1:3.0.7-27.el9.x86_64 154/156 2024-04-25T00:21:32.789 INFO:teuthology.orchestra.run.smithi121.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 155/156 2024-04-25T00:21:32.878 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 155/156 2024-04-25T00:21:32.946 INFO:teuthology.orchestra.run.smithi121.stdout: Cleanup : openssl-libs-1:3.0.7-27.el9.x86_64 156/156 2024-04-25T00:21:33.488 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : cephfs-top-2:19.0.0-2445.gd2c45d97.el9.noarch 148/156 2024-04-25T00:21:33.572 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : openssl-devel-1:3.2.1-1.el9.x86_64 149/156 2024-04-25T00:21:33.614 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libaio-devel-0.3.111-13.el9.x86_64 150/156 2024-04-25T00:21:33.701 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : flex-2.6.4-9.el9.x86_64 151/156 2024-04-25T00:21:33.743 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : bison-3.7.4-5.el9.x86_64 152/156 2024-04-25T00:21:33.743 INFO:teuthology.orchestra.run.smithi032.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 153/156 2024-04-25T00:21:33.836 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 153/156 2024-04-25T00:21:33.868 INFO:teuthology.orchestra.run.smithi032.stdout: Cleanup : openssl-1:3.0.7-27.el9.x86_64 154/156 2024-04-25T00:21:33.869 INFO:teuthology.orchestra.run.smithi032.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 155/156 2024-04-25T00:21:33.958 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 155/156 2024-04-25T00:21:34.029 INFO:teuthology.orchestra.run.smithi032.stdout: Cleanup : openssl-libs-1:3.0.7-27.el9.x86_64 156/156 2024-04-25T00:21:38.722 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: openssl-libs-1:3.0.7-27.el9.x86_64 156/156 2024-04-25T00:21:38.722 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-2:19.0.0-2445.gd2c45d97.el9.x86_64 1/156 2024-04-25T00:21:38.722 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-base-2:19.0.0-2445.gd2c45d97.el9.x86_64 2/156 2024-04-25T00:21:38.722 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-common-2:19.0.0-2445.gd2c45d97.el9.x86_64 3/156 2024-04-25T00:21:38.722 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-fuse-2:19.0.0-2445.gd2c45d97.el9.x86_64 4/156 2024-04-25T00:21:38.722 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-immutable-object-cache-2:19.0.0-2445.gd2c45 5/156 2024-04-25T00:21:38.722 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-mds-2:19.0.0-2445.gd2c45d97.el9.x86_64 6/156 2024-04-25T00:21:38.722 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-mgr-2:19.0.0-2445.gd2c45d97.el9.x86_64 7/156 2024-04-25T00:21:38.722 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-mon-2:19.0.0-2445.gd2c45d97.el9.x86_64 8/156 2024-04-25T00:21:38.723 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-osd-2:19.0.0-2445.gd2c45d97.el9.x86_64 9/156 2024-04-25T00:21:38.723 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-radosgw-2:19.0.0-2445.gd2c45d97.el9.x86_64 10/156 2024-04-25T00:21:38.723 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-selinux-2:19.0.0-2445.gd2c45d97.el9.x86_64 11/156 2024-04-25T00:21:38.723 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-test-2:19.0.0-2445.gd2c45d97.el9.x86_64 12/156 2024-04-25T00:21:38.723 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : cephfs-mirror-2:19.0.0-2445.gd2c45d97.el9.x86_64 13/156 2024-04-25T00:21:38.723 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : cephfs-shell-2:19.0.0-2445.gd2c45d97.el9.x86_64 14/156 2024-04-25T00:21:38.723 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : libcephfs-devel-2:19.0.0-2445.gd2c45d97.el9.x86_ 15/156 2024-04-25T00:21:38.723 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : libcephfs2-2:19.0.0-2445.gd2c45d97.el9.x86_64 16/156 2024-04-25T00:21:38.723 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : libcephsqlite-2:19.0.0-2445.gd2c45d97.el9.x86_64 17/156 2024-04-25T00:21:38.723 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : librados-devel-2:19.0.0-2445.gd2c45d97.el9.x86_6 18/156 2024-04-25T00:21:38.723 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : libradosstriper1-2:19.0.0-2445.gd2c45d97.el9.x86 19/156 2024-04-25T00:21:38.723 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : librgw2-2:19.0.0-2445.gd2c45d97.el9.x86_64 20/156 2024-04-25T00:21:38.723 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-ceph-argparse-2:19.0.0-2445.gd2c45d97.el 21/156 2024-04-25T00:21:38.724 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-ceph-common-2:19.0.0-2445.gd2c45d97.el9. 22/156 2024-04-25T00:21:38.724 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-cephfs-2:19.0.0-2445.gd2c45d97.el9.x86_6 23/156 2024-04-25T00:21:38.724 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-rados-2:19.0.0-2445.gd2c45d97.el9.x86_64 24/156 2024-04-25T00:21:38.724 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-rbd-2:19.0.0-2445.gd2c45d97.el9.x86_64 25/156 2024-04-25T00:21:38.724 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-rgw-2:19.0.0-2445.gd2c45d97.el9.x86_64 26/156 2024-04-25T00:21:38.724 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : rbd-fuse-2:19.0.0-2445.gd2c45d97.el9.x86_64 27/156 2024-04-25T00:21:38.724 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : rbd-mirror-2:19.0.0-2445.gd2c45d97.el9.x86_64 28/156 2024-04-25T00:21:38.724 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : rbd-nbd-2:19.0.0-2445.gd2c45d97.el9.x86_64 29/156 2024-04-25T00:21:38.739 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-grafana-dashboards-2:19.0.0-2445.gd2c45d97. 30/156 2024-04-25T00:21:38.739 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-mgr-cephadm-2:19.0.0-2445.gd2c45d97.el9.noa 31/156 2024-04-25T00:21:38.740 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-mgr-dashboard-2:19.0.0-2445.gd2c45d97.el9.n 32/156 2024-04-25T00:21:38.740 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.0.0-2445.gd2c 33/156 2024-04-25T00:21:38.740 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-mgr-modules-core-2:19.0.0-2445.gd2c45d97.el 34/156 2024-04-25T00:21:38.740 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-mgr-rook-2:19.0.0-2445.gd2c45d97.el9.noarch 35/156 2024-04-25T00:21:38.740 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-prometheus-alerts-2:19.0.0-2445.gd2c45d97.e 36/156 2024-04-25T00:21:38.740 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-volume-2:19.0.0-2445.gd2c45d97.el9.noarch 37/156 2024-04-25T00:21:38.740 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : cephadm-2:19.0.0-2445.gd2c45d97.el9.noarch 38/156 2024-04-25T00:21:38.740 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : cephfs-top-2:19.0.0-2445.gd2c45d97.el9.noarch 39/156 2024-04-25T00:21:38.740 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 40/156 2024-04-25T00:21:38.740 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 41/156 2024-04-25T00:21:38.740 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 42/156 2024-04-25T00:21:38.740 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 43/156 2024-04-25T00:21:38.740 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 44/156 2024-04-25T00:21:38.741 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-ply-3.11-14.el9.noarch 45/156 2024-04-25T00:21:38.741 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 46/156 2024-04-25T00:21:38.741 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 47/156 2024-04-25T00:21:38.741 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 48/156 2024-04-25T00:21:38.741 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : automake-1.16.2-8.el9.noarch 49/156 2024-04-25T00:21:38.741 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : bison-3.7.4-5.el9.x86_64 50/156 2024-04-25T00:21:38.741 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 51/156 2024-04-25T00:21:38.741 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : elfutils-libelf-devel-0.190-2.el9.x86_64 52/156 2024-04-25T00:21:38.741 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : flex-2.6.4-9.el9.x86_64 53/156 2024-04-25T00:21:38.741 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 54/156 2024-04-25T00:21:38.741 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 55/156 2024-04-25T00:21:38.741 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 56/156 2024-04-25T00:21:38.741 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : libaio-devel-0.3.111-13.el9.x86_64 57/156 2024-04-25T00:21:38.741 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 58/156 2024-04-25T00:21:38.742 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 59/156 2024-04-25T00:21:38.742 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 60/156 2024-04-25T00:21:38.742 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : libtool-2.4.6-45.el9.x86_64 61/156 2024-04-25T00:21:38.742 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : libuuid-devel-2.37.4-18.el9.x86_64 62/156 2024-04-25T00:21:38.742 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : libzstd-devel-1.5.1-2.el9.x86_64 63/156 2024-04-25T00:21:38.742 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 64/156 2024-04-25T00:21:38.742 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : lua-5.4.4-4.el9.x86_64 65/156 2024-04-25T00:21:38.742 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 66/156 2024-04-25T00:21:38.742 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 67/156 2024-04-25T00:21:38.742 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : openssl-devel-1:3.2.1-1.el9.x86_64 68/156 2024-04-25T00:21:38.742 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : postgresql-13.14-1.el9.x86_64 69/156 2024-04-25T00:21:38.742 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : postgresql-contrib-13.14-1.el9.x86_64 70/156 2024-04-25T00:21:38.742 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : postgresql-private-libs-13.14-1.el9.x86_64 71/156 2024-04-25T00:21:38.742 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : postgresql-server-13.14-1.el9.x86_64 72/156 2024-04-25T00:21:38.743 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : protobuf-3.14.0-13.el9.x86_64 73/156 2024-04-25T00:21:38.743 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 74/156 2024-04-25T00:21:38.743 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 75/156 2024-04-25T00:21:38.743 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 76/156 2024-04-25T00:21:38.743 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 77/156 2024-04-25T00:21:38.743 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 78/156 2024-04-25T00:21:38.743 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 79/156 2024-04-25T00:21:38.743 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 80/156 2024-04-25T00:21:38.743 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 81/156 2024-04-25T00:21:38.743 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 82/156 2024-04-25T00:21:38.743 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 83/156 2024-04-25T00:21:38.743 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 84/156 2024-04-25T00:21:38.743 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 85/156 2024-04-25T00:21:38.743 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 86/156 2024-04-25T00:21:38.744 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-protobuf-3.14.0-13.el9.noarch 87/156 2024-04-25T00:21:38.744 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 88/156 2024-04-25T00:21:38.744 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 89/156 2024-04-25T00:21:38.744 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 90/156 2024-04-25T00:21:38.744 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 91/156 2024-04-25T00:21:38.744 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 92/156 2024-04-25T00:21:38.744 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 93/156 2024-04-25T00:21:38.744 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-wcwidth-0.2.5-8.el9.noarch 94/156 2024-04-25T00:21:38.744 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : qatlib-24.02.0-1.el9.x86_64 95/156 2024-04-25T00:21:38.744 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : qatlib-service-24.02.0-1.el9.x86_64 96/156 2024-04-25T00:21:38.744 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : qatzip-libs-1.2.0-1.el9.x86_64 97/156 2024-04-25T00:21:38.744 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 98/156 2024-04-25T00:21:38.744 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : uuid-1.6.2-55.el9.x86_64 99/156 2024-04-25T00:21:38.744 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : xfsprogs-devel-6.3.0-1.el9.x86_64 100/156 2024-04-25T00:21:38.745 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 101/156 2024-04-25T00:21:38.745 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : zlib-devel-1.2.11-41.el9.x86_64 102/156 2024-04-25T00:21:38.745 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 103/156 2024-04-25T00:21:38.745 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : protobuf-compiler-3.14.0-13.el9.x86_64 104/156 2024-04-25T00:21:38.745 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : abseil-cpp-20211102.0-3.el9.x86_64 105/156 2024-04-25T00:21:38.745 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 106/156 2024-04-25T00:21:38.745 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 107/156 2024-04-25T00:21:38.745 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : libarrow-9.0.0-10.el9.x86_64 108/156 2024-04-25T00:21:38.745 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : libarrow-doc-9.0.0-10.el9.noarch 109/156 2024-04-25T00:21:38.745 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 110/156 2024-04-25T00:21:38.745 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 111/156 2024-04-25T00:21:38.745 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : luarocks-3.9.2-1.el9.noarch 112/156 2024-04-25T00:21:38.745 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : parquet-libs-9.0.0-10.el9.x86_64 113/156 2024-04-25T00:21:38.745 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 114/156 2024-04-25T00:21:38.746 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 115/156 2024-04-25T00:21:38.746 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 116/156 2024-04-25T00:21:38.746 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 117/156 2024-04-25T00:21:38.746 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 118/156 2024-04-25T00:21:38.746 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 119/156 2024-04-25T00:21:38.746 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-cmd2-2.4.2-2.el9.noarch 120/156 2024-04-25T00:21:38.746 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-colorama-0.4.4-12.el9.noarch 121/156 2024-04-25T00:21:38.746 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-google-auth-1:2.29.0-1.el9.noarch 122/156 2024-04-25T00:21:38.746 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 123/156 2024-04-25T00:21:38.746 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 124/156 2024-04-25T00:21:38.746 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 125/156 2024-04-25T00:21:38.746 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 126/156 2024-04-25T00:21:38.746 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 127/156 2024-04-25T00:21:38.746 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 128/156 2024-04-25T00:21:38.747 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 129/156 2024-04-25T00:21:38.747 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 130/156 2024-04-25T00:21:38.747 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 131/156 2024-04-25T00:21:38.747 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 132/156 2024-04-25T00:21:38.747 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 133/156 2024-04-25T00:21:38.747 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 134/156 2024-04-25T00:21:38.747 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 135/156 2024-04-25T00:21:38.747 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 136/156 2024-04-25T00:21:38.747 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-pyperclip-1.8.2-2.el9.noarch 137/156 2024-04-25T00:21:38.747 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 138/156 2024-04-25T00:21:38.747 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 139/156 2024-04-25T00:21:38.747 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 140/156 2024-04-25T00:21:38.747 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 141/156 2024-04-25T00:21:38.747 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 142/156 2024-04-25T00:21:38.748 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 143/156 2024-04-25T00:21:38.748 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 144/156 2024-04-25T00:21:38.748 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 145/156 2024-04-25T00:21:38.748 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 146/156 2024-04-25T00:21:38.748 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : re2-1:20211101-3.el9.x86_64 147/156 2024-04-25T00:21:38.748 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 148/156 2024-04-25T00:21:38.748 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : librados2-2:19.0.0-2445.gd2c45d97.el9.x86_64 149/156 2024-04-25T00:21:38.748 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 150/156 2024-04-25T00:21:38.748 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : librbd1-2:19.0.0-2445.gd2c45d97.el9.x86_64 151/156 2024-04-25T00:21:38.748 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 152/156 2024-04-25T00:21:38.748 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : openssl-1:3.2.1-1.el9.x86_64 153/156 2024-04-25T00:21:38.748 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : openssl-1:3.0.7-27.el9.x86_64 154/156 2024-04-25T00:21:38.748 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : openssl-libs-1:3.2.1-1.el9.x86_64 155/156 2024-04-25T00:21:39.883 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: openssl-libs-1:3.0.7-27.el9.x86_64 156/156 2024-04-25T00:21:39.883 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-2:19.0.0-2445.gd2c45d97.el9.x86_64 1/156 2024-04-25T00:21:39.883 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-base-2:19.0.0-2445.gd2c45d97.el9.x86_64 2/156 2024-04-25T00:21:39.883 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-common-2:19.0.0-2445.gd2c45d97.el9.x86_64 3/156 2024-04-25T00:21:39.883 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-fuse-2:19.0.0-2445.gd2c45d97.el9.x86_64 4/156 2024-04-25T00:21:39.883 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-immutable-object-cache-2:19.0.0-2445.gd2c45 5/156 2024-04-25T00:21:39.883 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-mds-2:19.0.0-2445.gd2c45d97.el9.x86_64 6/156 2024-04-25T00:21:39.883 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-mgr-2:19.0.0-2445.gd2c45d97.el9.x86_64 7/156 2024-04-25T00:21:39.883 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-mon-2:19.0.0-2445.gd2c45d97.el9.x86_64 8/156 2024-04-25T00:21:39.883 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-osd-2:19.0.0-2445.gd2c45d97.el9.x86_64 9/156 2024-04-25T00:21:39.883 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-radosgw-2:19.0.0-2445.gd2c45d97.el9.x86_64 10/156 2024-04-25T00:21:39.883 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-selinux-2:19.0.0-2445.gd2c45d97.el9.x86_64 11/156 2024-04-25T00:21:39.883 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-test-2:19.0.0-2445.gd2c45d97.el9.x86_64 12/156 2024-04-25T00:21:39.884 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : cephfs-mirror-2:19.0.0-2445.gd2c45d97.el9.x86_64 13/156 2024-04-25T00:21:39.884 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : cephfs-shell-2:19.0.0-2445.gd2c45d97.el9.x86_64 14/156 2024-04-25T00:21:39.884 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libcephfs-devel-2:19.0.0-2445.gd2c45d97.el9.x86_ 15/156 2024-04-25T00:21:39.884 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libcephfs2-2:19.0.0-2445.gd2c45d97.el9.x86_64 16/156 2024-04-25T00:21:39.884 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libcephsqlite-2:19.0.0-2445.gd2c45d97.el9.x86_64 17/156 2024-04-25T00:21:39.884 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : librados-devel-2:19.0.0-2445.gd2c45d97.el9.x86_6 18/156 2024-04-25T00:21:39.884 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libradosstriper1-2:19.0.0-2445.gd2c45d97.el9.x86 19/156 2024-04-25T00:21:39.884 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : librgw2-2:19.0.0-2445.gd2c45d97.el9.x86_64 20/156 2024-04-25T00:21:39.884 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-ceph-argparse-2:19.0.0-2445.gd2c45d97.el 21/156 2024-04-25T00:21:39.884 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-ceph-common-2:19.0.0-2445.gd2c45d97.el9. 22/156 2024-04-25T00:21:39.884 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-cephfs-2:19.0.0-2445.gd2c45d97.el9.x86_6 23/156 2024-04-25T00:21:39.884 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-rados-2:19.0.0-2445.gd2c45d97.el9.x86_64 24/156 2024-04-25T00:21:39.885 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-rbd-2:19.0.0-2445.gd2c45d97.el9.x86_64 25/156 2024-04-25T00:21:39.885 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-rgw-2:19.0.0-2445.gd2c45d97.el9.x86_64 26/156 2024-04-25T00:21:39.885 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : rbd-fuse-2:19.0.0-2445.gd2c45d97.el9.x86_64 27/156 2024-04-25T00:21:39.885 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : rbd-mirror-2:19.0.0-2445.gd2c45d97.el9.x86_64 28/156 2024-04-25T00:21:39.885 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : rbd-nbd-2:19.0.0-2445.gd2c45d97.el9.x86_64 29/156 2024-04-25T00:21:39.885 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-grafana-dashboards-2:19.0.0-2445.gd2c45d97. 30/156 2024-04-25T00:21:39.885 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-mgr-cephadm-2:19.0.0-2445.gd2c45d97.el9.noa 31/156 2024-04-25T00:21:39.885 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-mgr-dashboard-2:19.0.0-2445.gd2c45d97.el9.n 32/156 2024-04-25T00:21:39.885 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.0.0-2445.gd2c 33/156 2024-04-25T00:21:39.885 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-mgr-modules-core-2:19.0.0-2445.gd2c45d97.el 34/156 2024-04-25T00:21:39.885 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-mgr-rook-2:19.0.0-2445.gd2c45d97.el9.noarch 35/156 2024-04-25T00:21:39.885 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-prometheus-alerts-2:19.0.0-2445.gd2c45d97.e 36/156 2024-04-25T00:21:39.885 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-volume-2:19.0.0-2445.gd2c45d97.el9.noarch 37/156 2024-04-25T00:21:39.886 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : cephadm-2:19.0.0-2445.gd2c45d97.el9.noarch 38/156 2024-04-25T00:21:39.886 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : cephfs-top-2:19.0.0-2445.gd2c45d97.el9.noarch 39/156 2024-04-25T00:21:39.886 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 40/156 2024-04-25T00:21:39.886 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 41/156 2024-04-25T00:21:39.886 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 42/156 2024-04-25T00:21:39.886 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 43/156 2024-04-25T00:21:39.886 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 44/156 2024-04-25T00:21:39.886 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-ply-3.11-14.el9.noarch 45/156 2024-04-25T00:21:39.886 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 46/156 2024-04-25T00:21:39.886 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 47/156 2024-04-25T00:21:39.886 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 48/156 2024-04-25T00:21:39.886 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : automake-1.16.2-8.el9.noarch 49/156 2024-04-25T00:21:39.886 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : bison-3.7.4-5.el9.x86_64 50/156 2024-04-25T00:21:39.886 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 51/156 2024-04-25T00:21:39.887 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : elfutils-libelf-devel-0.190-2.el9.x86_64 52/156 2024-04-25T00:21:39.887 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : flex-2.6.4-9.el9.x86_64 53/156 2024-04-25T00:21:39.887 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 54/156 2024-04-25T00:21:39.887 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 55/156 2024-04-25T00:21:39.887 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 56/156 2024-04-25T00:21:39.887 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libaio-devel-0.3.111-13.el9.x86_64 57/156 2024-04-25T00:21:39.887 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 58/156 2024-04-25T00:21:39.887 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 59/156 2024-04-25T00:21:39.887 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 60/156 2024-04-25T00:21:39.887 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libtool-2.4.6-45.el9.x86_64 61/156 2024-04-25T00:21:39.887 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libuuid-devel-2.37.4-18.el9.x86_64 62/156 2024-04-25T00:21:39.887 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libzstd-devel-1.5.1-2.el9.x86_64 63/156 2024-04-25T00:21:39.887 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 64/156 2024-04-25T00:21:39.887 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : lua-5.4.4-4.el9.x86_64 65/156 2024-04-25T00:21:39.887 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 66/156 2024-04-25T00:21:39.888 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 67/156 2024-04-25T00:21:39.888 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : openssl-devel-1:3.2.1-1.el9.x86_64 68/156 2024-04-25T00:21:39.888 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : postgresql-13.14-1.el9.x86_64 69/156 2024-04-25T00:21:39.888 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : postgresql-contrib-13.14-1.el9.x86_64 70/156 2024-04-25T00:21:39.888 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : postgresql-private-libs-13.14-1.el9.x86_64 71/156 2024-04-25T00:21:39.888 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : postgresql-server-13.14-1.el9.x86_64 72/156 2024-04-25T00:21:39.888 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : protobuf-3.14.0-13.el9.x86_64 73/156 2024-04-25T00:21:39.888 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 74/156 2024-04-25T00:21:39.888 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 75/156 2024-04-25T00:21:39.888 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 76/156 2024-04-25T00:21:39.888 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 77/156 2024-04-25T00:21:39.888 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 78/156 2024-04-25T00:21:39.888 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 79/156 2024-04-25T00:21:39.889 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 80/156 2024-04-25T00:21:39.889 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 81/156 2024-04-25T00:21:39.889 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 82/156 2024-04-25T00:21:39.889 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 83/156 2024-04-25T00:21:39.889 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 84/156 2024-04-25T00:21:39.889 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 85/156 2024-04-25T00:21:39.889 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 86/156 2024-04-25T00:21:39.889 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-protobuf-3.14.0-13.el9.noarch 87/156 2024-04-25T00:21:39.889 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 88/156 2024-04-25T00:21:39.889 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 89/156 2024-04-25T00:21:39.889 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 90/156 2024-04-25T00:21:39.889 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 91/156 2024-04-25T00:21:39.889 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 92/156 2024-04-25T00:21:39.889 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 93/156 2024-04-25T00:21:39.889 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-wcwidth-0.2.5-8.el9.noarch 94/156 2024-04-25T00:21:39.890 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : qatlib-24.02.0-1.el9.x86_64 95/156 2024-04-25T00:21:39.890 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : qatlib-service-24.02.0-1.el9.x86_64 96/156 2024-04-25T00:21:39.890 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : qatzip-libs-1.2.0-1.el9.x86_64 97/156 2024-04-25T00:21:39.890 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 98/156 2024-04-25T00:21:39.890 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : uuid-1.6.2-55.el9.x86_64 99/156 2024-04-25T00:21:39.890 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : xfsprogs-devel-6.3.0-1.el9.x86_64 100/156 2024-04-25T00:21:39.890 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 101/156 2024-04-25T00:21:39.890 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : zlib-devel-1.2.11-41.el9.x86_64 102/156 2024-04-25T00:21:39.890 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 103/156 2024-04-25T00:21:39.890 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : protobuf-compiler-3.14.0-13.el9.x86_64 104/156 2024-04-25T00:21:39.890 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : abseil-cpp-20211102.0-3.el9.x86_64 105/156 2024-04-25T00:21:39.890 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 106/156 2024-04-25T00:21:39.890 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 107/156 2024-04-25T00:21:39.890 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libarrow-9.0.0-10.el9.x86_64 108/156 2024-04-25T00:21:39.891 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libarrow-doc-9.0.0-10.el9.noarch 109/156 2024-04-25T00:21:39.891 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 110/156 2024-04-25T00:21:39.891 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 111/156 2024-04-25T00:21:39.891 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : luarocks-3.9.2-1.el9.noarch 112/156 2024-04-25T00:21:39.891 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : parquet-libs-9.0.0-10.el9.x86_64 113/156 2024-04-25T00:21:39.891 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 114/156 2024-04-25T00:21:39.891 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 115/156 2024-04-25T00:21:39.891 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 116/156 2024-04-25T00:21:39.891 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 117/156 2024-04-25T00:21:39.891 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 118/156 2024-04-25T00:21:39.891 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 119/156 2024-04-25T00:21:39.891 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-cmd2-2.4.2-2.el9.noarch 120/156 2024-04-25T00:21:39.891 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-colorama-0.4.4-12.el9.noarch 121/156 2024-04-25T00:21:39.891 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-google-auth-1:2.29.0-1.el9.noarch 122/156 2024-04-25T00:21:39.892 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 123/156 2024-04-25T00:21:39.892 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 124/156 2024-04-25T00:21:39.892 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 125/156 2024-04-25T00:21:39.892 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 126/156 2024-04-25T00:21:39.892 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 127/156 2024-04-25T00:21:39.892 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 128/156 2024-04-25T00:21:39.892 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 129/156 2024-04-25T00:21:39.892 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 130/156 2024-04-25T00:21:39.892 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 131/156 2024-04-25T00:21:39.892 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 132/156 2024-04-25T00:21:39.892 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 133/156 2024-04-25T00:21:39.892 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 134/156 2024-04-25T00:21:39.892 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 135/156 2024-04-25T00:21:39.892 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 136/156 2024-04-25T00:21:39.892 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-pyperclip-1.8.2-2.el9.noarch 137/156 2024-04-25T00:21:39.893 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 138/156 2024-04-25T00:21:39.893 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 139/156 2024-04-25T00:21:39.893 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 140/156 2024-04-25T00:21:39.893 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 141/156 2024-04-25T00:21:39.893 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 142/156 2024-04-25T00:21:39.893 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 143/156 2024-04-25T00:21:39.893 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 144/156 2024-04-25T00:21:39.893 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 145/156 2024-04-25T00:21:39.893 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 146/156 2024-04-25T00:21:39.893 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : re2-1:20211101-3.el9.x86_64 147/156 2024-04-25T00:21:39.893 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 148/156 2024-04-25T00:21:39.893 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : librados2-2:19.0.0-2445.gd2c45d97.el9.x86_64 149/156 2024-04-25T00:21:39.893 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 150/156 2024-04-25T00:21:39.893 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : librbd1-2:19.0.0-2445.gd2c45d97.el9.x86_64 151/156 2024-04-25T00:21:39.893 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 152/156 2024-04-25T00:21:39.894 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : openssl-1:3.2.1-1.el9.x86_64 153/156 2024-04-25T00:21:39.894 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : openssl-1:3.0.7-27.el9.x86_64 154/156 2024-04-25T00:21:39.894 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : openssl-libs-1:3.2.1-1.el9.x86_64 155/156 2024-04-25T00:21:40.844 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : openssl-libs-1:3.0.7-27.el9.x86_64 156/156 2024-04-25T00:21:40.844 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-25T00:21:40.845 INFO:teuthology.orchestra.run.smithi121.stdout:Upgraded: 2024-04-25T00:21:40.845 INFO:teuthology.orchestra.run.smithi121.stdout: librados2-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.845 INFO:teuthology.orchestra.run.smithi121.stdout: librbd1-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.845 INFO:teuthology.orchestra.run.smithi121.stdout: openssl-1:3.2.1-1.el9.x86_64 2024-04-25T00:21:40.845 INFO:teuthology.orchestra.run.smithi121.stdout: openssl-libs-1:3.2.1-1.el9.x86_64 2024-04-25T00:21:40.845 INFO:teuthology.orchestra.run.smithi121.stdout:Installed: 2024-04-25T00:21:40.845 INFO:teuthology.orchestra.run.smithi121.stdout: abseil-cpp-20211102.0-3.el9.x86_64 2024-04-25T00:21:40.845 INFO:teuthology.orchestra.run.smithi121.stdout: automake-1.16.2-8.el9.noarch 2024-04-25T00:21:40.845 INFO:teuthology.orchestra.run.smithi121.stdout: bison-3.7.4-5.el9.x86_64 2024-04-25T00:21:40.845 INFO:teuthology.orchestra.run.smithi121.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-04-25T00:21:40.845 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.845 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-base-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.845 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-common-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-fuse-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-grafana-dashboards-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-immutable-object-cache-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mds-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mgr-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mgr-cephadm-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mgr-dashboard-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mgr-diskprediction-local-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mgr-modules-core-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mgr-rook-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mon-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-osd-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-prometheus-alerts-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-radosgw-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-selinux-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-test-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-volume-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: cephadm-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: cephfs-mirror-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: cephfs-shell-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: cephfs-top-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: elfutils-libelf-devel-0.190-2.el9.x86_64 2024-04-25T00:21:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: flex-2.6.4-9.el9.x86_64 2024-04-25T00:21:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-25T00:21:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-25T00:21:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-25T00:21:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: fuse-2.9.9-15.el9.x86_64 2024-04-25T00:21:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-25T00:21:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: grpc-data-1.46.7-10.el9.noarch 2024-04-25T00:21:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: libaio-devel-0.3.111-13.el9.x86_64 2024-04-25T00:21:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: libarrow-9.0.0-10.el9.x86_64 2024-04-25T00:21:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: libarrow-doc-9.0.0-10.el9.noarch 2024-04-25T00:21:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: libcephfs-devel-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: libcephfs2-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: libcephsqlite-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-25T00:21:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-25T00:21:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-25T00:21:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-25T00:21:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-25T00:21:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: librados-devel-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: libradosstriper1-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-25T00:21:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: librgw2-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: libtool-2.4.6-45.el9.x86_64 2024-04-25T00:21:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-25T00:21:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: libuuid-devel-2.37.4-18.el9.x86_64 2024-04-25T00:21:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: libzstd-devel-1.5.1-2.el9.x86_64 2024-04-25T00:21:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-25T00:21:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: lua-5.4.4-4.el9.x86_64 2024-04-25T00:21:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: lua-devel-5.4.4-4.el9.x86_64 2024-04-25T00:21:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: luarocks-3.9.2-1.el9.noarch 2024-04-25T00:21:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-25T00:21:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-25T00:21:40.850 INFO:teuthology.orchestra.run.smithi121.stdout: openssl-devel-1:3.2.1-1.el9.x86_64 2024-04-25T00:21:40.850 INFO:teuthology.orchestra.run.smithi121.stdout: parquet-libs-9.0.0-10.el9.x86_64 2024-04-25T00:21:40.850 INFO:teuthology.orchestra.run.smithi121.stdout: postgresql-13.14-1.el9.x86_64 2024-04-25T00:21:40.850 INFO:teuthology.orchestra.run.smithi121.stdout: postgresql-contrib-13.14-1.el9.x86_64 2024-04-25T00:21:40.850 INFO:teuthology.orchestra.run.smithi121.stdout: postgresql-private-libs-13.14-1.el9.x86_64 2024-04-25T00:21:40.850 INFO:teuthology.orchestra.run.smithi121.stdout: postgresql-server-13.14-1.el9.x86_64 2024-04-25T00:21:40.850 INFO:teuthology.orchestra.run.smithi121.stdout: protobuf-3.14.0-13.el9.x86_64 2024-04-25T00:21:40.850 INFO:teuthology.orchestra.run.smithi121.stdout: protobuf-compiler-3.14.0-13.el9.x86_64 2024-04-25T00:21:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-25T00:21:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-04-25T00:21:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-25T00:21:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-25T00:21:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-25T00:21:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: python3-ceph-argparse-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: python3-ceph-common-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cephfs-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-25T00:21:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-25T00:21:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-25T00:21:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-25T00:21:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cmd2-2.4.2-2.el9.noarch 2024-04-25T00:21:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: python3-colorama-0.4.4-12.el9.noarch 2024-04-25T00:21:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-25T00:21:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-04-25T00:21:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: python3-google-auth-1:2.29.0-1.el9.noarch 2024-04-25T00:21:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2024-04-25T00:21:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2024-04-25T00:21:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-25T00:21:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-25T00:21:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-25T00:21:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-25T00:21:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-25T00:21:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-25T00:21:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-25T00:21:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-25T00:21:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-25T00:21:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-25T00:21:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-25T00:21:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-25T00:21:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-25T00:21:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-25T00:21:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-25T00:21:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-25T00:21:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-25T00:21:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: python3-packaging-20.9-5.el9.noarch 2024-04-25T00:21:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-25T00:21:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: python3-ply-3.11-14.el9.noarch 2024-04-25T00:21:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-25T00:21:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-25T00:21:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: python3-protobuf-3.14.0-13.el9.noarch 2024-04-25T00:21:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-25T00:21:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-25T00:21:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-25T00:21:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-25T00:21:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pyperclip-1.8.2-2.el9.noarch 2024-04-25T00:21:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-25T00:21:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: python3-rados-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: python3-rbd-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-25T00:21:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-25T00:21:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-25T00:21:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: python3-rgw-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-25T00:21:40.855 INFO:teuthology.orchestra.run.smithi121.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-25T00:21:40.855 INFO:teuthology.orchestra.run.smithi121.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-25T00:21:40.855 INFO:teuthology.orchestra.run.smithi121.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-25T00:21:40.855 INFO:teuthology.orchestra.run.smithi121.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-25T00:21:40.855 INFO:teuthology.orchestra.run.smithi121.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-25T00:21:40.855 INFO:teuthology.orchestra.run.smithi121.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-25T00:21:40.855 INFO:teuthology.orchestra.run.smithi121.stdout: python3-wcwidth-0.2.5-8.el9.noarch 2024-04-25T00:21:40.855 INFO:teuthology.orchestra.run.smithi121.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-25T00:21:40.855 INFO:teuthology.orchestra.run.smithi121.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-25T00:21:40.855 INFO:teuthology.orchestra.run.smithi121.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-25T00:21:40.855 INFO:teuthology.orchestra.run.smithi121.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-25T00:21:40.856 INFO:teuthology.orchestra.run.smithi121.stdout: qatlib-24.02.0-1.el9.x86_64 2024-04-25T00:21:40.856 INFO:teuthology.orchestra.run.smithi121.stdout: qatlib-service-24.02.0-1.el9.x86_64 2024-04-25T00:21:40.856 INFO:teuthology.orchestra.run.smithi121.stdout: qatzip-libs-1.2.0-1.el9.x86_64 2024-04-25T00:21:40.856 INFO:teuthology.orchestra.run.smithi121.stdout: rbd-fuse-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.856 INFO:teuthology.orchestra.run.smithi121.stdout: rbd-mirror-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.856 INFO:teuthology.orchestra.run.smithi121.stdout: rbd-nbd-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:40.856 INFO:teuthology.orchestra.run.smithi121.stdout: re2-1:20211101-3.el9.x86_64 2024-04-25T00:21:40.856 INFO:teuthology.orchestra.run.smithi121.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-25T00:21:40.856 INFO:teuthology.orchestra.run.smithi121.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-25T00:21:40.856 INFO:teuthology.orchestra.run.smithi121.stdout: uuid-1.6.2-55.el9.x86_64 2024-04-25T00:21:40.857 INFO:teuthology.orchestra.run.smithi121.stdout: xfsprogs-devel-6.3.0-1.el9.x86_64 2024-04-25T00:21:40.857 INFO:teuthology.orchestra.run.smithi121.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-25T00:21:40.857 INFO:teuthology.orchestra.run.smithi121.stdout: zlib-devel-1.2.11-41.el9.x86_64 2024-04-25T00:21:40.857 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-25T00:21:40.857 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2024-04-25T00:21:41.239 DEBUG:teuthology.parallel:result is None 2024-04-25T00:21:41.951 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : openssl-libs-1:3.0.7-27.el9.x86_64 156/156 2024-04-25T00:21:41.951 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:21:41.951 INFO:teuthology.orchestra.run.smithi032.stdout:Upgraded: 2024-04-25T00:21:41.951 INFO:teuthology.orchestra.run.smithi032.stdout: librados2-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.951 INFO:teuthology.orchestra.run.smithi032.stdout: librbd1-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.951 INFO:teuthology.orchestra.run.smithi032.stdout: openssl-1:3.2.1-1.el9.x86_64 2024-04-25T00:21:41.951 INFO:teuthology.orchestra.run.smithi032.stdout: openssl-libs-1:3.2.1-1.el9.x86_64 2024-04-25T00:21:41.951 INFO:teuthology.orchestra.run.smithi032.stdout:Installed: 2024-04-25T00:21:41.951 INFO:teuthology.orchestra.run.smithi032.stdout: abseil-cpp-20211102.0-3.el9.x86_64 2024-04-25T00:21:41.951 INFO:teuthology.orchestra.run.smithi032.stdout: automake-1.16.2-8.el9.noarch 2024-04-25T00:21:41.952 INFO:teuthology.orchestra.run.smithi032.stdout: bison-3.7.4-5.el9.x86_64 2024-04-25T00:21:41.952 INFO:teuthology.orchestra.run.smithi032.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-04-25T00:21:41.952 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.952 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-base-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.952 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-common-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.952 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-fuse-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.952 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-grafana-dashboards-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:41.952 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-immutable-object-cache-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.952 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mds-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.952 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mgr-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.952 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mgr-cephadm-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:41.952 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mgr-dashboard-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:41.952 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mgr-diskprediction-local-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:41.953 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mgr-modules-core-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:41.953 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mgr-rook-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:41.953 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mon-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.953 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-osd-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.953 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-prometheus-alerts-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:41.953 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-radosgw-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.953 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-selinux-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.953 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-test-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.953 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-volume-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:41.953 INFO:teuthology.orchestra.run.smithi032.stdout: cephadm-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:41.953 INFO:teuthology.orchestra.run.smithi032.stdout: cephfs-mirror-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.953 INFO:teuthology.orchestra.run.smithi032.stdout: cephfs-shell-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.953 INFO:teuthology.orchestra.run.smithi032.stdout: cephfs-top-2:19.0.0-2445.gd2c45d97.el9.noarch 2024-04-25T00:21:41.954 INFO:teuthology.orchestra.run.smithi032.stdout: elfutils-libelf-devel-0.190-2.el9.x86_64 2024-04-25T00:21:41.954 INFO:teuthology.orchestra.run.smithi032.stdout: flex-2.6.4-9.el9.x86_64 2024-04-25T00:21:41.954 INFO:teuthology.orchestra.run.smithi032.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-25T00:21:41.954 INFO:teuthology.orchestra.run.smithi032.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-25T00:21:41.954 INFO:teuthology.orchestra.run.smithi032.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-25T00:21:41.954 INFO:teuthology.orchestra.run.smithi032.stdout: fuse-2.9.9-15.el9.x86_64 2024-04-25T00:21:41.954 INFO:teuthology.orchestra.run.smithi032.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-25T00:21:41.954 INFO:teuthology.orchestra.run.smithi032.stdout: grpc-data-1.46.7-10.el9.noarch 2024-04-25T00:21:41.954 INFO:teuthology.orchestra.run.smithi032.stdout: libaio-devel-0.3.111-13.el9.x86_64 2024-04-25T00:21:41.954 INFO:teuthology.orchestra.run.smithi032.stdout: libarrow-9.0.0-10.el9.x86_64 2024-04-25T00:21:41.954 INFO:teuthology.orchestra.run.smithi032.stdout: libarrow-doc-9.0.0-10.el9.noarch 2024-04-25T00:21:41.954 INFO:teuthology.orchestra.run.smithi032.stdout: libcephfs-devel-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.954 INFO:teuthology.orchestra.run.smithi032.stdout: libcephfs2-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.954 INFO:teuthology.orchestra.run.smithi032.stdout: libcephsqlite-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.955 INFO:teuthology.orchestra.run.smithi032.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-25T00:21:41.955 INFO:teuthology.orchestra.run.smithi032.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-25T00:21:41.955 INFO:teuthology.orchestra.run.smithi032.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-25T00:21:41.955 INFO:teuthology.orchestra.run.smithi032.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-25T00:21:41.955 INFO:teuthology.orchestra.run.smithi032.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-25T00:21:41.955 INFO:teuthology.orchestra.run.smithi032.stdout: librados-devel-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.955 INFO:teuthology.orchestra.run.smithi032.stdout: libradosstriper1-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.955 INFO:teuthology.orchestra.run.smithi032.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-25T00:21:41.955 INFO:teuthology.orchestra.run.smithi032.stdout: librgw2-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.955 INFO:teuthology.orchestra.run.smithi032.stdout: libtool-2.4.6-45.el9.x86_64 2024-04-25T00:21:41.955 INFO:teuthology.orchestra.run.smithi032.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-25T00:21:41.955 INFO:teuthology.orchestra.run.smithi032.stdout: libuuid-devel-2.37.4-18.el9.x86_64 2024-04-25T00:21:41.955 INFO:teuthology.orchestra.run.smithi032.stdout: libzstd-devel-1.5.1-2.el9.x86_64 2024-04-25T00:21:41.955 INFO:teuthology.orchestra.run.smithi032.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-25T00:21:41.956 INFO:teuthology.orchestra.run.smithi032.stdout: lua-5.4.4-4.el9.x86_64 2024-04-25T00:21:41.956 INFO:teuthology.orchestra.run.smithi032.stdout: lua-devel-5.4.4-4.el9.x86_64 2024-04-25T00:21:41.956 INFO:teuthology.orchestra.run.smithi032.stdout: luarocks-3.9.2-1.el9.noarch 2024-04-25T00:21:41.956 INFO:teuthology.orchestra.run.smithi032.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-25T00:21:41.956 INFO:teuthology.orchestra.run.smithi032.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-25T00:21:41.956 INFO:teuthology.orchestra.run.smithi032.stdout: openssl-devel-1:3.2.1-1.el9.x86_64 2024-04-25T00:21:41.956 INFO:teuthology.orchestra.run.smithi032.stdout: parquet-libs-9.0.0-10.el9.x86_64 2024-04-25T00:21:41.956 INFO:teuthology.orchestra.run.smithi032.stdout: postgresql-13.14-1.el9.x86_64 2024-04-25T00:21:41.956 INFO:teuthology.orchestra.run.smithi032.stdout: postgresql-contrib-13.14-1.el9.x86_64 2024-04-25T00:21:41.956 INFO:teuthology.orchestra.run.smithi032.stdout: postgresql-private-libs-13.14-1.el9.x86_64 2024-04-25T00:21:41.956 INFO:teuthology.orchestra.run.smithi032.stdout: postgresql-server-13.14-1.el9.x86_64 2024-04-25T00:21:41.956 INFO:teuthology.orchestra.run.smithi032.stdout: protobuf-3.14.0-13.el9.x86_64 2024-04-25T00:21:41.956 INFO:teuthology.orchestra.run.smithi032.stdout: protobuf-compiler-3.14.0-13.el9.x86_64 2024-04-25T00:21:41.956 INFO:teuthology.orchestra.run.smithi032.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-25T00:21:41.957 INFO:teuthology.orchestra.run.smithi032.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-04-25T00:21:41.957 INFO:teuthology.orchestra.run.smithi032.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-25T00:21:41.957 INFO:teuthology.orchestra.run.smithi032.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-25T00:21:41.957 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-25T00:21:41.957 INFO:teuthology.orchestra.run.smithi032.stdout: python3-ceph-argparse-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.957 INFO:teuthology.orchestra.run.smithi032.stdout: python3-ceph-common-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.957 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cephfs-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.957 INFO:teuthology.orchestra.run.smithi032.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-25T00:21:41.957 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-25T00:21:41.957 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-25T00:21:41.957 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-25T00:21:41.957 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cmd2-2.4.2-2.el9.noarch 2024-04-25T00:21:41.957 INFO:teuthology.orchestra.run.smithi032.stdout: python3-colorama-0.4.4-12.el9.noarch 2024-04-25T00:21:41.957 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-25T00:21:41.958 INFO:teuthology.orchestra.run.smithi032.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-04-25T00:21:41.958 INFO:teuthology.orchestra.run.smithi032.stdout: python3-google-auth-1:2.29.0-1.el9.noarch 2024-04-25T00:21:41.958 INFO:teuthology.orchestra.run.smithi032.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2024-04-25T00:21:41.958 INFO:teuthology.orchestra.run.smithi032.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2024-04-25T00:21:41.958 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-25T00:21:41.958 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-25T00:21:41.958 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-25T00:21:41.958 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-25T00:21:41.958 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-25T00:21:41.958 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-25T00:21:41.958 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-25T00:21:41.958 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-25T00:21:41.959 INFO:teuthology.orchestra.run.smithi032.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-25T00:21:41.959 INFO:teuthology.orchestra.run.smithi032.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-25T00:21:41.959 INFO:teuthology.orchestra.run.smithi032.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-25T00:21:41.959 INFO:teuthology.orchestra.run.smithi032.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-25T00:21:41.959 INFO:teuthology.orchestra.run.smithi032.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-25T00:21:41.959 INFO:teuthology.orchestra.run.smithi032.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-25T00:21:41.959 INFO:teuthology.orchestra.run.smithi032.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-25T00:21:41.959 INFO:teuthology.orchestra.run.smithi032.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-25T00:21:41.959 INFO:teuthology.orchestra.run.smithi032.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-25T00:21:41.959 INFO:teuthology.orchestra.run.smithi032.stdout: python3-packaging-20.9-5.el9.noarch 2024-04-25T00:21:41.959 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-25T00:21:41.959 INFO:teuthology.orchestra.run.smithi032.stdout: python3-ply-3.11-14.el9.noarch 2024-04-25T00:21:41.959 INFO:teuthology.orchestra.run.smithi032.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-25T00:21:41.960 INFO:teuthology.orchestra.run.smithi032.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-25T00:21:41.960 INFO:teuthology.orchestra.run.smithi032.stdout: python3-protobuf-3.14.0-13.el9.noarch 2024-04-25T00:21:41.960 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-25T00:21:41.960 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-25T00:21:41.960 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-25T00:21:41.960 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-25T00:21:41.960 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pyperclip-1.8.2-2.el9.noarch 2024-04-25T00:21:41.960 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-25T00:21:41.960 INFO:teuthology.orchestra.run.smithi032.stdout: python3-rados-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.960 INFO:teuthology.orchestra.run.smithi032.stdout: python3-rbd-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.960 INFO:teuthology.orchestra.run.smithi032.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-25T00:21:41.960 INFO:teuthology.orchestra.run.smithi032.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-25T00:21:41.960 INFO:teuthology.orchestra.run.smithi032.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-25T00:21:41.960 INFO:teuthology.orchestra.run.smithi032.stdout: python3-rgw-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.960 INFO:teuthology.orchestra.run.smithi032.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-25T00:21:41.961 INFO:teuthology.orchestra.run.smithi032.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-25T00:21:41.961 INFO:teuthology.orchestra.run.smithi032.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-25T00:21:41.961 INFO:teuthology.orchestra.run.smithi032.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-25T00:21:41.961 INFO:teuthology.orchestra.run.smithi032.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-25T00:21:41.961 INFO:teuthology.orchestra.run.smithi032.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-25T00:21:41.961 INFO:teuthology.orchestra.run.smithi032.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-25T00:21:41.961 INFO:teuthology.orchestra.run.smithi032.stdout: python3-wcwidth-0.2.5-8.el9.noarch 2024-04-25T00:21:41.961 INFO:teuthology.orchestra.run.smithi032.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-25T00:21:41.961 INFO:teuthology.orchestra.run.smithi032.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-25T00:21:41.961 INFO:teuthology.orchestra.run.smithi032.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-25T00:21:41.961 INFO:teuthology.orchestra.run.smithi032.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-25T00:21:41.961 INFO:teuthology.orchestra.run.smithi032.stdout: qatlib-24.02.0-1.el9.x86_64 2024-04-25T00:21:41.961 INFO:teuthology.orchestra.run.smithi032.stdout: qatlib-service-24.02.0-1.el9.x86_64 2024-04-25T00:21:41.961 INFO:teuthology.orchestra.run.smithi032.stdout: qatzip-libs-1.2.0-1.el9.x86_64 2024-04-25T00:21:41.961 INFO:teuthology.orchestra.run.smithi032.stdout: rbd-fuse-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.962 INFO:teuthology.orchestra.run.smithi032.stdout: rbd-mirror-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.962 INFO:teuthology.orchestra.run.smithi032.stdout: rbd-nbd-2:19.0.0-2445.gd2c45d97.el9.x86_64 2024-04-25T00:21:41.962 INFO:teuthology.orchestra.run.smithi032.stdout: re2-1:20211101-3.el9.x86_64 2024-04-25T00:21:41.962 INFO:teuthology.orchestra.run.smithi032.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-25T00:21:41.962 INFO:teuthology.orchestra.run.smithi032.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-25T00:21:41.962 INFO:teuthology.orchestra.run.smithi032.stdout: uuid-1.6.2-55.el9.x86_64 2024-04-25T00:21:41.962 INFO:teuthology.orchestra.run.smithi032.stdout: xfsprogs-devel-6.3.0-1.el9.x86_64 2024-04-25T00:21:41.962 INFO:teuthology.orchestra.run.smithi032.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-25T00:21:41.963 INFO:teuthology.orchestra.run.smithi032.stdout: zlib-devel-1.2.11-41.el9.x86_64 2024-04-25T00:21:41.963 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:21:41.963 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-04-25T00:21:42.323 DEBUG:teuthology.parallel:result is None 2024-04-25T00:21:42.324 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=d2c45d977142e024a59b68a8f1dce8f0b9840551 2024-04-25T00:21:42.553 DEBUG:teuthology.orchestra.run.smithi032:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-25T00:21:42.579 INFO:teuthology.orchestra.run.smithi032.stdout:19.0.0-2445.gd2c45d97.el9 2024-04-25T00:21:42.579 INFO:teuthology.packaging:The installed version of ceph is 19.0.0-2445.gd2c45d97.el9 2024-04-25T00:21:42.580 INFO:teuthology.task.install:The correct ceph version 19.0.0-2445.gd2c45d97 is installed. 2024-04-25T00:21:42.582 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=d2c45d977142e024a59b68a8f1dce8f0b9840551 2024-04-25T00:21:42.804 DEBUG:teuthology.orchestra.run.smithi121:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-25T00:21:42.825 INFO:teuthology.orchestra.run.smithi121.stdout:19.0.0-2445.gd2c45d97.el9 2024-04-25T00:21:42.825 INFO:teuthology.packaging:The installed version of ceph is 19.0.0-2445.gd2c45d97.el9 2024-04-25T00:21:42.825 INFO:teuthology.task.install:The correct ceph version 19.0.0-2445.gd2c45d97 is installed. 2024-04-25T00:21:42.828 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=d2c45d977142e024a59b68a8f1dce8f0b9840551 2024-04-25T00:21:43.060 DEBUG:teuthology.orchestra.run.smithi130:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-25T00:21:43.086 INFO:teuthology.orchestra.run.smithi130.stdout:19.0.0-2445.gd2c45d97.el9 2024-04-25T00:21:43.086 INFO:teuthology.packaging:The installed version of ceph is 19.0.0-2445.gd2c45d97.el9 2024-04-25T00:21:43.086 INFO:teuthology.task.install:The correct ceph version 19.0.0-2445.gd2c45d97 is installed. 2024-04-25T00:21:43.088 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-04-25T00:21:43.089 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-04-25T00:21:43.089 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-25T00:21:43.115 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-04-25T00:21:43.115 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-25T00:21:43.141 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-04-25T00:21:43.141 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-25T00:21:43.171 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-04-25T00:21:43.171 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-04-25T00:21:43.171 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd of=/usr/bin/daemon-helper 2024-04-25T00:21:43.197 DEBUG:teuthology.orchestra.run.smithi032:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-25T00:21:43.262 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-04-25T00:21:43.262 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/usr/bin/daemon-helper 2024-04-25T00:21:43.285 DEBUG:teuthology.orchestra.run.smithi121:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-25T00:21:43.349 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-04-25T00:21:43.349 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/usr/bin/daemon-helper 2024-04-25T00:21:43.377 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-25T00:21:43.444 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-04-25T00:21:43.445 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-04-25T00:21:43.445 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-25T00:21:43.470 DEBUG:teuthology.orchestra.run.smithi032:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-25T00:21:43.534 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-04-25T00:21:43.535 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-25T00:21:43.558 DEBUG:teuthology.orchestra.run.smithi121:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-25T00:21:43.621 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-04-25T00:21:43.621 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-25T00:21:43.645 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-25T00:21:43.710 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-04-25T00:21:43.710 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-04-25T00:21:43.710 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd of=/usr/bin/stdin-killer 2024-04-25T00:21:43.735 DEBUG:teuthology.orchestra.run.smithi032:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-25T00:21:43.800 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-04-25T00:21:43.800 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/usr/bin/stdin-killer 2024-04-25T00:21:43.824 DEBUG:teuthology.orchestra.run.smithi121:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-25T00:21:43.887 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-04-25T00:21:43.888 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/usr/bin/stdin-killer 2024-04-25T00:21:43.910 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-25T00:21:43.974 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-25T00:21:44.085 INFO:tasks.cephadm:Config: {'roleless': False, 'cephfs': {'ec_profile': ['m=2', 'k=2', 'crush-failure-domain=osd'], 'max_mds': 3, 'session_timeout': 300, 'standby_replay': True}, 'conf': {'client': {'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'global': {'ms die on skipped message': False}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 1, 'mds bal fragment size max': 10000, 'mds bal merge size': 5, 'mds bal split bits': 3, 'mds bal split size': 100, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mgr': {'cephsqlite lock renewal timeout': 900000, 'debug cephsqlite': 20, 'debug client': 20, 'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon down mkfs grace': 300, 'mon op complaint time': 120}, 'osd': {'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': 20, 'debug bluestore': 20, 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': 10, 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore', 'osd op complaint time': 180, 'osd shutdown pgref assert': True, 'osd_max_omap_entries_per_request': 10}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'MDS_UP_LESS_THAN_MAX', 'filesystem is online with fewer MDS than max_mds', 'POOL_APP_NOT_ENABLED', 'overall HEALTH_', 'Replacing daemon', 'deprecated feature inline_data', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding', 'MON_DOWN', 'PG_AVAILABILITY', 'PG_DEGRADED', 'Reduced data availability', 'slow metadata IO', 'SLOW_OPS', 'slow request'], 'sha1': 'd2c45d977142e024a59b68a8f1dce8f0b9840551', 'subvols': {'create': 2}} 2024-04-25T00:21:44.085 INFO:tasks.cephadm:Cluster image is quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 2024-04-25T00:21:44.085 INFO:tasks.cephadm:Cluster fsid is cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:21:44.085 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-25T00:21:44.086 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.32', 'mon.b': '172.21.15.121', 'mon.c': '172.21.15.130'} 2024-04-25T00:21:44.086 INFO:tasks.cephadm:First mon is mon.a on smithi032 2024-04-25T00:21:44.086 INFO:tasks.cephadm:First mgr is x 2024-04-25T00:21:44.086 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-25T00:21:44.086 DEBUG:teuthology.orchestra.run.smithi032:> sudo hostname $(hostname -s) 2024-04-25T00:21:44.111 DEBUG:teuthology.orchestra.run.smithi121:> sudo hostname $(hostname -s) 2024-04-25T00:21:44.135 DEBUG:teuthology.orchestra.run.smithi130:> sudo hostname $(hostname -s) 2024-04-25T00:21:44.159 INFO:tasks.cephadm:Downloading "compiled" cephadm from cachra 2024-04-25T00:21:44.159 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=d2c45d977142e024a59b68a8f1dce8f0b9840551 2024-04-25T00:21:44.387 INFO:tasks.cephadm:builder_project result: [{'status': 'ready', 'sha1': 'd2c45d977142e024a59b68a8f1dce8f0b9840551', 'extra': {'build_url': 'https://jenkins.ceph.com/job/ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic/54987/', 'root_build_cause': 'SCMTRIGGER', 'version': '19.0.0-2445-gd2c45d97', 'node_name': '172.21.5.36+adami06', 'job_name': 'ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic', 'package_manager_version': '19.0.0-2445.gd2c45d97'}, 'url': 'https://3.chacra.ceph.com/r/ceph/squid/d2c45d977142e024a59b68a8f1dce8f0b9840551/centos/9/flavors/default/', 'distro_codename': None, 'modified': '2024-04-24 22:37:46.422067', 'distro_version': '9', 'project': 'ceph', 'flavor': 'default', 'ref': 'squid', 'chacra_url': 'https://3.chacra.ceph.com/repos/ceph/squid/d2c45d977142e024a59b68a8f1dce8f0b9840551/centos/9/flavors/default/', 'archs': ['x86_64', 'source'], 'distro': 'centos'}, {'status': 'ready', 'sha1': 'd2c45d977142e024a59b68a8f1dce8f0b9840551', 'extra': {'build_url': 'https://jenkins.ceph.com/job/ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic/54981/', 'root_build_cause': 'SCMTRIGGER', 'version': '19.0.0-2445-gd2c45d97', 'node_name': '172.21.5.36+adami06', 'job_name': 'ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic', 'package_manager_version': '19.0.0-2445.gd2c45d97'}, 'url': 'https://1.chacra.ceph.com/r/ceph/squid/d2c45d977142e024a59b68a8f1dce8f0b9840551/centos/9/flavors/default/', 'distro_codename': None, 'modified': '2024-04-24 17:00:00.872117', 'distro_version': '9', 'project': 'ceph', 'flavor': 'default', 'ref': 'squid', 'chacra_url': 'https://1.chacra.ceph.com/repos/ceph/squid/d2c45d977142e024a59b68a8f1dce8f0b9840551/centos/9/flavors/default/', 'archs': ['source', 'x86_64'], 'distro': 'centos'}] 2024-04-25T00:21:44.615 INFO:tasks.util.chacra:got chacra host 3.chacra.ceph.com, ref squid, sha1 d2c45d977142e024a59b68a8f1dce8f0b9840551 from https://shaman.ceph.com/api/search/?project=ceph&distros=centos%2F9%2Fx86_64&flavor=default&sha1=d2c45d977142e024a59b68a8f1dce8f0b9840551 2024-04-25T00:21:44.616 INFO:tasks.cephadm:Discovered cachra url: https://3.chacra.ceph.com/binaries/ceph/squid/d2c45d977142e024a59b68a8f1dce8f0b9840551/centos/9/x86_64/flavors/default/cephadm 2024-04-25T00:21:44.616 DEBUG:teuthology.orchestra.run.smithi032:> curl --silent -L https://3.chacra.ceph.com/binaries/ceph/squid/d2c45d977142e024a59b68a8f1dce8f0b9840551/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-25T00:21:45.175 INFO:teuthology.orchestra.run.smithi032.stdout:-rw-r--r--. 1 ubuntu ubuntu 781409 Apr 25 00:21 /home/ubuntu/cephtest/cephadm 2024-04-25T00:21:45.175 DEBUG:teuthology.orchestra.run.smithi121:> curl --silent -L https://3.chacra.ceph.com/binaries/ceph/squid/d2c45d977142e024a59b68a8f1dce8f0b9840551/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-25T00:21:45.566 INFO:teuthology.orchestra.run.smithi121.stdout:-rw-r--r--. 1 ubuntu ubuntu 781409 Apr 25 00:21 /home/ubuntu/cephtest/cephadm 2024-04-25T00:21:45.566 DEBUG:teuthology.orchestra.run.smithi130:> curl --silent -L https://3.chacra.ceph.com/binaries/ceph/squid/d2c45d977142e024a59b68a8f1dce8f0b9840551/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-25T00:21:45.975 INFO:teuthology.orchestra.run.smithi130.stdout:-rw-r--r--. 1 ubuntu ubuntu 781409 Apr 25 00:21 /home/ubuntu/cephtest/cephadm 2024-04-25T00:21:45.976 DEBUG:teuthology.orchestra.run.smithi032:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-25T00:21:45.996 DEBUG:teuthology.orchestra.run.smithi121:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-25T00:21:46.015 DEBUG:teuthology.orchestra.run.smithi130:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-25T00:21:46.049 INFO:tasks.cephadm:Pulling image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 on all hosts... 2024-04-25T00:21:46.049 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 pull 2024-04-25T00:21:46.051 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 pull 2024-04-25T00:21:46.057 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 pull 2024-04-25T00:21:47.590 INFO:teuthology.orchestra.run.smithi130.stderr:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551... 2024-04-25T00:21:47.635 INFO:teuthology.orchestra.run.smithi032.stderr:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551... 2024-04-25T00:21:47.699 INFO:teuthology.orchestra.run.smithi121.stderr:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551... 2024-04-25T00:22:15.996 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-25T00:22:15.996 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_version": "ceph version 19.0.0-2445-gd2c45d97 (d2c45d977142e024a59b68a8f1dce8f0b9840551) squid (dev)", 2024-04-25T00:22:15.997 INFO:teuthology.orchestra.run.smithi130.stdout: "image_id": "f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e", 2024-04-25T00:22:15.997 INFO:teuthology.orchestra.run.smithi130.stdout: "repo_digests": [ 2024-04-25T00:22:15.997 INFO:teuthology.orchestra.run.smithi130.stdout: "quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901" 2024-04-25T00:22:15.997 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-04-25T00:22:15.997 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-25T00:22:30.779 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-25T00:22:30.779 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph_version": "ceph version 19.0.0-2445-gd2c45d97 (d2c45d977142e024a59b68a8f1dce8f0b9840551) squid (dev)", 2024-04-25T00:22:30.779 INFO:teuthology.orchestra.run.smithi121.stdout: "image_id": "f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e", 2024-04-25T00:22:30.779 INFO:teuthology.orchestra.run.smithi121.stdout: "repo_digests": [ 2024-04-25T00:22:30.779 INFO:teuthology.orchestra.run.smithi121.stdout: "quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901" 2024-04-25T00:22:30.779 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2024-04-25T00:22:30.779 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-25T00:22:36.897 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:22:36.897 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph_version": "ceph version 19.0.0-2445-gd2c45d97 (d2c45d977142e024a59b68a8f1dce8f0b9840551) squid (dev)", 2024-04-25T00:22:36.897 INFO:teuthology.orchestra.run.smithi032.stdout: "image_id": "f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e", 2024-04-25T00:22:36.898 INFO:teuthology.orchestra.run.smithi032.stdout: "repo_digests": [ 2024-04-25T00:22:36.898 INFO:teuthology.orchestra.run.smithi032.stdout: "quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901" 2024-04-25T00:22:36.898 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-04-25T00:22:36.898 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:22:36.924 DEBUG:teuthology.orchestra.run.smithi032:> sudo mkdir -p /etc/ceph 2024-04-25T00:22:36.964 DEBUG:teuthology.orchestra.run.smithi121:> sudo mkdir -p /etc/ceph 2024-04-25T00:22:36.996 DEBUG:teuthology.orchestra.run.smithi130:> sudo mkdir -p /etc/ceph 2024-04-25T00:22:37.028 DEBUG:teuthology.orchestra.run.smithi032:> sudo chmod 777 /etc/ceph 2024-04-25T00:22:37.053 DEBUG:teuthology.orchestra.run.smithi121:> sudo chmod 777 /etc/ceph 2024-04-25T00:22:37.079 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod 777 /etc/ceph 2024-04-25T00:22:37.108 INFO:tasks.cephadm:Writing seed config... 2024-04-25T00:22:37.109 INFO:tasks.cephadm: override: [client] client mount timeout = 600 2024-04-25T00:22:37.109 INFO:tasks.cephadm: override: [client] debug client = 20 2024-04-25T00:22:37.109 INFO:tasks.cephadm: override: [client] debug ms = 1 2024-04-25T00:22:37.109 INFO:tasks.cephadm: override: [client] rados mon op timeout = 900 2024-04-25T00:22:37.110 INFO:tasks.cephadm: override: [client] rados osd op timeout = 900 2024-04-25T00:22:37.110 INFO:tasks.cephadm: override: [global] ms die on skipped message = False 2024-04-25T00:22:37.110 INFO:tasks.cephadm: override: [mds] debug mds = 20 2024-04-25T00:22:37.110 INFO:tasks.cephadm: override: [mds] debug mds balancer = 20 2024-04-25T00:22:37.110 INFO:tasks.cephadm: override: [mds] debug ms = 1 2024-04-25T00:22:37.110 INFO:tasks.cephadm: override: [mds] mds bal fragment size max = 10000 2024-04-25T00:22:37.110 INFO:tasks.cephadm: override: [mds] mds bal merge size = 5 2024-04-25T00:22:37.110 INFO:tasks.cephadm: override: [mds] mds bal split bits = 3 2024-04-25T00:22:37.110 INFO:tasks.cephadm: override: [mds] mds bal split size = 100 2024-04-25T00:22:37.110 INFO:tasks.cephadm: override: [mds] mds debug frag = True 2024-04-25T00:22:37.110 INFO:tasks.cephadm: override: [mds] mds debug scatterstat = True 2024-04-25T00:22:37.110 INFO:tasks.cephadm: override: [mds] mds op complaint time = 180 2024-04-25T00:22:37.110 INFO:tasks.cephadm: override: [mds] mds verify scatter = True 2024-04-25T00:22:37.110 INFO:tasks.cephadm: override: [mds] osd op complaint time = 180 2024-04-25T00:22:37.111 INFO:tasks.cephadm: override: [mds] rados mon op timeout = 900 2024-04-25T00:22:37.111 INFO:tasks.cephadm: override: [mds] rados osd op timeout = 900 2024-04-25T00:22:37.111 INFO:tasks.cephadm: override: [mgr] cephsqlite lock renewal timeout = 900000 2024-04-25T00:22:37.111 INFO:tasks.cephadm: override: [mgr] debug cephsqlite = 20 2024-04-25T00:22:37.111 INFO:tasks.cephadm: override: [mgr] debug client = 20 2024-04-25T00:22:37.111 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-25T00:22:37.111 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-25T00:22:37.111 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-25T00:22:37.111 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-25T00:22:37.111 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-25T00:22:37.111 INFO:tasks.cephadm: override: [mon] mon down mkfs grace = 300 2024-04-25T00:22:37.111 INFO:tasks.cephadm: override: [mon] mon op complaint time = 120 2024-04-25T00:22:37.111 INFO:tasks.cephadm: override: [osd] bluestore block size = 96636764160 2024-04-25T00:22:37.111 INFO:tasks.cephadm: override: [osd] bluestore fsck on mount = True 2024-04-25T00:22:37.112 INFO:tasks.cephadm: override: [osd] debug bluefs = 20 2024-04-25T00:22:37.112 INFO:tasks.cephadm: override: [osd] debug bluestore = 20 2024-04-25T00:22:37.112 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-25T00:22:37.112 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-25T00:22:37.112 INFO:tasks.cephadm: override: [osd] debug rocksdb = 10 2024-04-25T00:22:37.112 INFO:tasks.cephadm: override: [osd] mon osd backfillfull_ratio = 0.85 2024-04-25T00:22:37.112 INFO:tasks.cephadm: override: [osd] mon osd full ratio = 0.9 2024-04-25T00:22:37.112 INFO:tasks.cephadm: override: [osd] mon osd nearfull ratio = 0.8 2024-04-25T00:22:37.112 INFO:tasks.cephadm: override: [osd] osd failsafe full ratio = 0.95 2024-04-25T00:22:37.112 INFO:tasks.cephadm: override: [osd] osd objectstore = bluestore 2024-04-25T00:22:37.112 INFO:tasks.cephadm: override: [osd] osd op complaint time = 180 2024-04-25T00:22:37.112 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-04-25T00:22:37.112 INFO:tasks.cephadm: override: [osd] osd_max_omap_entries_per_request = 10 2024-04-25T00:22:37.113 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-04-25T00:22:37.113 DEBUG:teuthology.orchestra.run.smithi032:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-25T00:22:37.129 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = cb997156-0299-11ef-bc93-c7b262605968 ms die on skipped message = False [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true bluestore block size = 96636764160 bluestore fsck on mount = True debug bluefs = 20 debug bluestore = 20 debug ms = 1 debug osd = 20 debug rocksdb = 10 mon osd backfillfull_ratio = 0.85 mon osd full ratio = 0.9 mon osd nearfull ratio = 0.8 osd failsafe full ratio = 0.95 osd objectstore = bluestore osd op complaint time = 180 osd_max_omap_entries_per_request = 10 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false cephsqlite lock renewal timeout = 900000 debug cephsqlite = 20 debug client = 20 debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 mon down mkfs grace = 300 mon op complaint time = 120 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true [client] client mount timeout = 600 debug client = 20 debug ms = 1 rados mon op timeout = 900 rados osd op timeout = 900 [mds] debug mds = 20 debug mds balancer = 20 debug ms = 1 mds bal fragment size max = 10000 mds bal merge size = 5 mds bal split bits = 3 mds bal split size = 100 mds debug frag = True mds debug scatterstat = True mds op complaint time = 180 mds verify scatter = True osd op complaint time = 180 rados mon op timeout = 900 rados osd op timeout = 900 2024-04-25T00:22:37.129 DEBUG:teuthology.orchestra.run.smithi032:mon.a> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mon.a.service 2024-04-25T00:22:37.172 DEBUG:teuthology.orchestra.run.smithi032:mgr.x> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mgr.x.service 2024-04-25T00:22:37.216 INFO:tasks.cephadm:Bootstrapping... 2024-04-25T00:22:37.216 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 -v bootstrap --fsid cb997156-0299-11ef-bc93-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id x --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.32 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-25T00:22:37.412 INFO:teuthology.orchestra.run.smithi032.stdout:-------------------------------------------------------------------------------- 2024-04-25T00:22:37.412 INFO:teuthology.orchestra.run.smithi032.stdout:cephadm ['--image', 'quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551', '-v', 'bootstrap', '--fsid', 'cb997156-0299-11ef-bc93-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'x', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.32', '--skip-admin-label'] 2024-04-25T00:22:37.412 INFO:teuthology.orchestra.run.smithi032.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-04-25T00:22:37.412 INFO:teuthology.orchestra.run.smithi032.stdout:Verifying podman|docker is present... 2024-04-25T00:22:37.439 INFO:teuthology.orchestra.run.smithi032.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-25T00:22:37.439 INFO:teuthology.orchestra.run.smithi032.stdout:Verifying lvm2 is present... 2024-04-25T00:22:37.440 INFO:teuthology.orchestra.run.smithi032.stdout:Verifying time synchronization is in place... 2024-04-25T00:22:37.447 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-25T00:22:37.447 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-25T00:22:37.453 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-25T00:22:37.453 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout inactive 2024-04-25T00:22:37.460 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout enabled 2024-04-25T00:22:37.466 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout active 2024-04-25T00:22:37.466 INFO:teuthology.orchestra.run.smithi032.stdout:Unit chronyd.service is enabled and running 2024-04-25T00:22:37.466 INFO:teuthology.orchestra.run.smithi032.stdout:Repeating the final host check... 2024-04-25T00:22:37.490 INFO:teuthology.orchestra.run.smithi032.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-25T00:22:37.490 INFO:teuthology.orchestra.run.smithi032.stdout:podman (/bin/podman) version 4.9.4 is present 2024-04-25T00:22:37.490 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl is present 2024-04-25T00:22:37.490 INFO:teuthology.orchestra.run.smithi032.stdout:lvcreate is present 2024-04-25T00:22:37.496 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-25T00:22:37.496 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-25T00:22:37.503 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-25T00:22:37.503 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout inactive 2024-04-25T00:22:37.510 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout enabled 2024-04-25T00:22:37.516 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout active 2024-04-25T00:22:37.516 INFO:teuthology.orchestra.run.smithi032.stdout:Unit chronyd.service is enabled and running 2024-04-25T00:22:37.516 INFO:teuthology.orchestra.run.smithi032.stdout:Host looks OK 2024-04-25T00:22:37.516 INFO:teuthology.orchestra.run.smithi032.stdout:Cluster fsid: cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:22:37.516 INFO:teuthology.orchestra.run.smithi032.stdout:Acquiring lock 139959150908320 on /run/cephadm/cb997156-0299-11ef-bc93-c7b262605968.lock 2024-04-25T00:22:37.516 INFO:teuthology.orchestra.run.smithi032.stdout:Lock 139959150908320 acquired on /run/cephadm/cb997156-0299-11ef-bc93-c7b262605968.lock 2024-04-25T00:22:37.517 INFO:teuthology.orchestra.run.smithi032.stdout:Verifying IP 172.21.15.32 port 3300 ... 2024-04-25T00:22:37.517 INFO:teuthology.orchestra.run.smithi032.stdout:Verifying IP 172.21.15.32 port 6789 ... 2024-04-25T00:22:37.517 INFO:teuthology.orchestra.run.smithi032.stdout:Base mon IP(s) is [172.21.15.32:3300, 172.21.15.32:6789], mon addrv is [v2:172.21.15.32:3300,v1:172.21.15.32:6789] 2024-04-25T00:22:37.520 INFO:teuthology.orchestra.run.smithi032.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.32 metric 100 2024-04-25T00:22:37.520 INFO:teuthology.orchestra.run.smithi032.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.32 metric 100 2024-04-25T00:22:37.528 INFO:teuthology.orchestra.run.smithi032.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-04-25T00:22:37.528 INFO:teuthology.orchestra.run.smithi032.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 1024 pref medium 2024-04-25T00:22:37.528 INFO:teuthology.orchestra.run.smithi032.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 100 pref medium 2024-04-25T00:22:37.528 INFO:teuthology.orchestra.run.smithi032.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-25T00:22:37.528 INFO:teuthology.orchestra.run.smithi032.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-04-25T00:22:37.528 INFO:teuthology.orchestra.run.smithi032.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-25T00:22:37.528 INFO:teuthology.orchestra.run.smithi032.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2024-04-25T00:22:37.528 INFO:teuthology.orchestra.run.smithi032.stdout:/sbin/ip: stdout inet6 fe80::ae1f:6bff:fef8:2602/64 scope link noprefixroute 2024-04-25T00:22:37.528 INFO:teuthology.orchestra.run.smithi032.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-25T00:22:37.528 INFO:teuthology.orchestra.run.smithi032.stdout:Mon IP `172.21.15.32` is in CIDR network `172.21.0.0/20` 2024-04-25T00:22:37.528 INFO:teuthology.orchestra.run.smithi032.stdout:Mon IP `172.21.15.32` is in CIDR network `172.21.0.0/20` 2024-04-25T00:22:37.528 INFO:teuthology.orchestra.run.smithi032.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-04-25T00:22:37.528 INFO:teuthology.orchestra.run.smithi032.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-25T00:22:37.529 INFO:teuthology.orchestra.run.smithi032.stdout:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551... 2024-04-25T00:22:38.017 INFO:teuthology.orchestra.run.smithi032.stdout:/bin/podman: stdout f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e 2024-04-25T00:22:38.017 INFO:teuthology.orchestra.run.smithi032.stdout:/bin/podman: stderr Trying to pull quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551... 2024-04-25T00:22:38.017 INFO:teuthology.orchestra.run.smithi032.stdout:/bin/podman: stderr Getting image source signatures 2024-04-25T00:22:38.017 INFO:teuthology.orchestra.run.smithi032.stdout:/bin/podman: stderr Copying blob sha256:58a99251f41e8a467bafa0ec849428709568906ca732f10972304f37056be6c5 2024-04-25T00:22:38.017 INFO:teuthology.orchestra.run.smithi032.stdout:/bin/podman: stderr Copying blob sha256:3d665f4c032644be9cae748af04bee69f2bfe96cdc3616dabe0ab17899633ab5 2024-04-25T00:22:38.017 INFO:teuthology.orchestra.run.smithi032.stdout:/bin/podman: stderr Copying config sha256:f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e 2024-04-25T00:22:38.018 INFO:teuthology.orchestra.run.smithi032.stdout:/bin/podman: stderr Writing manifest to image destination 2024-04-25T00:22:38.963 INFO:teuthology.orchestra.run.smithi032.stdout:ceph: stdout ceph version 19.0.0-2445-gd2c45d97 (d2c45d977142e024a59b68a8f1dce8f0b9840551) squid (dev) 2024-04-25T00:22:38.963 INFO:teuthology.orchestra.run.smithi032.stdout:Ceph version: ceph version 19.0.0-2445-gd2c45d97 (d2c45d977142e024a59b68a8f1dce8f0b9840551) squid (dev) 2024-04-25T00:22:38.963 INFO:teuthology.orchestra.run.smithi032.stdout:Extracting ceph user uid/gid from container image... 2024-04-25T00:22:39.784 INFO:teuthology.orchestra.run.smithi032.stdout:stat: stdout 167 167 2024-04-25T00:22:39.785 INFO:teuthology.orchestra.run.smithi032.stdout:Creating initial keys... 2024-04-25T00:22:40.585 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph-authtool: stdout AQBQoilm9u3KBhAADG+rUSqESmn95E8FUPlPsQ== 2024-04-25T00:22:41.484 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph-authtool: stdout AQBQoilmv/PdNRAAQz2j/dOQVadx1MDWDinUbw== 2024-04-25T00:22:42.431 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph-authtool: stdout AQBRoilmHPvMMBAA6KjTt+O+JZPtazrMfnpC+Q== 2024-04-25T00:22:42.432 INFO:teuthology.orchestra.run.smithi032.stdout:Creating initial monmap... 2024-04-25T00:22:43.437 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-25T00:22:43.437 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = quincy 2024-04-25T00:22:43.437 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:22:43.437 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-25T00:22:43.437 INFO:teuthology.orchestra.run.smithi032.stdout:monmaptool for a [v2:172.21.15.32:3300,v1:172.21.15.32:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-25T00:22:43.437 INFO:teuthology.orchestra.run.smithi032.stdout:setting min_mon_release = quincy 2024-04-25T00:22:43.437 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/monmaptool: set fsid to cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:22:43.437 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-25T00:22:43.438 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:22:43.438 INFO:teuthology.orchestra.run.smithi032.stdout:Creating mon... 2024-04-25T00:22:44.319 INFO:teuthology.orchestra.run.smithi032.stdout:create mon.a on 2024-04-25T00:22:44.541 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-04-25T00:22:44.700 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-25T00:22:44.890 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-cb997156-0299-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-cb997156-0299-11ef-bc93-c7b262605968.target. 2024-04-25T00:22:44.894 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-cb997156-0299-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-cb997156-0299-11ef-bc93-c7b262605968.target. 2024-04-25T00:22:45.279 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-cb997156-0299-11ef-bc93-c7b262605968@mon.a 2024-04-25T00:22:45.279 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stderr Failed to reset failed state of unit ceph-cb997156-0299-11ef-bc93-c7b262605968@mon.a.service: Unit ceph-cb997156-0299-11ef-bc93-c7b262605968@mon.a.service not loaded. 2024-04-25T00:22:45.458 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-cb997156-0299-11ef-bc93-c7b262605968.target.wants/ceph-cb997156-0299-11ef-bc93-c7b262605968@mon.a.service → /etc/systemd/system/ceph-cb997156-0299-11ef-bc93-c7b262605968@.service. 2024-04-25T00:22:46.145 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-25T00:22:46.145 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout disabled 2024-04-25T00:22:46.152 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-25T00:22:46.152 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout inactive 2024-04-25T00:22:46.152 INFO:teuthology.orchestra.run.smithi032.stdout:firewalld.service is not enabled 2024-04-25T00:22:46.152 INFO:teuthology.orchestra.run.smithi032.stdout:Not possible to enable service . firewalld.service is not available 2024-04-25T00:22:46.153 INFO:teuthology.orchestra.run.smithi032.stdout:Waiting for mon to start... 2024-04-25T00:22:46.153 INFO:teuthology.orchestra.run.smithi032.stdout:Waiting for mon... 2024-04-25T00:22:47.381 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout cluster: 2024-04-25T00:22:47.381 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout id: cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:22:47.382 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-04-25T00:22:47.382 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 2024-04-25T00:22:47.382 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout services: 2024-04-25T00:22:47.382 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.551609s) 2024-04-25T00:22:47.382 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-04-25T00:22:47.382 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-04-25T00:22:47.382 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 2024-04-25T00:22:47.382 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout data: 2024-04-25T00:22:47.382 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-04-25T00:22:47.382 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-04-25T00:22:47.382 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-04-25T00:22:47.382 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout pgs: 2024-04-25T00:22:47.382 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 2024-04-25T00:22:47.383 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.573+0000 7f96ca298700 1 Processor -- start 2024-04-25T00:22:47.383 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.573+0000 7f96ca298700 1 -- start start 2024-04-25T00:22:47.383 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.574+0000 7f96ca298700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f96c4107c50 0x7f96c4108070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:22:47.383 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.574+0000 7f96ca298700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f96c4108640 con 0x7f96c4107c50 2024-04-25T00:22:47.383 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.574+0000 7f96c3fff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f96c4107c50 0x7f96c4108070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:22:47.383 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.574+0000 7f96c3fff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f96c4107c50 0x7f96c4108070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:43540/0 (socket says 172.21.15.32:43540) 2024-04-25T00:22:47.383 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.574+0000 7f96c3fff700 1 -- 172.21.15.32:0/3531519837 learned_addr learned my addr 172.21.15.32:0/3531519837 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:22:47.383 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.574+0000 7f96c3fff700 1 -- 172.21.15.32:0/3531519837 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f96c4108db0 con 0x7f96c4107c50 2024-04-25T00:22:47.383 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.575+0000 7f96c3fff700 1 --2- 172.21.15.32:0/3531519837 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f96c4107c50 0x7f96c4108070 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f96b40098a0 tx=0x7f96b4009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=47587d7ce7cc8cea server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:22:47.383 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.575+0000 7f96c2ffd700 1 -- 172.21.15.32:0/3531519837 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f96b4004160 con 0x7f96c4107c50 2024-04-25T00:22:47.383 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.575+0000 7f96c2ffd700 1 -- 172.21.15.32:0/3531519837 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(0 keys) ==== 4+0+0 (secure 0 0 0) 0x7f96b4004300 con 0x7f96c4107c50 2024-04-25T00:22:47.383 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.575+0000 7f96c2ffd700 1 -- 172.21.15.32:0/3531519837 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f96b40036a0 con 0x7f96c4107c50 2024-04-25T00:22:47.383 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.575+0000 7f96ca298700 1 -- 172.21.15.32:0/3531519837 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f96c4107c50 msgr2=0x7f96c4108070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:22:47.384 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.575+0000 7f96ca298700 1 --2- 172.21.15.32:0/3531519837 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f96c4107c50 0x7f96c4108070 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f96b40098a0 tx=0x7f96b4009bb0 comp rx=0 tx=0).stop 2024-04-25T00:22:47.384 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.575+0000 7f96ca298700 1 -- 172.21.15.32:0/3531519837 shutdown_connections 2024-04-25T00:22:47.384 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.575+0000 7f96ca298700 1 --2- 172.21.15.32:0/3531519837 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f96c4107c50 0x7f96c4108070 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:22:47.384 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.575+0000 7f96ca298700 1 -- 172.21.15.32:0/3531519837 >> 172.21.15.32:0/3531519837 conn(0x7f96c4103230 msgr2=0x7f96c4105650 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:22:47.384 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.575+0000 7f96ca298700 1 -- 172.21.15.32:0/3531519837 shutdown_connections 2024-04-25T00:22:47.384 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.575+0000 7f96ca298700 1 -- 172.21.15.32:0/3531519837 wait complete. 2024-04-25T00:22:47.384 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.576+0000 7f96ca298700 1 Processor -- start 2024-04-25T00:22:47.384 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.576+0000 7f96ca298700 1 -- start start 2024-04-25T00:22:47.384 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.576+0000 7f96ca298700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f96c4107c50 0x7f96c419b2d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:22:47.384 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.576+0000 7f96ca298700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f96c41092e0 con 0x7f96c4107c50 2024-04-25T00:22:47.384 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.576+0000 7f96c3fff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f96c4107c50 0x7f96c419b2d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:22:47.384 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.576+0000 7f96c3fff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f96c4107c50 0x7f96c419b2d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:43552/0 (socket says 172.21.15.32:43552) 2024-04-25T00:22:47.384 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.576+0000 7f96c3fff700 1 -- 172.21.15.32:0/4215247724 learned_addr learned my addr 172.21.15.32:0/4215247724 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:22:47.385 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.577+0000 7f96c3fff700 1 -- 172.21.15.32:0/4215247724 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f96c419b810 con 0x7f96c4107c50 2024-04-25T00:22:47.385 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.577+0000 7f96c3fff700 1 --2- 172.21.15.32:0/4215247724 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f96c4107c50 0x7f96c419b2d0 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7f96b4000c00 tx=0x7f96b4004750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:22:47.385 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.577+0000 7f96c17fa700 1 -- 172.21.15.32:0/4215247724 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f96b4003db0 con 0x7f96c4107c50 2024-04-25T00:22:47.385 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.577+0000 7f96c17fa700 1 -- 172.21.15.32:0/4215247724 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(0 keys) ==== 4+0+0 (secure 0 0 0) 0x7f96b4011630 con 0x7f96c4107c50 2024-04-25T00:22:47.385 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.577+0000 7f96c17fa700 1 -- 172.21.15.32:0/4215247724 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f96b4011950 con 0x7f96c4107c50 2024-04-25T00:22:47.385 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.577+0000 7f96ca298700 1 -- 172.21.15.32:0/4215247724 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f96c419baa0 con 0x7f96c4107c50 2024-04-25T00:22:47.385 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.577+0000 7f96ca298700 1 -- 172.21.15.32:0/4215247724 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f96c419e790 con 0x7f96c4107c50 2024-04-25T00:22:47.385 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.577+0000 7f96c17fa700 1 -- 172.21.15.32:0/4215247724 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f96b401a460 con 0x7f96c4107c50 2024-04-25T00:22:47.385 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.577+0000 7f96c17fa700 1 -- 172.21.15.32:0/4215247724 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f96b401b780 con 0x7f96c4107c50 2024-04-25T00:22:47.385 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.578+0000 7f96ca298700 1 -- 172.21.15.32:0/4215247724 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f96c4063890 con 0x7f96c4107c50 2024-04-25T00:22:47.385 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.580+0000 7f96c17fa700 1 -- 172.21.15.32:0/4215247724 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+73141 (secure 0 0 0) 0x7f96b401b980 con 0x7f96c4107c50 2024-04-25T00:22:47.385 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.654+0000 7f96ca298700 1 -- 172.21.15.32:0/4215247724 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "status"} v 0) -- 0x7f96c402dd80 con 0x7f96c4107c50 2024-04-25T00:22:47.385 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.654+0000 7f96c17fa700 1 -- 172.21.15.32:0/4215247724 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "status"}]=0 v0) ==== 54+0+317 (secure 0 0 0) 0x7f96b401bb60 con 0x7f96c4107c50 2024-04-25T00:22:47.385 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.655+0000 7f96ca298700 1 -- 172.21.15.32:0/4215247724 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f96c4107c50 msgr2=0x7f96c419b2d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:22:47.385 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.655+0000 7f96ca298700 1 --2- 172.21.15.32:0/4215247724 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f96c4107c50 0x7f96c419b2d0 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7f96b4000c00 tx=0x7f96b4004750 comp rx=0 tx=0).stop 2024-04-25T00:22:47.386 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.655+0000 7f96ca298700 1 -- 172.21.15.32:0/4215247724 shutdown_connections 2024-04-25T00:22:47.386 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.655+0000 7f96ca298700 1 --2- 172.21.15.32:0/4215247724 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f96c4107c50 0x7f96c419b2d0 unknown :-1 s=CLOSED pgs=2 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:22:47.386 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.655+0000 7f96ca298700 1 -- 172.21.15.32:0/4215247724 >> 172.21.15.32:0/4215247724 conn(0x7f96c4103230 msgr2=0x7f96c4191f60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:22:47.386 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.655+0000 7f96ca298700 1 -- 172.21.15.32:0/4215247724 shutdown_connections 2024-04-25T00:22:47.386 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:46.655+0000 7f96ca298700 1 -- 172.21.15.32:0/4215247724 wait complete. 2024-04-25T00:22:47.386 INFO:teuthology.orchestra.run.smithi032.stdout:mon is available 2024-04-25T00:22:47.386 INFO:teuthology.orchestra.run.smithi032.stdout:Assimilating anything we can from ceph.conf... 2024-04-25T00:22:48.408 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 2024-04-25T00:22:48.408 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout [global] 2024-04-25T00:22:48.408 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout fsid = cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:22:48.408 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.32:3300,v1:172.21.15.32:6789] 2024-04-25T00:22:48.408 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-25T00:22:48.408 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-25T00:22:48.408 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-25T00:22:48.408 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-25T00:22:48.408 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 2024-04-25T00:22:48.408 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-25T00:22:48.409 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-25T00:22:48.409 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 2024-04-25T00:22:48.409 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout [osd] 2024-04-25T00:22:48.409 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mon_osd_backfillfull_ratio = 0.85 2024-04-25T00:22:48.409 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mon_osd_full_ratio = 0.9 2024-04-25T00:22:48.409 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mon_osd_nearfull_ratio = 0.8 2024-04-25T00:22:48.409 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-25T00:22:48.409 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-25T00:22:48.409 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.789+0000 7f30eb5f9700 1 Processor -- start 2024-04-25T00:22:48.409 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.789+0000 7f30eb5f9700 1 -- start start 2024-04-25T00:22:48.409 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.789+0000 7f30eb5f9700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f30e407de50 0x7f30e407c300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:22:48.409 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.789+0000 7f30eb5f9700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f30e407c840 con 0x7f30e407de50 2024-04-25T00:22:48.409 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.789+0000 7f30e9395700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f30e407de50 0x7f30e407c300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:22:48.409 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.789+0000 7f30e9395700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f30e407de50 0x7f30e407c300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:43556/0 (socket says 172.21.15.32:43556) 2024-04-25T00:22:48.409 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.789+0000 7f30e9395700 1 -- 172.21.15.32:0/3919650773 learned_addr learned my addr 172.21.15.32:0/3919650773 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:22:48.410 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.789+0000 7f30e9395700 1 -- 172.21.15.32:0/3919650773 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f30e407c9c0 con 0x7f30e407de50 2024-04-25T00:22:48.410 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.790+0000 7f30e9395700 1 --2- 172.21.15.32:0/3919650773 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f30e407de50 0x7f30e407c300 secure :-1 s=READY pgs=3 cs=0 l=1 rev1=1 crypto rx=0x7f30e00098a0 tx=0x7f30e0009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7da456a769dd9fe server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:22:48.410 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.790+0000 7f30dbfff700 1 -- 172.21.15.32:0/3919650773 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f30e0004160 con 0x7f30e407de50 2024-04-25T00:22:48.410 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.790+0000 7f30dbfff700 1 -- 172.21.15.32:0/3919650773 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(0 keys) ==== 4+0+0 (secure 0 0 0) 0x7f30e0004300 con 0x7f30e407de50 2024-04-25T00:22:48.410 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.790+0000 7f30dbfff700 1 -- 172.21.15.32:0/3919650773 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f30e00036a0 con 0x7f30e407de50 2024-04-25T00:22:48.410 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.790+0000 7f30eb5f9700 1 -- 172.21.15.32:0/3919650773 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f30e407de50 msgr2=0x7f30e407c300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:22:48.410 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.790+0000 7f30eb5f9700 1 --2- 172.21.15.32:0/3919650773 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f30e407de50 0x7f30e407c300 secure :-1 s=READY pgs=3 cs=0 l=1 rev1=1 crypto rx=0x7f30e00098a0 tx=0x7f30e0009bb0 comp rx=0 tx=0).stop 2024-04-25T00:22:48.410 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.790+0000 7f30eb5f9700 1 -- 172.21.15.32:0/3919650773 shutdown_connections 2024-04-25T00:22:48.410 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.790+0000 7f30eb5f9700 1 --2- 172.21.15.32:0/3919650773 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f30e407de50 0x7f30e407c300 unknown :-1 s=CLOSED pgs=3 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:22:48.410 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.790+0000 7f30eb5f9700 1 -- 172.21.15.32:0/3919650773 >> 172.21.15.32:0/3919650773 conn(0x7f30e4103170 msgr2=0x7f30e41055d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:22:48.410 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.791+0000 7f30eb5f9700 1 -- 172.21.15.32:0/3919650773 shutdown_connections 2024-04-25T00:22:48.411 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.791+0000 7f30eb5f9700 1 -- 172.21.15.32:0/3919650773 wait complete. 2024-04-25T00:22:48.411 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.791+0000 7f30eb5f9700 1 Processor -- start 2024-04-25T00:22:48.411 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.791+0000 7f30eb5f9700 1 -- start start 2024-04-25T00:22:48.411 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.791+0000 7f30eb5f9700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f30e407de50 0x7f30e419d470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:22:48.411 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.791+0000 7f30eb5f9700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f30e407cef0 con 0x7f30e407de50 2024-04-25T00:22:48.411 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.791+0000 7f30e9395700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f30e407de50 0x7f30e419d470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:22:48.411 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.791+0000 7f30e9395700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f30e407de50 0x7f30e419d470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:43566/0 (socket says 172.21.15.32:43566) 2024-04-25T00:22:48.411 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.791+0000 7f30e9395700 1 -- 172.21.15.32:0/2343428638 learned_addr learned my addr 172.21.15.32:0/2343428638 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:22:48.411 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.792+0000 7f30e9395700 1 -- 172.21.15.32:0/2343428638 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f30e419d9b0 con 0x7f30e407de50 2024-04-25T00:22:48.411 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.792+0000 7f30e9395700 1 --2- 172.21.15.32:0/2343428638 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f30e407de50 0x7f30e419d470 secure :-1 s=READY pgs=4 cs=0 l=1 rev1=1 crypto rx=0x7f30e0000c00 tx=0x7f30e0004750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:22:48.411 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.792+0000 7f30da7fc700 1 -- 172.21.15.32:0/2343428638 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f30e0003db0 con 0x7f30e407de50 2024-04-25T00:22:48.411 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.792+0000 7f30da7fc700 1 -- 172.21.15.32:0/2343428638 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(0 keys) ==== 4+0+0 (secure 0 0 0) 0x7f30e0011630 con 0x7f30e407de50 2024-04-25T00:22:48.411 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.792+0000 7f30eb5f9700 1 -- 172.21.15.32:0/2343428638 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f30e419dc40 con 0x7f30e407de50 2024-04-25T00:22:48.411 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.792+0000 7f30da7fc700 1 -- 172.21.15.32:0/2343428638 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f30e0011950 con 0x7f30e407de50 2024-04-25T00:22:48.412 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.792+0000 7f30eb5f9700 1 -- 172.21.15.32:0/2343428638 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f30e41a0930 con 0x7f30e407de50 2024-04-25T00:22:48.412 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.792+0000 7f30da7fc700 1 -- 172.21.15.32:0/2343428638 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f30e0011af0 con 0x7f30e407de50 2024-04-25T00:22:48.412 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.792+0000 7f30da7fc700 1 -- 172.21.15.32:0/2343428638 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f30e001c450 con 0x7f30e407de50 2024-04-25T00:22:48.412 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.793+0000 7f30eb5f9700 1 -- 172.21.15.32:0/2343428638 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f30e4063890 con 0x7f30e407de50 2024-04-25T00:22:48.412 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.795+0000 7f30da7fc700 1 -- 172.21.15.32:0/2343428638 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+73141 (secure 0 0 0) 0x7f30e0032020 con 0x7f30e407de50 2024-04-25T00:22:48.412 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.866+0000 7f30eb5f9700 1 -- 172.21.15.32:0/2343428638 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "config assimilate-conf"} v 0) -- 0x7f30e402dcb0 con 0x7f30e407de50 2024-04-25T00:22:48.412 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.869+0000 7f30da7fc700 1 -- 172.21.15.32:0/2343428638 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "config assimilate-conf"}]=0 v2) ==== 70+0+429 (secure 0 0 0) 0x7f30e0004260 con 0x7f30e407de50 2024-04-25T00:22:48.412 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.870+0000 7f30da7fc700 1 -- 172.21.15.32:0/2343428638 <== mon.0 v2:172.21.15.32:3300/0 8 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f30e001b450 con 0x7f30e407de50 2024-04-25T00:22:48.412 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.871+0000 7f30eb5f9700 1 -- 172.21.15.32:0/2343428638 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f30e407de50 msgr2=0x7f30e419d470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:22:48.412 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.871+0000 7f30eb5f9700 1 --2- 172.21.15.32:0/2343428638 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f30e407de50 0x7f30e419d470 secure :-1 s=READY pgs=4 cs=0 l=1 rev1=1 crypto rx=0x7f30e0000c00 tx=0x7f30e0004750 comp rx=0 tx=0).stop 2024-04-25T00:22:48.412 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.871+0000 7f30eb5f9700 1 -- 172.21.15.32:0/2343428638 shutdown_connections 2024-04-25T00:22:48.412 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.871+0000 7f30eb5f9700 1 --2- 172.21.15.32:0/2343428638 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f30e407de50 0x7f30e419d470 unknown :-1 s=CLOSED pgs=4 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:22:48.412 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.871+0000 7f30eb5f9700 1 -- 172.21.15.32:0/2343428638 >> 172.21.15.32:0/2343428638 conn(0x7f30e4103170 msgr2=0x7f30e41054b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:22:48.412 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.871+0000 7f30eb5f9700 1 -- 172.21.15.32:0/2343428638 shutdown_connections 2024-04-25T00:22:48.413 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:47.871+0000 7f30eb5f9700 1 -- 172.21.15.32:0/2343428638 wait complete. 2024-04-25T00:22:48.413 INFO:teuthology.orchestra.run.smithi032.stdout:Generating new minimal ceph.conf... 2024-04-25T00:22:49.572 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.803+0000 7f9fbbbae700 1 Processor -- start 2024-04-25T00:22:49.572 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.803+0000 7f9fbbbae700 1 -- start start 2024-04-25T00:22:49.572 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.804+0000 7f9fbbbae700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fb407de90 0x7f9fb407c340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:22:49.572 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.804+0000 7f9fbbbae700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f9fb407c910 con 0x7f9fb407de90 2024-04-25T00:22:49.573 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.804+0000 7f9fb994a700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fb407de90 0x7f9fb407c340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:22:49.573 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.804+0000 7f9fb994a700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fb407de90 0x7f9fb407c340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:43574/0 (socket says 172.21.15.32:43574) 2024-04-25T00:22:49.573 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.804+0000 7f9fb994a700 1 -- 172.21.15.32:0/2132145474 learned_addr learned my addr 172.21.15.32:0/2132145474 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:22:49.573 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.804+0000 7f9fb994a700 1 -- 172.21.15.32:0/2132145474 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9fb407ca90 con 0x7f9fb407de90 2024-04-25T00:22:49.573 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.804+0000 7f9fb994a700 1 --2- 172.21.15.32:0/2132145474 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fb407de90 0x7f9fb407c340 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7f9fa80098a0 tx=0x7f9fa8009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4963806fa17d809b server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:22:49.573 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.804+0000 7f9fb8948700 1 -- 172.21.15.32:0/2132145474 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f9fa8004160 con 0x7f9fb407de90 2024-04-25T00:22:49.573 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.804+0000 7f9fb8948700 1 -- 172.21.15.32:0/2132145474 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f9fa8004300 con 0x7f9fb407de90 2024-04-25T00:22:49.573 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.804+0000 7f9fb8948700 1 -- 172.21.15.32:0/2132145474 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f9fa8011b50 con 0x7f9fb407de90 2024-04-25T00:22:49.573 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.805+0000 7f9fbbbae700 1 -- 172.21.15.32:0/2132145474 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fb407de90 msgr2=0x7f9fb407c340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:22:49.573 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.805+0000 7f9fbbbae700 1 --2- 172.21.15.32:0/2132145474 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fb407de90 0x7f9fb407c340 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7f9fa80098a0 tx=0x7f9fa8009bb0 comp rx=0 tx=0).stop 2024-04-25T00:22:49.573 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.805+0000 7f9fbbbae700 1 -- 172.21.15.32:0/2132145474 shutdown_connections 2024-04-25T00:22:49.573 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.805+0000 7f9fbbbae700 1 --2- 172.21.15.32:0/2132145474 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fb407de90 0x7f9fb407c340 unknown :-1 s=CLOSED pgs=5 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:22:49.574 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.805+0000 7f9fbbbae700 1 -- 172.21.15.32:0/2132145474 >> 172.21.15.32:0/2132145474 conn(0x7f9fb4103150 msgr2=0x7f9fb41055b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:22:49.574 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.805+0000 7f9fbbbae700 1 -- 172.21.15.32:0/2132145474 shutdown_connections 2024-04-25T00:22:49.574 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.805+0000 7f9fbbbae700 1 -- 172.21.15.32:0/2132145474 wait complete. 2024-04-25T00:22:49.574 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.806+0000 7f9fbbbae700 1 Processor -- start 2024-04-25T00:22:49.574 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.806+0000 7f9fbbbae700 1 -- start start 2024-04-25T00:22:49.574 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.806+0000 7f9fbbbae700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fb407de90 0x7f9fb419dd40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:22:49.574 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.806+0000 7f9fbbbae700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f9fb410e160 con 0x7f9fb407de90 2024-04-25T00:22:49.574 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.806+0000 7f9fb994a700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fb407de90 0x7f9fb419dd40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:22:49.574 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.806+0000 7f9fb994a700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fb407de90 0x7f9fb419dd40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:43582/0 (socket says 172.21.15.32:43582) 2024-04-25T00:22:49.574 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.806+0000 7f9fb994a700 1 -- 172.21.15.32:0/2871703604 learned_addr learned my addr 172.21.15.32:0/2871703604 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:22:49.574 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.806+0000 7f9fb994a700 1 -- 172.21.15.32:0/2871703604 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9fb419e280 con 0x7f9fb407de90 2024-04-25T00:22:49.574 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.806+0000 7f9fb994a700 1 --2- 172.21.15.32:0/2871703604 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fb407de90 0x7f9fb419dd40 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7f9fa8011b60 tx=0x7f9fa8011c40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:22:49.575 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.807+0000 7f9fa6ffd700 1 -- 172.21.15.32:0/2871703604 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f9fa8003b20 con 0x7f9fb407de90 2024-04-25T00:22:49.575 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.807+0000 7f9fa6ffd700 1 -- 172.21.15.32:0/2871703604 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f9fa8003cc0 con 0x7f9fb407de90 2024-04-25T00:22:49.575 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.807+0000 7f9fbbbae700 1 -- 172.21.15.32:0/2871703604 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9fb419e510 con 0x7f9fb407de90 2024-04-25T00:22:49.575 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.807+0000 7f9fa6ffd700 1 -- 172.21.15.32:0/2871703604 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f9fa8024840 con 0x7f9fb407de90 2024-04-25T00:22:49.575 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.807+0000 7f9fbbbae700 1 -- 172.21.15.32:0/2871703604 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9fb41a1200 con 0x7f9fb407de90 2024-04-25T00:22:49.575 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.807+0000 7f9fa6ffd700 1 -- 172.21.15.32:0/2871703604 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f9fa8022020 con 0x7f9fb407de90 2024-04-25T00:22:49.575 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.807+0000 7f9fa6ffd700 1 -- 172.21.15.32:0/2871703604 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f9fa8011e00 con 0x7f9fb407de90 2024-04-25T00:22:49.575 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.808+0000 7f9fbbbae700 1 -- 172.21.15.32:0/2871703604 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9fb4063890 con 0x7f9fb407de90 2024-04-25T00:22:49.575 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.810+0000 7f9fa6ffd700 1 -- 172.21.15.32:0/2871703604 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+73141 (secure 0 0 0) 0x7f9fa802d3f0 con 0x7f9fb407de90 2024-04-25T00:22:49.575 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.880+0000 7f9fbbbae700 1 -- 172.21.15.32:0/2871703604 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "config generate-minimal-conf"} v 0) -- 0x7f9fb41a13d0 con 0x7f9fb407de90 2024-04-25T00:22:49.575 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.880+0000 7f9fa6ffd700 1 -- 172.21.15.32:0/2871703604 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "config generate-minimal-conf"}]=0 v2) ==== 76+0+175 (secure 0 0 0) 0x7f9fa8032030 con 0x7f9fb407de90 2024-04-25T00:22:49.575 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.881+0000 7f9fbbbae700 1 -- 172.21.15.32:0/2871703604 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fb407de90 msgr2=0x7f9fb419dd40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:22:49.575 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.881+0000 7f9fbbbae700 1 --2- 172.21.15.32:0/2871703604 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fb407de90 0x7f9fb419dd40 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7f9fa8011b60 tx=0x7f9fa8011c40 comp rx=0 tx=0).stop 2024-04-25T00:22:49.575 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.881+0000 7f9fbbbae700 1 -- 172.21.15.32:0/2871703604 shutdown_connections 2024-04-25T00:22:49.575 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.881+0000 7f9fbbbae700 1 --2- 172.21.15.32:0/2871703604 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fb407de90 0x7f9fb419dd40 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:22:49.576 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.881+0000 7f9fbbbae700 1 -- 172.21.15.32:0/2871703604 >> 172.21.15.32:0/2871703604 conn(0x7f9fb4103150 msgr2=0x7f9fb4103d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:22:49.576 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.881+0000 7f9fbbbae700 1 -- 172.21.15.32:0/2871703604 shutdown_connections 2024-04-25T00:22:49.576 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:48.881+0000 7f9fbbbae700 1 -- 172.21.15.32:0/2871703604 wait complete. 2024-04-25T00:22:49.576 INFO:teuthology.orchestra.run.smithi032.stdout:Restarting the monitor... 2024-04-25T00:22:49.826 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:49 smithi032 systemd[1]: Stopping Ceph mon.a for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:22:50.101 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:49 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-a[28979]: 2024-04-25T00:22:49.824+0000 7f4eed84f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:22:50.101 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:49 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-a[28979]: 2024-04-25T00:22:49.824+0000 7f4eed84f700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-04-25T00:22:50.101 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:49 smithi032 podman[29231]: 2024-04-25 00:22:49.887656182 +0000 UTC m=+0.143809618 container died 5181fda467e1791b9e68a2435e5ab8af1daf9d957035f72d03c244a0395852c2 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-a, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=squid-d2c45d9) 2024-04-25T00:22:50.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:50 smithi032 podman[29231]: 2024-04-25 00:22:50.200034791 +0000 UTC m=+0.456188216 container cleanup 5181fda467e1791b9e68a2435e5ab8af1daf9d957035f72d03c244a0395852c2 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-a, RELEASE=squid-d2c45d9, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-04-25T00:22:50.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:50 smithi032 bash[29231]: ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-a 2024-04-25T00:22:50.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:50 smithi032 podman[29243]: 2024-04-25 00:22:50.334146691 +0000 UTC m=+0.444632181 container remove 5181fda467e1791b9e68a2435e5ab8af1daf9d957035f72d03c244a0395852c2 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-a, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-d2c45d9, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-04-25T00:22:50.892 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:50 smithi032 systemd[1]: ceph-cb997156-0299-11ef-bc93-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-25T00:22:50.893 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:50 smithi032 systemd[1]: Stopped Ceph mon.a for cb997156-0299-11ef-bc93-c7b262605968. 2024-04-25T00:22:50.893 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:50 smithi032 systemd[1]: Starting Ceph mon.a for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:22:50.893 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:50 smithi032 podman[29339]: 2024-04-25 00:22:50.655622689 +0000 UTC m=+0.018280660 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 2024-04-25T00:22:50.893 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:50 smithi032 podman[29339]: 2024-04-25 00:22:50.784906633 +0000 UTC m=+0.147564608 container create 6ab91bb240b189eb93adc0fa53b419bd511bfa26b3a62228a47ce326d35bc156 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-a, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-d2c45d9, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-25T00:22:51.108 INFO:teuthology.orchestra.run.smithi032.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-04-25T00:22:51.222 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 podman[29339]: 2024-04-25 00:22:51.020408745 +0000 UTC m=+0.383066714 container init 6ab91bb240b189eb93adc0fa53b419bd511bfa26b3a62228a47ce326d35bc156 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-a, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-d2c45d9, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T00:22:51.223 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 podman[29339]: 2024-04-25 00:22:51.023585874 +0000 UTC m=+0.386243843 container start 6ab91bb240b189eb93adc0fa53b419bd511bfa26b3a62228a47ce326d35bc156 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-a, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=squid-d2c45d9, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-25T00:22:51.223 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: set uid:gid to 167:167 (ceph:ceph) 2024-04-25T00:22:51.223 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: ceph version 19.0.0-2445-gd2c45d97 (d2c45d977142e024a59b68a8f1dce8f0b9840551) squid (dev), process ceph-mon, pid 2 2024-04-25T00:22:51.223 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: pidfile_write: ignore empty --pid-file 2024-04-25T00:22:51.223 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: load: jerasure load: lrc 2024-04-25T00:22:51.223 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: RocksDB version: 7.9.2 2024-04-25T00:22:51.223 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Git sha 0 2024-04-25T00:22:51.223 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Compile date 2024-04-24 16:41:23 2024-04-25T00:22:51.223 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: DB SUMMARY 2024-04-25T00:22:51.223 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: DB Session ID: A840WNQEIH43JYO5OLFB 2024-04-25T00:22:51.223 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: CURRENT file: CURRENT 2024-04-25T00:22:51.223 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: IDENTITY file: IDENTITY 2024-04-25T00:22:51.223 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: MANIFEST file: MANIFEST-000010 size: 179 Bytes 2024-04-25T00:22:51.224 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-04-25T00:22:51.224 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000009.log size: 104160 ; 2024-04-25T00:22:51.224 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.error_if_exists: 0 2024-04-25T00:22:51.224 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.create_if_missing: 0 2024-04-25T00:22:51.224 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.paranoid_checks: 1 2024-04-25T00:22:51.224 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-25T00:22:51.224 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-25T00:22:51.224 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-25T00:22:51.224 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.env: 0x55c05a40ac60 2024-04-25T00:22:51.224 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.fs: PosixFileSystem 2024-04-25T00:22:51.224 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.info_log: 0x55c05b8d7aa0 2024-04-25T00:22:51.224 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_file_opening_threads: 16 2024-04-25T00:22:51.224 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.statistics: (nil) 2024-04-25T00:22:51.224 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.use_fsync: 0 2024-04-25T00:22:51.225 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_log_file_size: 0 2024-04-25T00:22:51.225 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-25T00:22:51.225 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-25T00:22:51.225 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.keep_log_file_num: 1000 2024-04-25T00:22:51.225 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.recycle_log_file_num: 0 2024-04-25T00:22:51.225 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.allow_fallocate: 1 2024-04-25T00:22:51.225 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.allow_mmap_reads: 0 2024-04-25T00:22:51.225 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.allow_mmap_writes: 0 2024-04-25T00:22:51.225 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.use_direct_reads: 0 2024-04-25T00:22:51.225 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-25T00:22:51.225 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.create_missing_column_families: 0 2024-04-25T00:22:51.225 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.db_log_dir: 2024-04-25T00:22:51.225 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.wal_dir: 2024-04-25T00:22:51.225 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-25T00:22:51.225 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-25T00:22:51.226 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-25T00:22:51.226 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-25T00:22:51.226 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-25T00:22:51.226 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-25T00:22:51.226 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.advise_random_on_open: 1 2024-04-25T00:22:51.226 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.db_write_buffer_size: 0 2024-04-25T00:22:51.226 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.write_buffer_manager: 0x55c05b8aa1e0 2024-04-25T00:22:51.226 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-25T00:22:51.226 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-25T00:22:51.227 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-25T00:22:51.227 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.rate_limiter: (nil) 2024-04-25T00:22:51.227 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-25T00:22:51.227 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.wal_recovery_mode: 2 2024-04-25T00:22:51.227 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.enable_thread_tracking: 0 2024-04-25T00:22:51.227 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.enable_pipelined_write: 0 2024-04-25T00:22:51.227 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.unordered_write: 0 2024-04-25T00:22:51.227 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-25T00:22:51.227 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-25T00:22:51.227 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-25T00:22:51.227 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-25T00:22:51.228 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.row_cache: None 2024-04-25T00:22:51.228 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.wal_filter: None 2024-04-25T00:22:51.228 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-25T00:22:51.229 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.allow_ingest_behind: 0 2024-04-25T00:22:51.229 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.two_write_queues: 0 2024-04-25T00:22:51.229 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.manual_wal_flush: 0 2024-04-25T00:22:51.229 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.wal_compression: 0 2024-04-25T00:22:51.229 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.atomic_flush: 0 2024-04-25T00:22:51.230 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-25T00:22:51.230 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-25T00:22:51.230 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-25T00:22:51.230 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.log_readahead_size: 0 2024-04-25T00:22:51.230 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-25T00:22:51.230 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.best_efforts_recovery: 0 2024-04-25T00:22:51.230 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-25T00:22:51.230 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-25T00:22:51.230 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.allow_data_in_errors: 0 2024-04-25T00:22:51.230 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.db_host_id: __hostname__ 2024-04-25T00:22:51.230 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-25T00:22:51.230 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_background_jobs: 2 2024-04-25T00:22:51.230 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_background_compactions: -1 2024-04-25T00:22:51.231 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_subcompactions: 1 2024-04-25T00:22:51.231 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-25T00:22:51.231 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-25T00:22:51.231 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-25T00:22:51.231 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_total_wal_size: 0 2024-04-25T00:22:51.231 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-25T00:22:51.231 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-25T00:22:51.231 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-25T00:22:51.231 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-25T00:22:51.231 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_open_files: -1 2024-04-25T00:22:51.231 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.bytes_per_sync: 0 2024-04-25T00:22:51.231 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-25T00:22:51.231 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-25T00:22:51.231 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.compaction_readahead_size: 0 2024-04-25T00:22:51.231 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_background_flushes: -1 2024-04-25T00:22:51.232 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Compression algorithms supported: 2024-04-25T00:22:51.232 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-25T00:22:51.232 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: kZSTD supported: 0 2024-04-25T00:22:51.232 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: kXpressCompression supported: 0 2024-04-25T00:22:51.232 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: kLZ4HCCompression supported: 1 2024-04-25T00:22:51.232 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: kZlibCompression supported: 1 2024-04-25T00:22:51.232 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: kSnappyCompression supported: 1 2024-04-25T00:22:51.232 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: kLZ4Compression supported: 1 2024-04-25T00:22:51.232 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: kBZip2Compression supported: 0 2024-04-25T00:22:51.232 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-25T00:22:51.232 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-25T00:22:51.233 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 2024-04-25T00:22:51.233 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-25T00:22:51.233 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-25T00:22:51.233 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.merge_operator: 2024-04-25T00:22:51.233 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.compaction_filter: None 2024-04-25T00:22:51.233 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.compaction_filter_factory: None 2024-04-25T00:22:51.233 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.sst_partitioner_factory: None 2024-04-25T00:22:51.233 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-25T00:22:51.233 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-25T00:22:51.233 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c05b8d7ba0) 2024-04-25T00:22:51.233 INFO:journalctl@ceph.mon.a.smithi032.stdout: cache_index_and_filter_blocks: 1 2024-04-25T00:22:51.233 INFO:journalctl@ceph.mon.a.smithi032.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-25T00:22:51.234 INFO:journalctl@ceph.mon.a.smithi032.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-25T00:22:51.234 INFO:journalctl@ceph.mon.a.smithi032.stdout: pin_top_level_index_and_filter: 1 2024-04-25T00:22:51.234 INFO:journalctl@ceph.mon.a.smithi032.stdout: index_type: 0 2024-04-25T00:22:51.234 INFO:journalctl@ceph.mon.a.smithi032.stdout: data_block_index_type: 0 2024-04-25T00:22:51.234 INFO:journalctl@ceph.mon.a.smithi032.stdout: index_shortening: 1 2024-04-25T00:22:51.234 INFO:journalctl@ceph.mon.a.smithi032.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-25T00:22:51.234 INFO:journalctl@ceph.mon.a.smithi032.stdout: checksum: 4 2024-04-25T00:22:51.234 INFO:journalctl@ceph.mon.a.smithi032.stdout: no_block_cache: 0 2024-04-25T00:22:51.234 INFO:journalctl@ceph.mon.a.smithi032.stdout: block_cache: 0x55c05b8db090 2024-04-25T00:22:51.234 INFO:journalctl@ceph.mon.a.smithi032.stdout: block_cache_name: BinnedLRUCache 2024-04-25T00:22:51.234 INFO:journalctl@ceph.mon.a.smithi032.stdout: block_cache_options: 2024-04-25T00:22:51.234 INFO:journalctl@ceph.mon.a.smithi032.stdout: capacity : 536870912 2024-04-25T00:22:51.234 INFO:journalctl@ceph.mon.a.smithi032.stdout: num_shard_bits : 4 2024-04-25T00:22:51.234 INFO:journalctl@ceph.mon.a.smithi032.stdout: strict_capacity_limit : 0 2024-04-25T00:22:51.235 INFO:journalctl@ceph.mon.a.smithi032.stdout: high_pri_pool_ratio: 0.000 2024-04-25T00:22:51.235 INFO:journalctl@ceph.mon.a.smithi032.stdout: block_cache_compressed: (nil) 2024-04-25T00:22:51.235 INFO:journalctl@ceph.mon.a.smithi032.stdout: persistent_cache: (nil) 2024-04-25T00:22:51.235 INFO:journalctl@ceph.mon.a.smithi032.stdout: block_size: 4096 2024-04-25T00:22:51.235 INFO:journalctl@ceph.mon.a.smithi032.stdout: block_size_deviation: 10 2024-04-25T00:22:51.235 INFO:journalctl@ceph.mon.a.smithi032.stdout: block_restart_interval: 16 2024-04-25T00:22:51.235 INFO:journalctl@ceph.mon.a.smithi032.stdout: index_block_restart_interval: 1 2024-04-25T00:22:51.235 INFO:journalctl@ceph.mon.a.smithi032.stdout: metadata_block_size: 4096 2024-04-25T00:22:51.235 INFO:journalctl@ceph.mon.a.smithi032.stdout: partition_filters: 0 2024-04-25T00:22:51.235 INFO:journalctl@ceph.mon.a.smithi032.stdout: use_delta_encoding: 1 2024-04-25T00:22:51.235 INFO:journalctl@ceph.mon.a.smithi032.stdout: filter_policy: bloomfilter 2024-04-25T00:22:51.235 INFO:journalctl@ceph.mon.a.smithi032.stdout: whole_key_filtering: 1 2024-04-25T00:22:51.235 INFO:journalctl@ceph.mon.a.smithi032.stdout: verify_compression: 0 2024-04-25T00:22:51.235 INFO:journalctl@ceph.mon.a.smithi032.stdout: read_amp_bytes_per_bit: 0 2024-04-25T00:22:51.235 INFO:journalctl@ceph.mon.a.smithi032.stdout: format_version: 5 2024-04-25T00:22:51.235 INFO:journalctl@ceph.mon.a.smithi032.stdout: enable_index_compression: 1 2024-04-25T00:22:51.235 INFO:journalctl@ceph.mon.a.smithi032.stdout: block_align: 0 2024-04-25T00:22:51.236 INFO:journalctl@ceph.mon.a.smithi032.stdout: max_auto_readahead_size: 262144 2024-04-25T00:22:51.236 INFO:journalctl@ceph.mon.a.smithi032.stdout: prepopulate_block_cache: 0 2024-04-25T00:22:51.236 INFO:journalctl@ceph.mon.a.smithi032.stdout: initial_auto_readahead_size: 8192 2024-04-25T00:22:51.236 INFO:journalctl@ceph.mon.a.smithi032.stdout: num_file_reads_for_auto_readahead: 2 2024-04-25T00:22:51.236 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.write_buffer_size: 33554432 2024-04-25T00:22:51.236 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_write_buffer_number: 2 2024-04-25T00:22:51.236 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.compression: NoCompression 2024-04-25T00:22:51.236 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.bottommost_compression: Disabled 2024-04-25T00:22:51.236 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.prefix_extractor: nullptr 2024-04-25T00:22:51.236 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-25T00:22:51.236 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.num_levels: 7 2024-04-25T00:22:51.236 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-25T00:22:51.236 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-25T00:22:51.236 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-25T00:22:51.236 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-25T00:22:51.237 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-25T00:22:51.237 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-25T00:22:51.237 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-25T00:22:51.237 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-25T00:22:51.237 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-25T00:22:51.237 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-25T00:22:51.237 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-25T00:22:51.237 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-25T00:22:51.238 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-25T00:22:51.238 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.compression_opts.level: 32767 2024-04-25T00:22:51.238 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.compression_opts.strategy: 0 2024-04-25T00:22:51.238 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-25T00:22:51.238 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-25T00:22:51.238 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-25T00:22:51.238 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-25T00:22:51.238 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.compression_opts.enabled: false 2024-04-25T00:22:51.238 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-25T00:22:51.238 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-25T00:22:51.238 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-25T00:22:51.238 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-25T00:22:51.238 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.target_file_size_base: 67108864 2024-04-25T00:22:51.239 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-25T00:22:51.239 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-25T00:22:51.239 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-25T00:22:51.239 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-25T00:22:51.239 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-25T00:22:51.239 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-25T00:22:51.239 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-25T00:22:51.239 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-25T00:22:51.239 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-25T00:22:51.239 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-25T00:22:51.239 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-25T00:22:51.239 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-25T00:22:51.240 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-25T00:22:51.240 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-25T00:22:51.240 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.arena_block_size: 1048576 2024-04-25T00:22:51.240 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-25T00:22:51.240 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-25T00:22:51.240 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.disable_auto_compactions: 0 2024-04-25T00:22:51.240 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-25T00:22:51.240 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-25T00:22:51.240 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-25T00:22:51.240 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-25T00:22:51.240 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-25T00:22:51.240 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-25T00:22:51.240 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-25T00:22:51.240 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-25T00:22:51.241 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-25T00:22:51.241 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-25T00:22:51.241 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.table_properties_collectors: 2024-04-25T00:22:51.241 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.inplace_update_support: 0 2024-04-25T00:22:51.241 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-25T00:22:51.241 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-25T00:22:51.241 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-25T00:22:51.241 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-25T00:22:51.241 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.bloom_locality: 0 2024-04-25T00:22:51.241 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.max_successive_merges: 0 2024-04-25T00:22:51.241 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-25T00:22:51.241 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.paranoid_file_checks: 0 2024-04-25T00:22:51.241 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.force_consistency_checks: 1 2024-04-25T00:22:51.241 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.report_bg_io_stats: 0 2024-04-25T00:22:51.241 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.ttl: 2592000 2024-04-25T00:22:51.242 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-25T00:22:51.242 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-25T00:22:51.242 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-25T00:22:51.242 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.enable_blob_files: false 2024-04-25T00:22:51.242 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.min_blob_size: 0 2024-04-25T00:22:51.242 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.blob_file_size: 268435456 2024-04-25T00:22:51.242 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-25T00:22:51.242 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-25T00:22:51.243 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-25T00:22:51.243 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-25T00:22:51.243 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-25T00:22:51.243 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.blob_file_starting_level: 0 2024-04-25T00:22:51.243 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-25T00:22:51.243 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 12, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 5 2024-04-25T00:22:51.243 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 5 2024-04-25T00:22:51.243 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 7c4c9001-311d-4641-8e50-5d6f3a7ae219 2024-04-25T00:22:51.243 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714004571053820, "job": 1, "event": "recovery_started", "wal_files": [9]} 2024-04-25T00:22:51.243 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #9 mode 2 2024-04-25T00:22:51.244 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714004571054918, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 13, "file_size": 99365, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 8, "largest_seqno": 324, "table_properties": {"data_size": 97285, "index_size": 298, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 773, "raw_key_size": 14034, "raw_average_key_size": 49, "raw_value_size": 89621, "raw_average_value_size": 316, "num_data_blocks": 12, "num_entries": 283, "num_filter_entries": 283, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714004571, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "7c4c9001-311d-4641-8e50-5d6f3a7ae219", "db_session_id": "A840WNQEIH43JYO5OLFB", "orig_file_number": 13, "seqno_to_time_mapping": "N/A"}} 2024-04-25T00:22:51.244 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714004571055003, "job": 1, "event": "recovery_finished"} 2024-04-25T00:22:51.244 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: [db/version_set.cc:5047] Creating manifest 15 2024-04-25T00:22:51.244 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000009.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-25T00:22:51.244 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55c05b9cc000 2024-04-25T00:22:51.244 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: DB pointer 0x55c05b9b6000 2024-04-25T00:22:51.244 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-25T00:22:51.244 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-25T00:22:51.244 INFO:journalctl@ceph.mon.a.smithi032.stdout: ** DB Stats ** 2024-04-25T00:22:51.244 INFO:journalctl@ceph.mon.a.smithi032.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-25T00:22:51.244 INFO:journalctl@ceph.mon.a.smithi032.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-25T00:22:51.244 INFO:journalctl@ceph.mon.a.smithi032.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-25T00:22:51.244 INFO:journalctl@ceph.mon.a.smithi032.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-25T00:22:51.244 INFO:journalctl@ceph.mon.a.smithi032.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-25T00:22:51.245 INFO:journalctl@ceph.mon.a.smithi032.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-25T00:22:51.245 INFO:journalctl@ceph.mon.a.smithi032.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-25T00:22:51.245 INFO:journalctl@ceph.mon.a.smithi032.stdout: 2024-04-25T00:22:51.245 INFO:journalctl@ceph.mon.a.smithi032.stdout: ** Compaction Stats [default] ** 2024-04-25T00:22:51.245 INFO:journalctl@ceph.mon.a.smithi032.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-25T00:22:51.245 INFO:journalctl@ceph.mon.a.smithi032.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-25T00:22:51.245 INFO:journalctl@ceph.mon.a.smithi032.stdout: L0 2/0 98.87 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.7 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-25T00:22:51.245 INFO:journalctl@ceph.mon.a.smithi032.stdout: Sum 2/0 98.87 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.7 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-25T00:22:51.245 INFO:journalctl@ceph.mon.a.smithi032.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.7 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-25T00:22:51.245 INFO:journalctl@ceph.mon.a.smithi032.stdout: 2024-04-25T00:22:51.245 INFO:journalctl@ceph.mon.a.smithi032.stdout: ** Compaction Stats [default] ** 2024-04-25T00:22:51.245 INFO:journalctl@ceph.mon.a.smithi032.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-25T00:22:51.245 INFO:journalctl@ceph.mon.a.smithi032.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-25T00:22:51.245 INFO:journalctl@ceph.mon.a.smithi032.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 137.7 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-25T00:22:51.245 INFO:journalctl@ceph.mon.a.smithi032.stdout: 2024-04-25T00:22:51.246 INFO:journalctl@ceph.mon.a.smithi032.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-25T00:22:51.246 INFO:journalctl@ceph.mon.a.smithi032.stdout: 2024-04-25T00:22:51.246 INFO:journalctl@ceph.mon.a.smithi032.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-25T00:22:51.246 INFO:journalctl@ceph.mon.a.smithi032.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-25T00:22:51.246 INFO:journalctl@ceph.mon.a.smithi032.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-25T00:22:51.246 INFO:journalctl@ceph.mon.a.smithi032.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-25T00:22:51.246 INFO:journalctl@ceph.mon.a.smithi032.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-25T00:22:51.246 INFO:journalctl@ceph.mon.a.smithi032.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-25T00:22:51.246 INFO:journalctl@ceph.mon.a.smithi032.stdout: Cumulative compaction: 0.00 GB write, 30.10 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-25T00:22:51.246 INFO:journalctl@ceph.mon.a.smithi032.stdout: Interval compaction: 0.00 GB write, 30.10 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-25T00:22:51.246 INFO:journalctl@ceph.mon.a.smithi032.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-25T00:22:51.246 INFO:journalctl@ceph.mon.a.smithi032.stdout: Block cache BinnedLRUCache@0x55c05b8db090#2 capacity: 512.00 MB usage: 1.52 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 8e-06 secs_since: 0 2024-04-25T00:22:51.246 INFO:journalctl@ceph.mon.a.smithi032.stdout: Block cache entry stats(count,size,portion): FilterBlock(2,1.02 KB,0.000193715%) IndexBlock(2,0.50 KB,9.53674e-05%) Misc(1,0.00 KB,0%) 2024-04-25T00:22:51.246 INFO:journalctl@ceph.mon.a.smithi032.stdout: 2024-04-25T00:22:51.246 INFO:journalctl@ceph.mon.a.smithi032.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-25T00:22:51.247 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: starting mon.a rank 0 at public addrs [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] at bind addrs [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:22:51.247 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: mon.a@-1(???) e1 preinit fsid cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:22:51.247 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: mon.a@-1(???).mds e1 new map 2024-04-25T00:22:51.247 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: mon.a@-1(???).mds e1 print_map 2024-04-25T00:22:51.247 INFO:journalctl@ceph.mon.a.smithi032.stdout: e1 2024-04-25T00:22:51.247 INFO:journalctl@ceph.mon.a.smithi032.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-25T00:22:51.247 INFO:journalctl@ceph.mon.a.smithi032.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2,11=minor log segments,12=quiesce subvolumes} 2024-04-25T00:22:51.247 INFO:journalctl@ceph.mon.a.smithi032.stdout: legacy client fscid: -1 2024-04-25T00:22:51.247 INFO:journalctl@ceph.mon.a.smithi032.stdout: 2024-04-25T00:22:51.247 INFO:journalctl@ceph.mon.a.smithi032.stdout: No filesystems configured 2024-04-25T00:22:51.247 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-25T00:22:51.247 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-25T00:22:51.247 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-25T00:22:51.247 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-25T00:22:51.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 ceph-mon[29357]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-25T00:22:51.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 bash[29339]: 6ab91bb240b189eb93adc0fa53b419bd511bfa26b3a62228a47ce326d35bc156 2024-04-25T00:22:51.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:51 smithi032 systemd[1]: Started Ceph mon.a for cb997156-0299-11ef-bc93-c7b262605968. 2024-04-25T00:22:52.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:52 smithi032 ceph-mon[29357]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-25T00:22:52.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:52 smithi032 ceph-mon[29357]: monmap epoch 1 2024-04-25T00:22:52.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:52 smithi032 ceph-mon[29357]: fsid cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:22:52.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:52 smithi032 ceph-mon[29357]: last_changed 2024-04-25T00:22:42.756376+0000 2024-04-25T00:22:52.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:52 smithi032 ceph-mon[29357]: created 2024-04-25T00:22:42.756376+0000 2024-04-25T00:22:52.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:52 smithi032 ceph-mon[29357]: min_mon_release 19 (squid) 2024-04-25T00:22:52.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:52 smithi032 ceph-mon[29357]: election_strategy: 1 2024-04-25T00:22:52.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:52 smithi032 ceph-mon[29357]: 0: [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] mon.a 2024-04-25T00:22:52.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:52 smithi032 ceph-mon[29357]: fsmap 2024-04-25T00:22:52.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:52 smithi032 ceph-mon[29357]: osdmap e1: 0 total, 0 up, 0 in 2024-04-25T00:22:52.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:52 smithi032 ceph-mon[29357]: mgrmap e1: no daemons active 2024-04-25T00:22:52.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:52 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3492017373' entity='client.admin' 2024-04-25T00:22:52.284 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.538+0000 7f0481486700 1 Processor -- start 2024-04-25T00:22:52.285 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.538+0000 7f0481486700 1 -- start start 2024-04-25T00:22:52.285 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.538+0000 7f0481486700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f047c108620 0x7f047c10aa10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:22:52.285 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.538+0000 7f0481486700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f047c07bed0 con 0x7f047c108620 2024-04-25T00:22:52.285 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.539+0000 7f047affd700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f047c108620 0x7f047c10aa10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:22:52.285 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.539+0000 7f047affd700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f047c108620 0x7f047c10aa10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:43588/0 (socket says 172.21.15.32:43588) 2024-04-25T00:22:52.285 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.539+0000 7f047affd700 1 -- 172.21.15.32:0/1932920103 learned_addr learned my addr 172.21.15.32:0/1932920103 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:22:52.285 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.539+0000 7f047affd700 1 -- 172.21.15.32:0/1932920103 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f047c10af50 con 0x7f047c108620 2024-04-25T00:22:52.285 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.539+0000 7f047affd700 1 --2- 172.21.15.32:0/1932920103 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f047c108620 0x7f047c10aa10 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f046c0098a0 tx=0x7f046c009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3dc233efbab79606 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:22:52.285 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.539+0000 7f0479ffb700 1 -- 172.21.15.32:0/1932920103 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f046c004160 con 0x7f047c108620 2024-04-25T00:22:52.285 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.539+0000 7f0479ffb700 1 -- 172.21.15.32:0/1932920103 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f046c004300 con 0x7f047c108620 2024-04-25T00:22:52.285 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.539+0000 7f0479ffb700 1 -- 172.21.15.32:0/1932920103 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f046c011b50 con 0x7f047c108620 2024-04-25T00:22:52.285 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.540+0000 7f0481486700 1 -- 172.21.15.32:0/1932920103 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f047c108620 msgr2=0x7f047c10aa10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:22:52.285 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.540+0000 7f0481486700 1 --2- 172.21.15.32:0/1932920103 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f047c108620 0x7f047c10aa10 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f046c0098a0 tx=0x7f046c009bb0 comp rx=0 tx=0).stop 2024-04-25T00:22:52.285 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.540+0000 7f0481486700 1 -- 172.21.15.32:0/1932920103 shutdown_connections 2024-04-25T00:22:52.286 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.540+0000 7f0481486700 1 --2- 172.21.15.32:0/1932920103 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f047c108620 0x7f047c10aa10 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:22:52.286 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.540+0000 7f0481486700 1 -- 172.21.15.32:0/1932920103 >> 172.21.15.32:0/1932920103 conn(0x7f047c102070 msgr2=0x7f047c104490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:22:52.286 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.540+0000 7f0481486700 1 -- 172.21.15.32:0/1932920103 shutdown_connections 2024-04-25T00:22:52.286 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.540+0000 7f0481486700 1 -- 172.21.15.32:0/1932920103 wait complete. 2024-04-25T00:22:52.286 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.541+0000 7f0481486700 1 Processor -- start 2024-04-25T00:22:52.286 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.541+0000 7f0481486700 1 -- start start 2024-04-25T00:22:52.286 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.541+0000 7f0481486700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f047c108620 0x7f047c107ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:22:52.286 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.541+0000 7f0481486700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f047c10bf80 con 0x7f047c108620 2024-04-25T00:22:52.286 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.541+0000 7f047affd700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f047c108620 0x7f047c107ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:22:52.286 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.541+0000 7f047affd700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f047c108620 0x7f047c107ea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:43590/0 (socket says 172.21.15.32:43590) 2024-04-25T00:22:52.286 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.541+0000 7f047affd700 1 -- 172.21.15.32:0/3492017373 learned_addr learned my addr 172.21.15.32:0/3492017373 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:22:52.286 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.541+0000 7f047affd700 1 -- 172.21.15.32:0/3492017373 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f047c1083e0 con 0x7f047c108620 2024-04-25T00:22:52.286 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.542+0000 7f047affd700 1 --2- 172.21.15.32:0/3492017373 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f047c108620 0x7f047c107ea0 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7f046c011670 tx=0x7f046c011b60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:22:52.287 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.542+0000 7f0463fff700 1 -- 172.21.15.32:0/3492017373 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f046c0039c0 con 0x7f047c108620 2024-04-25T00:22:52.287 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.542+0000 7f0463fff700 1 -- 172.21.15.32:0/3492017373 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f046c011e50 con 0x7f047c108620 2024-04-25T00:22:52.287 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.542+0000 7f0463fff700 1 -- 172.21.15.32:0/3492017373 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f046c01a5c0 con 0x7f047c108620 2024-04-25T00:22:52.287 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.542+0000 7f0481486700 1 -- 172.21.15.32:0/3492017373 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f047c1065d0 con 0x7f047c108620 2024-04-25T00:22:52.287 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.542+0000 7f0481486700 1 -- 172.21.15.32:0/3492017373 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f047c106b10 con 0x7f047c108620 2024-04-25T00:22:52.287 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.542+0000 7f0463fff700 1 -- 172.21.15.32:0/3492017373 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f046c01ac00 con 0x7f047c108620 2024-04-25T00:22:52.287 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.542+0000 7f0463fff700 1 -- 172.21.15.32:0/3492017373 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f046c019700 con 0x7f047c108620 2024-04-25T00:22:52.287 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.543+0000 7f0481486700 1 -- 172.21.15.32:0/3492017373 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f047c063890 con 0x7f047c108620 2024-04-25T00:22:52.287 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.545+0000 7f0463fff700 1 -- 172.21.15.32:0/3492017373 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+73141 (secure 0 0 0) 0x7f046c003b60 con 0x7f047c108620 2024-04-25T00:22:52.287 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.615+0000 7f0481486700 1 -- 172.21.15.32:0/3492017373 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command([{prefix=config set, name=public_network}] v 0) -- 0x7f047c107040 con 0x7f047c108620 2024-04-25T00:22:52.287 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.617+0000 7f0463fff700 1 -- 172.21.15.32:0/3492017373 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{prefix=config set, name=public_network}]=0 v3)=0 v3) ==== 124+0+0 (secure 0 0 0) 0x7f046c019900 con 0x7f047c108620 2024-04-25T00:22:52.287 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.618+0000 7f0481486700 1 -- 172.21.15.32:0/3492017373 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f047c108620 msgr2=0x7f047c107ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:22:52.287 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.618+0000 7f0481486700 1 --2- 172.21.15.32:0/3492017373 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f047c108620 0x7f047c107ea0 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7f046c011670 tx=0x7f046c011b60 comp rx=0 tx=0).stop 2024-04-25T00:22:52.287 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.618+0000 7f0481486700 1 -- 172.21.15.32:0/3492017373 shutdown_connections 2024-04-25T00:22:52.288 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.618+0000 7f0481486700 1 --2- 172.21.15.32:0/3492017373 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f047c108620 0x7f047c107ea0 unknown :-1 s=CLOSED pgs=2 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:22:52.288 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.618+0000 7f0481486700 1 -- 172.21.15.32:0/3492017373 >> 172.21.15.32:0/3492017373 conn(0x7f047c102070 msgr2=0x7f047c102bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:22:52.288 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.618+0000 7f0481486700 1 -- 172.21.15.32:0/3492017373 shutdown_connections 2024-04-25T00:22:52.288 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:51.618+0000 7f0481486700 1 -- 172.21.15.32:0/3492017373 wait complete. 2024-04-25T00:22:52.288 INFO:teuthology.orchestra.run.smithi032.stdout:Wrote config to /etc/ceph/ceph.conf 2024-04-25T00:22:52.310 INFO:teuthology.orchestra.run.smithi032.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-25T00:22:52.310 INFO:teuthology.orchestra.run.smithi032.stdout:Creating mgr... 2024-04-25T00:22:52.310 INFO:teuthology.orchestra.run.smithi032.stdout:Verifying port 0.0.0.0:9283 ... 2024-04-25T00:22:52.311 INFO:teuthology.orchestra.run.smithi032.stdout:Verifying port 0.0.0.0:8765 ... 2024-04-25T00:22:52.571 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-cb997156-0299-11ef-bc93-c7b262605968@mgr.x 2024-04-25T00:22:52.571 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stderr Failed to reset failed state of unit ceph-cb997156-0299-11ef-bc93-c7b262605968@mgr.x.service: Unit ceph-cb997156-0299-11ef-bc93-c7b262605968@mgr.x.service not loaded. 2024-04-25T00:22:52.728 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-cb997156-0299-11ef-bc93-c7b262605968.target.wants/ceph-cb997156-0299-11ef-bc93-c7b262605968@mgr.x.service → /etc/systemd/system/ceph-cb997156-0299-11ef-bc93-c7b262605968@.service. 2024-04-25T00:22:52.963 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:52 smithi032 systemd[1]: Starting Ceph mgr.x for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:22:52.963 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:52 smithi032 podman[29576]: 2024-04-25 00:22:52.962337217 +0000 UTC m=+0.101669625 container create a5f04bc0c425d5ef5a16cbf24cb99d27ff513e1f65cfb00edc375567a1319949 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-d2c45d9, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-25T00:22:53.223 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:52 smithi032 podman[29576]: 2024-04-25 00:22:52.878920068 +0000 UTC m=+0.018252465 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 2024-04-25T00:22:53.223 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:53 smithi032 podman[29576]: 2024-04-25 00:22:53.156352856 +0000 UTC m=+0.295685221 container init a5f04bc0c425d5ef5a16cbf24cb99d27ff513e1f65cfb00edc375567a1319949 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=squid-d2c45d9, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240424) 2024-04-25T00:22:53.223 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:53 smithi032 podman[29576]: 2024-04-25 00:22:53.158905977 +0000 UTC m=+0.298238352 container start a5f04bc0c425d5ef5a16cbf24cb99d27ff513e1f65cfb00edc375567a1319949 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-d2c45d9, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240424) 2024-04-25T00:22:53.237 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-25T00:22:53.237 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout disabled 2024-04-25T00:22:53.244 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-25T00:22:53.244 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout inactive 2024-04-25T00:22:53.244 INFO:teuthology.orchestra.run.smithi032.stdout:firewalld.service is not enabled 2024-04-25T00:22:53.244 INFO:teuthology.orchestra.run.smithi032.stdout:Not possible to enable service . firewalld.service is not available 2024-04-25T00:22:53.256 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-25T00:22:53.256 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout disabled 2024-04-25T00:22:53.264 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-25T00:22:53.264 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout inactive 2024-04-25T00:22:53.264 INFO:teuthology.orchestra.run.smithi032.stdout:firewalld.service is not enabled 2024-04-25T00:22:53.265 INFO:teuthology.orchestra.run.smithi032.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-04-25T00:22:53.265 INFO:teuthology.orchestra.run.smithi032.stdout:Waiting for mgr to start... 2024-04-25T00:22:53.265 INFO:teuthology.orchestra.run.smithi032.stdout:Waiting for mgr... 2024-04-25T00:22:53.488 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:53 smithi032 bash[29576]: a5f04bc0c425d5ef5a16cbf24cb99d27ff513e1f65cfb00edc375567a1319949 2024-04-25T00:22:53.488 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:53 smithi032 systemd[1]: Started Ceph mgr.x for cb997156-0299-11ef-bc93-c7b262605968. 2024-04-25T00:22:53.488 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:53 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:22:53.321+0000 7f7d4190b200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-25T00:22:53.748 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:53 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:22:53.518+0000 7f7d4190b200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-25T00:22:53.748 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:53 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:22:53.623+0000 7f7d4190b200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-25T00:22:54.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:22:53 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/4166933318' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-25T00:22:54.248 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:53 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:22:53.954+0000 7f7d4190b200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-25T00:22:54.248 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:54 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:22:54.048+0000 7f7d4190b200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-25T00:22:55.690 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:55 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:22:55.333+0000 7f7d4190b200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-25T00:22:55.690 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:55 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:22:55.510+0000 7f7d4190b200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-25T00:22:55.691 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:55 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:22:55.690+0000 7f7d4190b200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-25T00:22:56.316 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:56 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:22:56.002+0000 7f7d4190b200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-25T00:22:56.316 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:56 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:22:56.101+0000 7f7d4190b200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-25T00:22:56.316 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:56 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:22:56.315+0000 7f7d4190b200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-25T00:22:56.748 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:56 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:22:56.516+0000 7f7d4190b200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-25T00:22:57.468 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:57 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:22:57.144+0000 7f7d4190b200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-25T00:22:57.471 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 2024-04-25T00:22:57.482 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout { 2024-04-25T00:22:57.482 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "fsid": "cb997156-0299-11ef-bc93-c7b262605968", 2024-04-25T00:22:57.482 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "health": { 2024-04-25T00:22:57.482 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-25T00:22:57.482 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-25T00:22:57.483 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-25T00:22:57.483 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-04-25T00:22:57.483 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-25T00:22:57.483 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-25T00:22:57.483 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 0 2024-04-25T00:22:57.483 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout ], 2024-04-25T00:22:57.483 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-25T00:22:57.483 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "a" 2024-04-25T00:22:57.483 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout ], 2024-04-25T00:22:57.483 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-04-25T00:22:57.483 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-25T00:22:57.483 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-25T00:22:57.483 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "squid", 2024-04-25T00:22:57.483 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-25T00:22:57.483 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-04-25T00:22:57.484 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-25T00:22:57.484 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-25T00:22:57.484 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-25T00:22:57.484 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-25T00:22:57.484 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-25T00:22:57.484 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-25T00:22:57.484 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-25T00:22:57.484 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-25T00:22:57.484 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-04-25T00:22:57.484 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-25T00:22:57.484 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-25T00:22:57.484 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-25T00:22:57.484 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-25T00:22:57.484 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-25T00:22:57.485 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-25T00:22:57.485 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-25T00:22:57.485 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-25T00:22:57.485 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-25T00:22:57.485 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-04-25T00:22:57.485 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-25T00:22:57.485 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-25T00:22:57.485 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-25T00:22:57.485 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-25T00:22:57.485 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-04-25T00:22:57.485 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-25T00:22:57.485 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-25T00:22:57.485 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-25T00:22:57.486 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-25T00:22:57.486 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-25T00:22:57.486 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-25T00:22:57.486 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "restful" 2024-04-25T00:22:57.486 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout ], 2024-04-25T00:22:57.486 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-25T00:22:57.486 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-04-25T00:22:57.486 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-25T00:22:57.486 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-25T00:22:57.486 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "modified": "2024-04-25T00:22:46.105890+0000", 2024-04-25T00:22:57.486 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-25T00:22:57.486 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-04-25T00:22:57.486 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-25T00:22:57.486 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout } 2024-04-25T00:22:57.487 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.719+0000 7ffb4353d700 1 Processor -- start 2024-04-25T00:22:57.487 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.720+0000 7ffb4353d700 1 -- start start 2024-04-25T00:22:57.487 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.720+0000 7ffb4353d700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffb3c108720 0x7ffb3c10ab10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:22:57.487 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.720+0000 7ffb4353d700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7ffb3c07be10 con 0x7ffb3c108720 2024-04-25T00:22:57.487 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.720+0000 7ffb412d9700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffb3c108720 0x7ffb3c10ab10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:22:57.487 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.720+0000 7ffb412d9700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffb3c108720 0x7ffb3c10ab10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:54440/0 (socket says 172.21.15.32:54440) 2024-04-25T00:22:57.487 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.720+0000 7ffb412d9700 1 -- 172.21.15.32:0/659198550 learned_addr learned my addr 172.21.15.32:0/659198550 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:22:57.487 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.721+0000 7ffb412d9700 1 -- 172.21.15.32:0/659198550 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ffb3c10b050 con 0x7ffb3c108720 2024-04-25T00:22:57.487 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.721+0000 7ffb412d9700 1 --2- 172.21.15.32:0/659198550 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffb3c108720 0x7ffb3c10ab10 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7ffb380098a0 tx=0x7ffb38009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6400a1713acf7b6 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:22:57.487 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.721+0000 7ffb33fff700 1 -- 172.21.15.32:0/659198550 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7ffb38004160 con 0x7ffb3c108720 2024-04-25T00:22:57.487 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.721+0000 7ffb33fff700 1 -- 172.21.15.32:0/659198550 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7ffb38004300 con 0x7ffb3c108720 2024-04-25T00:22:57.487 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.721+0000 7ffb33fff700 1 -- 172.21.15.32:0/659198550 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7ffb38011b50 con 0x7ffb3c108720 2024-04-25T00:22:57.487 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.722+0000 7ffb4353d700 1 -- 172.21.15.32:0/659198550 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffb3c108720 msgr2=0x7ffb3c10ab10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:22:57.487 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.722+0000 7ffb4353d700 1 --2- 172.21.15.32:0/659198550 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffb3c108720 0x7ffb3c10ab10 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7ffb380098a0 tx=0x7ffb38009bb0 comp rx=0 tx=0).stop 2024-04-25T00:22:57.488 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.722+0000 7ffb4353d700 1 -- 172.21.15.32:0/659198550 shutdown_connections 2024-04-25T00:22:57.488 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.722+0000 7ffb4353d700 1 --2- 172.21.15.32:0/659198550 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffb3c108720 0x7ffb3c10ab10 unknown :-1 s=CLOSED pgs=5 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:22:57.488 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.722+0000 7ffb4353d700 1 -- 172.21.15.32:0/659198550 >> 172.21.15.32:0/659198550 conn(0x7ffb3c1020d0 msgr2=0x7ffb3c1044f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:22:57.488 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.722+0000 7ffb4353d700 1 -- 172.21.15.32:0/659198550 shutdown_connections 2024-04-25T00:22:57.488 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.722+0000 7ffb4353d700 1 -- 172.21.15.32:0/659198550 wait complete. 2024-04-25T00:22:57.488 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.722+0000 7ffb4353d700 1 Processor -- start 2024-04-25T00:22:57.488 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.722+0000 7ffb4353d700 1 -- start start 2024-04-25T00:22:57.488 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.723+0000 7ffb4353d700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffb3c108720 0x7ffb3c19bb30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:22:57.488 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.723+0000 7ffb4353d700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7ffb3c10bfc0 con 0x7ffb3c108720 2024-04-25T00:22:57.488 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.723+0000 7ffb412d9700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffb3c108720 0x7ffb3c19bb30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:22:57.488 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.723+0000 7ffb412d9700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffb3c108720 0x7ffb3c19bb30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:54448/0 (socket says 172.21.15.32:54448) 2024-04-25T00:22:57.488 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.723+0000 7ffb412d9700 1 -- 172.21.15.32:0/4166933318 learned_addr learned my addr 172.21.15.32:0/4166933318 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:22:57.488 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.723+0000 7ffb412d9700 1 -- 172.21.15.32:0/4166933318 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ffb3c19c070 con 0x7ffb3c108720 2024-04-25T00:22:57.488 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.723+0000 7ffb412d9700 1 --2- 172.21.15.32:0/4166933318 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffb3c108720 0x7ffb3c19bb30 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7ffb38011670 tx=0x7ffb38011b60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:22:57.488 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.723+0000 7ffb327fc700 1 -- 172.21.15.32:0/4166933318 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7ffb380039c0 con 0x7ffb3c108720 2024-04-25T00:22:57.489 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.723+0000 7ffb4353d700 1 -- 172.21.15.32:0/4166933318 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ffb3c19c300 con 0x7ffb3c108720 2024-04-25T00:22:57.489 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.723+0000 7ffb327fc700 1 -- 172.21.15.32:0/4166933318 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7ffb38011e50 con 0x7ffb3c108720 2024-04-25T00:22:57.489 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.724+0000 7ffb327fc700 1 -- 172.21.15.32:0/4166933318 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7ffb3801a5c0 con 0x7ffb3c108720 2024-04-25T00:22:57.489 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.724+0000 7ffb4353d700 1 -- 172.21.15.32:0/4166933318 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ffb3c19eff0 con 0x7ffb3c108720 2024-04-25T00:22:57.489 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.724+0000 7ffb327fc700 1 -- 172.21.15.32:0/4166933318 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7ffb3801ac00 con 0x7ffb3c108720 2024-04-25T00:22:57.489 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.724+0000 7ffb327fc700 1 -- 172.21.15.32:0/4166933318 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7ffb38019700 con 0x7ffb3c108720 2024-04-25T00:22:57.489 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.724+0000 7ffb4353d700 1 -- 172.21.15.32:0/4166933318 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ffb3c063890 con 0x7ffb3c108720 2024-04-25T00:22:57.489 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.727+0000 7ffb327fc700 1 -- 172.21.15.32:0/4166933318 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+73141 (secure 0 0 0) 0x7ffb38003b60 con 0x7ffb3c108720 2024-04-25T00:22:57.489 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.800+0000 7ffb4353d700 1 -- 172.21.15.32:0/4166933318 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) -- 0x7ffb3c19f1c0 con 0x7ffb3c108720 2024-04-25T00:22:57.489 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.800+0000 7ffb327fc700 1 -- 172.21.15.32:0/4166933318 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) ==== 79+0+1239 (secure 0 0 0) 0x7ffb3c19f1c0 con 0x7ffb3c108720 2024-04-25T00:22:57.489 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.801+0000 7ffb4353d700 1 -- 172.21.15.32:0/4166933318 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffb3c108720 msgr2=0x7ffb3c19bb30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:22:57.489 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.801+0000 7ffb4353d700 1 --2- 172.21.15.32:0/4166933318 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffb3c108720 0x7ffb3c19bb30 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7ffb38011670 tx=0x7ffb38011b60 comp rx=0 tx=0).stop 2024-04-25T00:22:57.489 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.801+0000 7ffb4353d700 1 -- 172.21.15.32:0/4166933318 shutdown_connections 2024-04-25T00:22:57.490 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.801+0000 7ffb4353d700 1 --2- 172.21.15.32:0/4166933318 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffb3c108720 0x7ffb3c19bb30 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:22:57.490 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.801+0000 7ffb4353d700 1 -- 172.21.15.32:0/4166933318 >> 172.21.15.32:0/4166933318 conn(0x7ffb3c1020d0 msgr2=0x7ffb3c102b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:22:57.490 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.802+0000 7ffb4353d700 1 -- 172.21.15.32:0/4166933318 shutdown_connections 2024-04-25T00:22:57.490 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:22:53.802+0000 7ffb4353d700 1 -- 172.21.15.32:0/4166933318 wait complete. 2024-04-25T00:22:57.490 INFO:teuthology.orchestra.run.smithi032.stdout:mgr not available, waiting (1/15)... 2024-04-25T00:22:57.748 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:57 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:22:57.586+0000 7f7d4190b200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-25T00:22:57.748 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:57 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:22:57.687+0000 7f7d4190b200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-25T00:22:58.248 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:57 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:22:57.895+0000 7f7d4190b200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-25T00:22:58.748 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:58 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:22:58.440+0000 7f7d4190b200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-25T00:22:58.748 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:58 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:22:58.535+0000 7f7d4190b200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-25T00:22:59.148 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:58 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:22:58.789+0000 7f7d4190b200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-25T00:22:59.148 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:58 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:22:58.928+0000 7f7d4190b200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-25T00:22:59.148 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:59 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:22:59.027+0000 7f7d4190b200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-25T00:22:59.148 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:59 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:22:59.147+0000 7f7d4190b200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-25T00:22:59.498 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:22:59 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:22:59.242+0000 7f7d4190b200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-25T00:23:00.371 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:00 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:00.095+0000 7f7d4190b200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-25T00:23:00.372 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:00 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:00.318+0000 7f7d4190b200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-25T00:23:00.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:00 smithi032 ceph-mon[29357]: Activating manager daemon x 2024-04-25T00:23:00.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:00 smithi032 ceph-mon[29357]: mgrmap e2: x(active, starting, since 0.00334884s) 2024-04-25T00:23:00.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:00 smithi032 ceph-mon[29357]: from='mgr.14100 172.21.15.32:0/1098972696' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-25T00:23:00.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:00 smithi032 ceph-mon[29357]: from='mgr.14100 172.21.15.32:0/1098972696' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-25T00:23:00.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:00 smithi032 ceph-mon[29357]: from='mgr.14100 172.21.15.32:0/1098972696' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-25T00:23:00.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:00 smithi032 ceph-mon[29357]: from='mgr.14100 172.21.15.32:0/1098972696' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-25T00:23:00.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:00 smithi032 ceph-mon[29357]: from='mgr.14100 172.21.15.32:0/1098972696' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-25T00:23:00.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:00 smithi032 ceph-mon[29357]: Manager daemon x is now available 2024-04-25T00:23:00.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:00 smithi032 ceph-mon[29357]: from='mgr.14100 172.21.15.32:0/1098972696' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-25T00:23:00.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:00 smithi032 ceph-mon[29357]: from='mgr.14100 172.21.15.32:0/1098972696' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-25T00:23:00.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:00 smithi032 ceph-mon[29357]: from='mgr.14100 172.21.15.32:0/1098972696' entity='mgr.x' 2024-04-25T00:23:00.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:00 smithi032 ceph-mon[29357]: from='mgr.14100 172.21.15.32:0/1098972696' entity='mgr.x' 2024-04-25T00:23:00.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:00 smithi032 ceph-mon[29357]: from='mgr.14100 172.21.15.32:0/1098972696' entity='mgr.x' 2024-04-25T00:23:02.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:02 smithi032 ceph-mon[29357]: mgrmap e3: x(active, since 1.00745s) 2024-04-25T00:23:03.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:03 smithi032 ceph-mon[29357]: mgrmap e4: x(active, since 2s) 2024-04-25T00:23:03.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:03 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1573564542' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-25T00:23:05.505 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 2024-04-25T00:23:05.505 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout { 2024-04-25T00:23:05.505 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "fsid": "cb997156-0299-11ef-bc93-c7b262605968", 2024-04-25T00:23:05.505 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "health": { 2024-04-25T00:23:05.505 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-25T00:23:05.505 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-25T00:23:05.505 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-25T00:23:05.505 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-04-25T00:23:05.505 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-25T00:23:05.505 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-25T00:23:05.506 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 0 2024-04-25T00:23:05.506 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout ], 2024-04-25T00:23:05.506 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-25T00:23:05.506 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "a" 2024-04-25T00:23:05.506 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout ], 2024-04-25T00:23:05.506 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "quorum_age": 11, 2024-04-25T00:23:05.506 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-25T00:23:05.506 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-25T00:23:05.506 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "squid", 2024-04-25T00:23:05.506 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-25T00:23:05.506 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-04-25T00:23:05.506 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-25T00:23:05.506 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-25T00:23:05.506 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-25T00:23:05.507 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-25T00:23:05.507 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-25T00:23:05.507 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-25T00:23:05.507 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-25T00:23:05.507 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-25T00:23:05.507 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-04-25T00:23:05.507 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-25T00:23:05.507 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-25T00:23:05.507 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-25T00:23:05.507 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-25T00:23:05.507 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-25T00:23:05.507 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-25T00:23:05.507 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-25T00:23:05.507 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-25T00:23:05.508 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-25T00:23:05.508 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-04-25T00:23:05.508 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-25T00:23:05.508 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-25T00:23:05.508 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-25T00:23:05.508 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-25T00:23:05.508 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-04-25T00:23:05.508 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-25T00:23:05.508 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-25T00:23:05.508 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-25T00:23:05.508 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-25T00:23:05.508 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-25T00:23:05.508 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-25T00:23:05.508 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "restful" 2024-04-25T00:23:05.508 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout ], 2024-04-25T00:23:05.509 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-25T00:23:05.509 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-04-25T00:23:05.509 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-25T00:23:05.509 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-25T00:23:05.509 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "modified": "2024-04-25T00:22:46.105890+0000", 2024-04-25T00:23:05.509 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-25T00:23:05.509 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-04-25T00:23:05.509 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-25T00:23:05.509 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout } 2024-04-25T00:23:05.509 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.541+0000 7f19b3ebb700 1 Processor -- start 2024-04-25T00:23:05.509 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.541+0000 7f19b3ebb700 1 -- start start 2024-04-25T00:23:05.509 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.541+0000 7f19b3ebb700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f19ac106490 0x7f19ac1068b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:05.509 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.541+0000 7f19b3ebb700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f19ac07be50 con 0x7f19ac106490 2024-04-25T00:23:05.509 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.541+0000 7f19b1c57700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f19ac106490 0x7f19ac1068b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:05.510 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.541+0000 7f19b1c57700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f19ac106490 0x7f19ac1068b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:51914/0 (socket says 172.21.15.32:51914) 2024-04-25T00:23:05.510 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.541+0000 7f19b1c57700 1 -- 172.21.15.32:0/1021717015 learned_addr learned my addr 172.21.15.32:0/1021717015 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:05.510 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.542+0000 7f19b1c57700 1 -- 172.21.15.32:0/1021717015 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f19ac106df0 con 0x7f19ac106490 2024-04-25T00:23:05.510 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.542+0000 7f19b1c57700 1 --2- 172.21.15.32:0/1021717015 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f19ac106490 0x7f19ac1068b0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f19a80098a0 tx=0x7f19a8009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=57bb97015a80cb86 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:05.510 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.542+0000 7f19b0c55700 1 -- 172.21.15.32:0/1021717015 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f19a8004160 con 0x7f19ac106490 2024-04-25T00:23:05.510 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.542+0000 7f19b0c55700 1 -- 172.21.15.32:0/1021717015 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f19a8004300 con 0x7f19ac106490 2024-04-25T00:23:05.510 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.542+0000 7f19b0c55700 1 -- 172.21.15.32:0/1021717015 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f19a8011b50 con 0x7f19ac106490 2024-04-25T00:23:05.510 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.542+0000 7f19b3ebb700 1 -- 172.21.15.32:0/1021717015 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f19ac106490 msgr2=0x7f19ac1068b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:05.510 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.542+0000 7f19b3ebb700 1 --2- 172.21.15.32:0/1021717015 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f19ac106490 0x7f19ac1068b0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f19a80098a0 tx=0x7f19a8009bb0 comp rx=0 tx=0).stop 2024-04-25T00:23:05.510 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.542+0000 7f19b3ebb700 1 -- 172.21.15.32:0/1021717015 shutdown_connections 2024-04-25T00:23:05.510 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.542+0000 7f19b3ebb700 1 --2- 172.21.15.32:0/1021717015 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f19ac106490 0x7f19ac1068b0 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:05.510 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.542+0000 7f19b3ebb700 1 -- 172.21.15.32:0/1021717015 >> 172.21.15.32:0/1021717015 conn(0x7f19ac102070 msgr2=0x7f19ac104490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:05.510 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.543+0000 7f19b3ebb700 1 -- 172.21.15.32:0/1021717015 shutdown_connections 2024-04-25T00:23:05.510 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.543+0000 7f19b3ebb700 1 -- 172.21.15.32:0/1021717015 wait complete. 2024-04-25T00:23:05.511 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.543+0000 7f19b3ebb700 1 Processor -- start 2024-04-25T00:23:05.511 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.543+0000 7f19b3ebb700 1 -- start start 2024-04-25T00:23:05.511 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.543+0000 7f19b3ebb700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f19ac106490 0x7f19ac19dd60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:05.511 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.544+0000 7f19b3ebb700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f19ac107e80 con 0x7f19ac106490 2024-04-25T00:23:05.511 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.544+0000 7f19b1c57700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f19ac106490 0x7f19ac19dd60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:05.511 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.544+0000 7f19b1c57700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f19ac106490 0x7f19ac19dd60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:51928/0 (socket says 172.21.15.32:51928) 2024-04-25T00:23:05.511 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.544+0000 7f19b1c57700 1 -- 172.21.15.32:0/1573564542 learned_addr learned my addr 172.21.15.32:0/1573564542 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:05.511 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.544+0000 7f19b1c57700 1 -- 172.21.15.32:0/1573564542 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f19ac19e2a0 con 0x7f19ac106490 2024-04-25T00:23:05.511 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.544+0000 7f19b1c57700 1 --2- 172.21.15.32:0/1573564542 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f19ac106490 0x7f19ac19dd60 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f19a8011670 tx=0x7f19a8011b60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:05.511 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.544+0000 7f19a2ffd700 1 -- 172.21.15.32:0/1573564542 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f19a80039c0 con 0x7f19ac106490 2024-04-25T00:23:05.511 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.544+0000 7f19a2ffd700 1 -- 172.21.15.32:0/1573564542 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f19a8011e50 con 0x7f19ac106490 2024-04-25T00:23:05.511 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.544+0000 7f19b3ebb700 1 -- 172.21.15.32:0/1573564542 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f19ac19e530 con 0x7f19ac106490 2024-04-25T00:23:05.511 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.544+0000 7f19a2ffd700 1 -- 172.21.15.32:0/1573564542 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f19a801a5c0 con 0x7f19ac106490 2024-04-25T00:23:05.511 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.544+0000 7f19b3ebb700 1 -- 172.21.15.32:0/1573564542 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f19ac1a1220 con 0x7f19ac106490 2024-04-25T00:23:05.512 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.545+0000 7f19a2ffd700 1 -- 172.21.15.32:0/1573564542 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 4) ==== 46276+0+0 (secure 0 0 0) 0x7f19a801a8c0 con 0x7f19ac106490 2024-04-25T00:23:05.512 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.545+0000 7f19a2ffd700 1 --2- 172.21.15.32:0/1573564542 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7f1998038fb0 0x7f199803b470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:05.512 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.545+0000 7f19a2ffd700 1 -- 172.21.15.32:0/1573564542 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f19a804d480 con 0x7f19ac106490 2024-04-25T00:23:05.512 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.545+0000 7f19b3ebb700 1 -- 172.21.15.32:0/1573564542 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f19ac063890 con 0x7f19ac106490 2024-04-25T00:23:05.512 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.545+0000 7f19b1456700 1 --2- 172.21.15.32:0/1573564542 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7f1998038fb0 0x7f199803b470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:05.512 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.546+0000 7f19b1456700 1 --2- 172.21.15.32:0/1573564542 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7f1998038fb0 0x7f199803b470 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7f199c006fd0 tx=0x7f199c006e40 comp rx=0 tx=0).ready entity=mgr.14100 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:05.512 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.551+0000 7f19a2ffd700 1 -- 172.21.15.32:0/1573564542 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f19a8019c70 con 0x7f19ac106490 2024-04-25T00:23:05.512 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.841+0000 7f19b3ebb700 1 -- 172.21.15.32:0/1573564542 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) -- 0x7f19ac02dd30 con 0x7f19ac106490 2024-04-25T00:23:05.512 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.842+0000 7f19a2ffd700 1 -- 172.21.15.32:0/1573564542 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) ==== 79+0+1239 (secure 0 0 0) 0x7f19a8032030 con 0x7f19ac106490 2024-04-25T00:23:05.512 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.844+0000 7f19b3ebb700 1 -- 172.21.15.32:0/1573564542 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7f1998038fb0 msgr2=0x7f199803b470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:05.512 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.844+0000 7f19b3ebb700 1 --2- 172.21.15.32:0/1573564542 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7f1998038fb0 0x7f199803b470 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7f199c006fd0 tx=0x7f199c006e40 comp rx=0 tx=0).stop 2024-04-25T00:23:05.512 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.844+0000 7f19b3ebb700 1 -- 172.21.15.32:0/1573564542 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f19ac106490 msgr2=0x7f19ac19dd60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:05.512 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.844+0000 7f19b3ebb700 1 --2- 172.21.15.32:0/1573564542 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f19ac106490 0x7f19ac19dd60 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f19a8011670 tx=0x7f19a8011b60 comp rx=0 tx=0).stop 2024-04-25T00:23:05.512 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.844+0000 7f19b3ebb700 1 -- 172.21.15.32:0/1573564542 shutdown_connections 2024-04-25T00:23:05.513 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.844+0000 7f19b3ebb700 1 --2- 172.21.15.32:0/1573564542 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7f1998038fb0 0x7f199803b470 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:05.513 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.844+0000 7f19b3ebb700 1 --2- 172.21.15.32:0/1573564542 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f19ac106490 0x7f19ac19dd60 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:05.513 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.844+0000 7f19b3ebb700 1 -- 172.21.15.32:0/1573564542 >> 172.21.15.32:0/1573564542 conn(0x7f19ac102070 msgr2=0x7f19ac194940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:05.513 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.844+0000 7f19b3ebb700 1 -- 172.21.15.32:0/1573564542 shutdown_connections 2024-04-25T00:23:05.513 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:02.844+0000 7f19b3ebb700 1 -- 172.21.15.32:0/1573564542 wait complete. 2024-04-25T00:23:05.513 INFO:teuthology.orchestra.run.smithi032.stdout:mgr is available 2024-04-25T00:23:06.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:06 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1172794532' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-25T00:23:06.789 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 2024-04-25T00:23:06.789 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout [global] 2024-04-25T00:23:06.790 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout fsid = cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:23:06.790 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-25T00:23:06.790 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-25T00:23:06.790 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-25T00:23:06.790 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-25T00:23:06.790 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 2024-04-25T00:23:06.790 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-25T00:23:06.790 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-25T00:23:06.790 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 2024-04-25T00:23:06.790 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout [osd] 2024-04-25T00:23:06.790 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mon_osd_backfillfull_ratio = 0.85 2024-04-25T00:23:06.790 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mon_osd_full_ratio = 0.9 2024-04-25T00:23:06.790 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mon_osd_nearfull_ratio = 0.8 2024-04-25T00:23:06.790 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-25T00:23:06.791 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-25T00:23:06.791 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.971+0000 7f37ac42f700 1 Processor -- start 2024-04-25T00:23:06.791 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.972+0000 7f37ac42f700 1 -- start start 2024-04-25T00:23:06.791 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.972+0000 7f37ac42f700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f37a4108650 0x7f37a410aa40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:06.791 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.972+0000 7f37ac42f700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f37a407bed0 con 0x7f37a4108650 2024-04-25T00:23:06.791 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.972+0000 7f37aa1cb700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f37a4108650 0x7f37a410aa40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:06.791 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.972+0000 7f37aa1cb700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f37a4108650 0x7f37a410aa40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:51934/0 (socket says 172.21.15.32:51934) 2024-04-25T00:23:06.791 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.972+0000 7f37aa1cb700 1 -- 172.21.15.32:0/1152253177 learned_addr learned my addr 172.21.15.32:0/1152253177 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:06.791 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.972+0000 7f37aa1cb700 1 -- 172.21.15.32:0/1152253177 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f37a410af80 con 0x7f37a4108650 2024-04-25T00:23:06.791 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.973+0000 7f37aa1cb700 1 --2- 172.21.15.32:0/1152253177 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f37a4108650 0x7f37a410aa40 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f37a00098a0 tx=0x7f37a0009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=853ec3d4225c6b80 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:06.791 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.973+0000 7f37a91c9700 1 -- 172.21.15.32:0/1152253177 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f37a0004160 con 0x7f37a4108650 2024-04-25T00:23:06.791 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.973+0000 7f37a91c9700 1 -- 172.21.15.32:0/1152253177 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f37a0004300 con 0x7f37a4108650 2024-04-25T00:23:06.792 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.973+0000 7f37a91c9700 1 -- 172.21.15.32:0/1152253177 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f37a0011b50 con 0x7f37a4108650 2024-04-25T00:23:06.792 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.974+0000 7f37ac42f700 1 -- 172.21.15.32:0/1152253177 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f37a4108650 msgr2=0x7f37a410aa40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:06.792 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.974+0000 7f37ac42f700 1 --2- 172.21.15.32:0/1152253177 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f37a4108650 0x7f37a410aa40 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f37a00098a0 tx=0x7f37a0009bb0 comp rx=0 tx=0).stop 2024-04-25T00:23:06.792 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.974+0000 7f37ac42f700 1 -- 172.21.15.32:0/1152253177 shutdown_connections 2024-04-25T00:23:06.792 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.974+0000 7f37ac42f700 1 --2- 172.21.15.32:0/1152253177 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f37a4108650 0x7f37a410aa40 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:06.792 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.974+0000 7f37ac42f700 1 -- 172.21.15.32:0/1152253177 >> 172.21.15.32:0/1152253177 conn(0x7f37a41020a0 msgr2=0x7f37a41044c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:06.792 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.974+0000 7f37ac42f700 1 -- 172.21.15.32:0/1152253177 shutdown_connections 2024-04-25T00:23:06.792 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.974+0000 7f37ac42f700 1 -- 172.21.15.32:0/1152253177 wait complete. 2024-04-25T00:23:06.792 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.974+0000 7f37ac42f700 1 Processor -- start 2024-04-25T00:23:06.792 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.974+0000 7f37ac42f700 1 -- start start 2024-04-25T00:23:06.792 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.975+0000 7f37ac42f700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f37a4108650 0x7f37a4107ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:06.792 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.975+0000 7f37ac42f700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f37a410bfb0 con 0x7f37a4108650 2024-04-25T00:23:06.792 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.975+0000 7f37aa1cb700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f37a4108650 0x7f37a4107ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:06.793 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.975+0000 7f37aa1cb700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f37a4108650 0x7f37a4107ed0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:51946/0 (socket says 172.21.15.32:51946) 2024-04-25T00:23:06.793 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.975+0000 7f37aa1cb700 1 -- 172.21.15.32:0/1172794532 learned_addr learned my addr 172.21.15.32:0/1172794532 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:06.793 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.975+0000 7f37aa1cb700 1 -- 172.21.15.32:0/1172794532 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f37a4108410 con 0x7f37a4108650 2024-04-25T00:23:06.793 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.975+0000 7f37aa1cb700 1 --2- 172.21.15.32:0/1172794532 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f37a4108650 0x7f37a4107ed0 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f37a0011b60 tx=0x7f37a0011c40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:06.793 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.975+0000 7f379b7fe700 1 -- 172.21.15.32:0/1172794532 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f37a0003b20 con 0x7f37a4108650 2024-04-25T00:23:06.793 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.975+0000 7f37ac42f700 1 -- 172.21.15.32:0/1172794532 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f37a4106600 con 0x7f37a4108650 2024-04-25T00:23:06.793 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.975+0000 7f379b7fe700 1 -- 172.21.15.32:0/1172794532 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f37a0003cc0 con 0x7f37a4108650 2024-04-25T00:23:06.793 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.975+0000 7f379b7fe700 1 -- 172.21.15.32:0/1172794532 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f37a0024840 con 0x7f37a4108650 2024-04-25T00:23:06.793 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.975+0000 7f37ac42f700 1 -- 172.21.15.32:0/1172794532 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f37a4106b40 con 0x7f37a4108650 2024-04-25T00:23:06.793 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.976+0000 7f37ac42f700 1 -- 172.21.15.32:0/1172794532 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f37a4063890 con 0x7f37a4108650 2024-04-25T00:23:06.793 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.977+0000 7f379b7fe700 1 -- 172.21.15.32:0/1172794532 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 4) ==== 46276+0+0 (secure 0 0 0) 0x7f37a0022020 con 0x7f37a4108650 2024-04-25T00:23:06.793 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.977+0000 7f379b7fe700 1 --2- 172.21.15.32:0/1172794532 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7f3790039000 0x7f379003b4c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:06.794 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.977+0000 7f379b7fe700 1 -- 172.21.15.32:0/1172794532 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f37a0011e00 con 0x7f37a4108650 2024-04-25T00:23:06.794 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.977+0000 7f37a99ca700 1 --2- 172.21.15.32:0/1172794532 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7f3790039000 0x7f379003b4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:06.794 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.977+0000 7f37a99ca700 1 --2- 172.21.15.32:0/1172794532 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7f3790039000 0x7f379003b4c0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f3794006fd0 tx=0x7f3794006e40 comp rx=0 tx=0).ready entity=mgr.14100 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:06.794 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:05.982+0000 7f379b7fe700 1 -- 172.21.15.32:0/1172794532 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f37a0052020 con 0x7f37a4108650 2024-04-25T00:23:06.794 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:06.190+0000 7f37ac42f700 1 -- 172.21.15.32:0/1172794532 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "config assimilate-conf"} v 0) -- 0x7f37a4107330 con 0x7f37a4108650 2024-04-25T00:23:06.794 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:06.191+0000 7f379b7fe700 1 -- 172.21.15.32:0/1172794532 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "config assimilate-conf"}]=0 v3) ==== 70+0+373 (secure 0 0 0) 0x7f37a000bc70 con 0x7f37a4108650 2024-04-25T00:23:06.794 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:06.193+0000 7f37ac42f700 1 -- 172.21.15.32:0/1172794532 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7f3790039000 msgr2=0x7f379003b4c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:06.794 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:06.193+0000 7f37ac42f700 1 --2- 172.21.15.32:0/1172794532 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7f3790039000 0x7f379003b4c0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f3794006fd0 tx=0x7f3794006e40 comp rx=0 tx=0).stop 2024-04-25T00:23:06.794 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:06.193+0000 7f37ac42f700 1 -- 172.21.15.32:0/1172794532 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f37a4108650 msgr2=0x7f37a4107ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:06.794 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:06.193+0000 7f37ac42f700 1 --2- 172.21.15.32:0/1172794532 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f37a4108650 0x7f37a4107ed0 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f37a0011b60 tx=0x7f37a0011c40 comp rx=0 tx=0).stop 2024-04-25T00:23:06.794 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:06.194+0000 7f37ac42f700 1 -- 172.21.15.32:0/1172794532 shutdown_connections 2024-04-25T00:23:06.794 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:06.194+0000 7f37ac42f700 1 --2- 172.21.15.32:0/1172794532 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7f3790039000 0x7f379003b4c0 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:06.794 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:06.194+0000 7f37ac42f700 1 --2- 172.21.15.32:0/1172794532 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f37a4108650 0x7f37a4107ed0 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:06.795 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:06.194+0000 7f37ac42f700 1 -- 172.21.15.32:0/1172794532 >> 172.21.15.32:0/1172794532 conn(0x7f37a41020a0 msgr2=0x7f37a4102b90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:06.795 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:06.194+0000 7f37ac42f700 1 -- 172.21.15.32:0/1172794532 shutdown_connections 2024-04-25T00:23:06.795 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:06.194+0000 7f37ac42f700 1 -- 172.21.15.32:0/1172794532 wait complete. 2024-04-25T00:23:06.795 INFO:teuthology.orchestra.run.smithi032.stdout:Enabling cephadm module... 2024-04-25T00:23:07.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:07 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/4158428006' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-25T00:23:07.748 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:07 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: ignoring --setuser ceph since I am not root 2024-04-25T00:23:07.749 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:07 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: ignoring --setgroup ceph since I am not root 2024-04-25T00:23:07.749 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:07 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:07.701+0000 7fa024283200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-25T00:23:08.119 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:07 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:07.893+0000 7fa024283200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-25T00:23:08.119 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:08 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:07.999+0000 7fa024283200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-25T00:23:08.121 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.194+0000 7f71d89e7700 1 Processor -- start 2024-04-25T00:23:08.127 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.194+0000 7f71d89e7700 1 -- start start 2024-04-25T00:23:08.127 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.194+0000 7f71d89e7700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71d407de70 0x7f71d407c320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:08.127 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.194+0000 7f71d89e7700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f71d407c8f0 con 0x7f71d407de70 2024-04-25T00:23:08.127 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.194+0000 7f71d259c700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71d407de70 0x7f71d407c320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:08.127 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.194+0000 7f71d259c700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71d407de70 0x7f71d407c320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:51956/0 (socket says 172.21.15.32:51956) 2024-04-25T00:23:08.127 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.194+0000 7f71d259c700 1 -- 172.21.15.32:0/3617592583 learned_addr learned my addr 172.21.15.32:0/3617592583 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:08.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.195+0000 7f71d259c700 1 -- 172.21.15.32:0/3617592583 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f71d407ca70 con 0x7f71d407de70 2024-04-25T00:23:08.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.195+0000 7f71d259c700 1 --2- 172.21.15.32:0/3617592583 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71d407de70 0x7f71d407c320 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f71c40098a0 tx=0x7f71c4009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=406e0509ec6299ee server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:08.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.195+0000 7f71d159a700 1 -- 172.21.15.32:0/3617592583 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f71c4004160 con 0x7f71d407de70 2024-04-25T00:23:08.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.195+0000 7f71d159a700 1 -- 172.21.15.32:0/3617592583 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f71c4004300 con 0x7f71d407de70 2024-04-25T00:23:08.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.195+0000 7f71d159a700 1 -- 172.21.15.32:0/3617592583 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f71c4011b50 con 0x7f71d407de70 2024-04-25T00:23:08.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.196+0000 7f71d89e7700 1 -- 172.21.15.32:0/3617592583 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71d407de70 msgr2=0x7f71d407c320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:08.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.196+0000 7f71d89e7700 1 --2- 172.21.15.32:0/3617592583 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71d407de70 0x7f71d407c320 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f71c40098a0 tx=0x7f71c4009bb0 comp rx=0 tx=0).stop 2024-04-25T00:23:08.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.196+0000 7f71d89e7700 1 -- 172.21.15.32:0/3617592583 shutdown_connections 2024-04-25T00:23:08.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.196+0000 7f71d89e7700 1 --2- 172.21.15.32:0/3617592583 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71d407de70 0x7f71d407c320 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:08.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.196+0000 7f71d89e7700 1 -- 172.21.15.32:0/3617592583 >> 172.21.15.32:0/3617592583 conn(0x7f71d4103150 msgr2=0x7f71d41055b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:08.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.196+0000 7f71d89e7700 1 -- 172.21.15.32:0/3617592583 shutdown_connections 2024-04-25T00:23:08.129 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.196+0000 7f71d89e7700 1 -- 172.21.15.32:0/3617592583 wait complete. 2024-04-25T00:23:08.129 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.196+0000 7f71d89e7700 1 Processor -- start 2024-04-25T00:23:08.129 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.197+0000 7f71d89e7700 1 -- start start 2024-04-25T00:23:08.129 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.197+0000 7f71d89e7700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71d407de70 0x7f71d419dd10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:08.129 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.197+0000 7f71d89e7700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f71d410e180 con 0x7f71d407de70 2024-04-25T00:23:08.129 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.197+0000 7f71d259c700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71d407de70 0x7f71d419dd10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:08.129 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.197+0000 7f71d259c700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71d407de70 0x7f71d419dd10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:51964/0 (socket says 172.21.15.32:51964) 2024-04-25T00:23:08.129 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.197+0000 7f71d259c700 1 -- 172.21.15.32:0/4158428006 learned_addr learned my addr 172.21.15.32:0/4158428006 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:08.129 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.197+0000 7f71d259c700 1 -- 172.21.15.32:0/4158428006 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f71d419e250 con 0x7f71d407de70 2024-04-25T00:23:08.129 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.197+0000 7f71d259c700 1 --2- 172.21.15.32:0/4158428006 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71d407de70 0x7f71d419dd10 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f71c4011b60 tx=0x7f71c4011c40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:08.129 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.197+0000 7f71c37fe700 1 -- 172.21.15.32:0/4158428006 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f71c4003b20 con 0x7f71d407de70 2024-04-25T00:23:08.130 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.197+0000 7f71d89e7700 1 -- 172.21.15.32:0/4158428006 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f71d419e4e0 con 0x7f71d407de70 2024-04-25T00:23:08.130 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.197+0000 7f71c37fe700 1 -- 172.21.15.32:0/4158428006 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f71c4003cc0 con 0x7f71d407de70 2024-04-25T00:23:08.130 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.197+0000 7f71c37fe700 1 -- 172.21.15.32:0/4158428006 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f71c4024840 con 0x7f71d407de70 2024-04-25T00:23:08.130 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.197+0000 7f71d89e7700 1 -- 172.21.15.32:0/4158428006 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f71d41a11c0 con 0x7f71d407de70 2024-04-25T00:23:08.130 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.198+0000 7f71c37fe700 1 -- 172.21.15.32:0/4158428006 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 4) ==== 46276+0+0 (secure 0 0 0) 0x7f71c4022020 con 0x7f71d407de70 2024-04-25T00:23:08.130 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.198+0000 7f71d89e7700 1 -- 172.21.15.32:0/4158428006 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f71d4063890 con 0x7f71d407de70 2024-04-25T00:23:08.130 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.198+0000 7f71c37fe700 1 --2- 172.21.15.32:0/4158428006 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7f71bc039050 0x7f71bc03b510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:08.130 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.199+0000 7f71c37fe700 1 -- 172.21.15.32:0/4158428006 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f71c4011e00 con 0x7f71d407de70 2024-04-25T00:23:08.130 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.199+0000 7f71d1d9b700 1 --2- 172.21.15.32:0/4158428006 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7f71bc039050 0x7f71bc03b510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:08.130 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.199+0000 7f71d1d9b700 1 --2- 172.21.15.32:0/4158428006 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7f71bc039050 0x7f71bc03b510 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f71c8006fd0 tx=0x7f71c8006e40 comp rx=0 tx=0).ready entity=mgr.14100 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:08.130 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.204+0000 7f71c37fe700 1 -- 172.21.15.32:0/4158428006 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f71c400ba30 con 0x7f71d407de70 2024-04-25T00:23:08.130 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.469+0000 7f71d89e7700 1 -- 172.21.15.32:0/4158428006 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "mgr module enable", "module": "cephadm"} v 0) -- 0x7f71d41a1390 con 0x7f71d407de70 2024-04-25T00:23:08.130 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.521+0000 7f71c37fe700 1 -- 172.21.15.32:0/4158428006 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "mgr module enable", "module": "cephadm"}]=0 v5) ==== 86+0+0 (secure 0 0 0) 0x7f71d41a1390 con 0x7f71d407de70 2024-04-25T00:23:08.130 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.522+0000 7f71c37fe700 1 -- 172.21.15.32:0/4158428006 <== mon.0 v2:172.21.15.32:3300/0 8 ==== mgrmap(e 5) ==== 46287+0+0 (secure 0 0 0) 0x7f71c402d720 con 0x7f71d407de70 2024-04-25T00:23:08.131 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.524+0000 7f71d89e7700 1 -- 172.21.15.32:0/4158428006 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7f71bc039050 msgr2=0x7f71bc03b510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:08.131 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.524+0000 7f71d89e7700 1 --2- 172.21.15.32:0/4158428006 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7f71bc039050 0x7f71bc03b510 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f71c8006fd0 tx=0x7f71c8006e40 comp rx=0 tx=0).stop 2024-04-25T00:23:08.131 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.524+0000 7f71d89e7700 1 -- 172.21.15.32:0/4158428006 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71d407de70 msgr2=0x7f71d419dd10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:08.131 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.524+0000 7f71d89e7700 1 --2- 172.21.15.32:0/4158428006 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71d407de70 0x7f71d419dd10 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f71c4011b60 tx=0x7f71c4011c40 comp rx=0 tx=0).stop 2024-04-25T00:23:08.131 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.524+0000 7f71d89e7700 1 -- 172.21.15.32:0/4158428006 shutdown_connections 2024-04-25T00:23:08.131 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.524+0000 7f71d89e7700 1 --2- 172.21.15.32:0/4158428006 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7f71bc039050 0x7f71bc03b510 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:08.131 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.524+0000 7f71d89e7700 1 --2- 172.21.15.32:0/4158428006 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71d407de70 0x7f71d419dd10 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:08.131 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.524+0000 7f71d89e7700 1 -- 172.21.15.32:0/4158428006 >> 172.21.15.32:0/4158428006 conn(0x7f71d4103150 msgr2=0x7f71d4103cd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:08.131 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.524+0000 7f71d89e7700 1 -- 172.21.15.32:0/4158428006 shutdown_connections 2024-04-25T00:23:08.131 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:07.524+0000 7f71d89e7700 1 -- 172.21.15.32:0/4158428006 wait complete. 2024-04-25T00:23:08.370 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:08 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:08.333+0000 7fa024283200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-25T00:23:08.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:08 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/4158428006' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-25T00:23:08.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:08 smithi032 ceph-mon[29357]: mgrmap e5: x(active, since 7s) 2024-04-25T00:23:08.748 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:08 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:08.424+0000 7fa024283200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-25T00:23:09.450 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout { 2024-04-25T00:23:09.450 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-04-25T00:23:09.450 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-25T00:23:09.450 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "active_name": "x", 2024-04-25T00:23:09.450 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-25T00:23:09.451 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout } 2024-04-25T00:23:09.451 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.558+0000 7ff1137d4700 1 Processor -- start 2024-04-25T00:23:09.451 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.558+0000 7ff1137d4700 1 -- start start 2024-04-25T00:23:09.451 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.558+0000 7ff1137d4700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff10c105eb0 0x7ff10c1062d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:09.451 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.558+0000 7ff1137d4700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7ff10c1068a0 con 0x7ff10c105eb0 2024-04-25T00:23:09.451 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.559+0000 7ff111570700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff10c105eb0 0x7ff10c1062d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:09.451 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.559+0000 7ff111570700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff10c105eb0 0x7ff10c1062d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:51996/0 (socket says 172.21.15.32:51996) 2024-04-25T00:23:09.451 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.559+0000 7ff111570700 1 -- 172.21.15.32:0/4176443612 learned_addr learned my addr 172.21.15.32:0/4176443612 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:09.451 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.559+0000 7ff111570700 1 -- 172.21.15.32:0/4176443612 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff10c106a20 con 0x7ff10c105eb0 2024-04-25T00:23:09.451 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.559+0000 7ff111570700 1 --2- 172.21.15.32:0/4176443612 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff10c105eb0 0x7ff10c1062d0 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7ff0fc0098a0 tx=0x7ff0fc009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7f51489365a546a server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:09.451 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.559+0000 7ff103fff700 1 -- 172.21.15.32:0/4176443612 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7ff0fc004160 con 0x7ff10c105eb0 2024-04-25T00:23:09.451 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.559+0000 7ff103fff700 1 -- 172.21.15.32:0/4176443612 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7ff0fc004300 con 0x7ff10c105eb0 2024-04-25T00:23:09.453 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.559+0000 7ff103fff700 1 -- 172.21.15.32:0/4176443612 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7ff0fc011b50 con 0x7ff10c105eb0 2024-04-25T00:23:09.453 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.560+0000 7ff1137d4700 1 -- 172.21.15.32:0/4176443612 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff10c105eb0 msgr2=0x7ff10c1062d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:09.453 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.560+0000 7ff1137d4700 1 --2- 172.21.15.32:0/4176443612 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff10c105eb0 0x7ff10c1062d0 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7ff0fc0098a0 tx=0x7ff0fc009bb0 comp rx=0 tx=0).stop 2024-04-25T00:23:09.453 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.560+0000 7ff1137d4700 1 -- 172.21.15.32:0/4176443612 shutdown_connections 2024-04-25T00:23:09.453 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.560+0000 7ff1137d4700 1 --2- 172.21.15.32:0/4176443612 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff10c105eb0 0x7ff10c1062d0 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:09.453 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.560+0000 7ff1137d4700 1 -- 172.21.15.32:0/4176443612 >> 172.21.15.32:0/4176443612 conn(0x7ff10c101830 msgr2=0x7ff10c103c50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:09.453 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.560+0000 7ff1137d4700 1 -- 172.21.15.32:0/4176443612 shutdown_connections 2024-04-25T00:23:09.453 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.560+0000 7ff1137d4700 1 -- 172.21.15.32:0/4176443612 wait complete. 2024-04-25T00:23:09.453 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.561+0000 7ff1137d4700 1 Processor -- start 2024-04-25T00:23:09.453 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.561+0000 7ff1137d4700 1 -- start start 2024-04-25T00:23:09.453 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.561+0000 7ff1137d4700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff10c105eb0 0x7ff10c19bb50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:09.454 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.561+0000 7ff1137d4700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7ff10c109d70 con 0x7ff10c105eb0 2024-04-25T00:23:09.454 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.561+0000 7ff111570700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff10c105eb0 0x7ff10c19bb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:09.454 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.561+0000 7ff111570700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff10c105eb0 0x7ff10c19bb50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:52000/0 (socket says 172.21.15.32:52000) 2024-04-25T00:23:09.454 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.561+0000 7ff111570700 1 -- 172.21.15.32:0/3785991760 learned_addr learned my addr 172.21.15.32:0/3785991760 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:09.454 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.561+0000 7ff111570700 1 -- 172.21.15.32:0/3785991760 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff10c19c090 con 0x7ff10c105eb0 2024-04-25T00:23:09.454 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.561+0000 7ff111570700 1 --2- 172.21.15.32:0/3785991760 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff10c105eb0 0x7ff10c19bb50 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7ff0fc000c00 tx=0x7ff0fc011c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:09.454 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.562+0000 7ff1027fc700 1 -- 172.21.15.32:0/3785991760 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7ff0fc003b20 con 0x7ff10c105eb0 2024-04-25T00:23:09.454 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.562+0000 7ff1027fc700 1 -- 172.21.15.32:0/3785991760 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7ff0fc003cc0 con 0x7ff10c105eb0 2024-04-25T00:23:09.454 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.562+0000 7ff1137d4700 1 -- 172.21.15.32:0/3785991760 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff10c19c380 con 0x7ff10c105eb0 2024-04-25T00:23:09.454 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.562+0000 7ff1027fc700 1 -- 172.21.15.32:0/3785991760 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7ff0fc024910 con 0x7ff10c105eb0 2024-04-25T00:23:09.454 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.562+0000 7ff1137d4700 1 -- 172.21.15.32:0/3785991760 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff10c19f010 con 0x7ff10c105eb0 2024-04-25T00:23:09.454 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.562+0000 7ff1027fc700 1 -- 172.21.15.32:0/3785991760 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 5) ==== 46287+0+0 (secure 0 0 0) 0x7ff0fc022020 con 0x7ff10c105eb0 2024-04-25T00:23:09.454 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.563+0000 7ff1027fc700 1 --2- 172.21.15.32:0/3785991760 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7ff0f8039050 0x7ff0f803b510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:09.455 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.563+0000 7ff110d6f700 1 -- 172.21.15.32:0/3785991760 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7ff0f8039050 msgr2=0x7ff0f803b510 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/1637256323 2024-04-25T00:23:09.455 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.563+0000 7ff1027fc700 1 -- 172.21.15.32:0/3785991760 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7ff0fc011e00 con 0x7ff10c105eb0 2024-04-25T00:23:09.455 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.563+0000 7ff110d6f700 1 --2- 172.21.15.32:0/3785991760 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7ff0f8039050 0x7ff0f803b510 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-25T00:23:09.455 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.563+0000 7ff1137d4700 1 -- 172.21.15.32:0/3785991760 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff10c063890 con 0x7ff10c105eb0 2024-04-25T00:23:09.455 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.568+0000 7ff1027fc700 1 -- 172.21.15.32:0/3785991760 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7ff0fc04ce20 con 0x7ff10c105eb0 2024-04-25T00:23:09.455 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.764+0000 7ff110d6f700 1 -- 172.21.15.32:0/3785991760 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7ff0f8039050 msgr2=0x7ff0f803b510 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/1637256323 2024-04-25T00:23:09.455 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.764+0000 7ff110d6f700 1 --2- 172.21.15.32:0/3785991760 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7ff0f8039050 0x7ff0f803b510 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-04-25T00:23:09.455 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.875+0000 7ff1137d4700 1 -- 172.21.15.32:0/3785991760 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "mgr stat"} v 0) -- 0x7ff10c19f5f0 con 0x7ff10c105eb0 2024-04-25T00:23:09.455 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.876+0000 7ff1027fc700 1 -- 172.21.15.32:0/3785991760 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "mgr stat"}]=0 v5) ==== 56+0+88 (secure 0 0 0) 0x7ff0fc04cab0 con 0x7ff10c105eb0 2024-04-25T00:23:09.455 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.881+0000 7ff1137d4700 1 -- 172.21.15.32:0/3785991760 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7ff0f8039050 msgr2=0x7ff0f803b510 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:23:09.455 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.881+0000 7ff1137d4700 1 --2- 172.21.15.32:0/3785991760 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7ff0f8039050 0x7ff0f803b510 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:09.455 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.881+0000 7ff1137d4700 1 -- 172.21.15.32:0/3785991760 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff10c105eb0 msgr2=0x7ff10c19bb50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:09.455 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.881+0000 7ff1137d4700 1 --2- 172.21.15.32:0/3785991760 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff10c105eb0 0x7ff10c19bb50 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7ff0fc000c00 tx=0x7ff0fc011c10 comp rx=0 tx=0).stop 2024-04-25T00:23:09.455 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.881+0000 7ff1137d4700 1 -- 172.21.15.32:0/3785991760 shutdown_connections 2024-04-25T00:23:09.456 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.881+0000 7ff1137d4700 1 --2- 172.21.15.32:0/3785991760 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7ff0f8039050 0x7ff0f803b510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:09.456 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.881+0000 7ff1137d4700 1 --2- 172.21.15.32:0/3785991760 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff10c105eb0 0x7ff10c19bb50 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:09.456 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.881+0000 7ff1137d4700 1 -- 172.21.15.32:0/3785991760 >> 172.21.15.32:0/3785991760 conn(0x7ff10c101830 msgr2=0x7ff10c102390 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:09.456 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.882+0000 7ff1137d4700 1 -- 172.21.15.32:0/3785991760 shutdown_connections 2024-04-25T00:23:09.456 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:08.882+0000 7ff1137d4700 1 -- 172.21.15.32:0/3785991760 wait complete. 2024-04-25T00:23:09.456 INFO:teuthology.orchestra.run.smithi032.stdout:Waiting for the mgr to restart... 2024-04-25T00:23:09.456 INFO:teuthology.orchestra.run.smithi032.stdout:Waiting for mgr epoch 5... 2024-04-25T00:23:09.693 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:09 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3785991760' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-25T00:23:09.693 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:09 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:09.682+0000 7fa024283200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-25T00:23:10.103 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:09 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:09.862+0000 7fa024283200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-25T00:23:10.103 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:10 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:10.040+0000 7fa024283200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-25T00:23:10.436 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:10 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:10.338+0000 7fa024283200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-25T00:23:10.436 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:10 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:10.435+0000 7fa024283200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-25T00:23:10.748 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:10 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:10.646+0000 7fa024283200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-25T00:23:11.248 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:10 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:10.846+0000 7fa024283200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-25T00:23:11.748 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:11 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:11.468+0000 7fa024283200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-25T00:23:12.212 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:11 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:11.906+0000 7fa024283200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-25T00:23:12.213 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:12 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:12.005+0000 7fa024283200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-25T00:23:12.213 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:12 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:12.212+0000 7fa024283200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-25T00:23:13.100 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:12 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:12.755+0000 7fa024283200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-25T00:23:13.100 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:12 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:12.849+0000 7fa024283200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-25T00:23:13.455 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:13 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:13.099+0000 7fa024283200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-25T00:23:13.455 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:13 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:13.237+0000 7fa024283200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-25T00:23:13.455 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:13 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:13.334+0000 7fa024283200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-25T00:23:13.455 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:13 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:13.454+0000 7fa024283200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-25T00:23:13.748 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:13 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:13.549+0000 7fa024283200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-25T00:23:14.665 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:14 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:14.390+0000 7fa024283200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-25T00:23:14.665 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:14 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:14.612+0000 7fa024283200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-25T00:23:14.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:14 smithi032 ceph-mon[29357]: Active manager daemon x restarted 2024-04-25T00:23:14.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:14 smithi032 ceph-mon[29357]: Activating manager daemon x 2024-04-25T00:23:14.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:14 smithi032 ceph-mon[29357]: osdmap e2: 0 total, 0 up, 0 in 2024-04-25T00:23:14.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:14 smithi032 ceph-mon[29357]: mgrmap e6: x(active, starting, since 0.00351897s) 2024-04-25T00:23:14.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:14 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-25T00:23:14.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:14 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-25T00:23:14.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:14 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-25T00:23:14.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:14 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-25T00:23:14.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:14 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-25T00:23:14.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:14 smithi032 ceph-mon[29357]: Manager daemon x is now available 2024-04-25T00:23:14.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:14 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' 2024-04-25T00:23:14.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:14 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' 2024-04-25T00:23:14.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:14 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:23:14.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:14 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:23:14.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:14 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-25T00:23:15.942 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:15 smithi032 ceph-mon[29357]: Found migration_current of "None". Setting to last migration. 2024-04-25T00:23:15.942 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:15 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-25T00:23:15.942 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:15 smithi032 ceph-mon[29357]: mgrmap e7: x(active, since 1.00537s) 2024-04-25T00:23:15.942 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:15 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' 2024-04-25T00:23:15.942 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:15 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' 2024-04-25T00:23:16.194 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout { 2024-04-25T00:23:16.194 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-04-25T00:23:16.194 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-25T00:23:16.194 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout } 2024-04-25T00:23:16.194 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.897+0000 7fcbc1bef700 1 Processor -- start 2024-04-25T00:23:16.194 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.897+0000 7fcbc1bef700 1 -- start start 2024-04-25T00:23:16.195 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.897+0000 7fcbc1bef700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcbbc107c50 0x7fcbbc108070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:16.195 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.897+0000 7fcbc1bef700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fcbbc108640 con 0x7fcbbc107c50 2024-04-25T00:23:16.195 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.898+0000 7fcbbb7fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcbbc107c50 0x7fcbbc108070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:16.195 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.898+0000 7fcbbb7fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcbbc107c50 0x7fcbbc108070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:52004/0 (socket says 172.21.15.32:52004) 2024-04-25T00:23:16.195 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.898+0000 7fcbbb7fe700 1 -- 172.21.15.32:0/2660903909 learned_addr learned my addr 172.21.15.32:0/2660903909 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:16.195 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.898+0000 7fcbbb7fe700 1 -- 172.21.15.32:0/2660903909 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fcbbc108db0 con 0x7fcbbc107c50 2024-04-25T00:23:16.195 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.898+0000 7fcbbb7fe700 1 --2- 172.21.15.32:0/2660903909 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcbbc107c50 0x7fcbbc108070 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7fcbb00098a0 tx=0x7fcbb0009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ac09e89bcce78d1b server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:16.195 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.898+0000 7fcbba7fc700 1 -- 172.21.15.32:0/2660903909 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fcbb0004160 con 0x7fcbbc107c50 2024-04-25T00:23:16.195 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.898+0000 7fcbba7fc700 1 -- 172.21.15.32:0/2660903909 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7fcbb0004300 con 0x7fcbbc107c50 2024-04-25T00:23:16.195 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.898+0000 7fcbba7fc700 1 -- 172.21.15.32:0/2660903909 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fcbb0011b50 con 0x7fcbbc107c50 2024-04-25T00:23:16.195 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.899+0000 7fcbc1bef700 1 -- 172.21.15.32:0/2660903909 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcbbc107c50 msgr2=0x7fcbbc108070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:16.195 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.899+0000 7fcbc1bef700 1 --2- 172.21.15.32:0/2660903909 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcbbc107c50 0x7fcbbc108070 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7fcbb00098a0 tx=0x7fcbb0009bb0 comp rx=0 tx=0).stop 2024-04-25T00:23:16.195 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.899+0000 7fcbc1bef700 1 -- 172.21.15.32:0/2660903909 shutdown_connections 2024-04-25T00:23:16.196 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.899+0000 7fcbc1bef700 1 --2- 172.21.15.32:0/2660903909 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcbbc107c50 0x7fcbbc108070 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:16.196 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.899+0000 7fcbc1bef700 1 -- 172.21.15.32:0/2660903909 >> 172.21.15.32:0/2660903909 conn(0x7fcbbc103230 msgr2=0x7fcbbc105650 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:16.196 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.899+0000 7fcbc1bef700 1 -- 172.21.15.32:0/2660903909 shutdown_connections 2024-04-25T00:23:16.196 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.899+0000 7fcbc1bef700 1 -- 172.21.15.32:0/2660903909 wait complete. 2024-04-25T00:23:16.196 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.900+0000 7fcbc1bef700 1 Processor -- start 2024-04-25T00:23:16.196 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.900+0000 7fcbc1bef700 1 -- start start 2024-04-25T00:23:16.196 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.900+0000 7fcbc1bef700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcbbc107c50 0x7fcbbc19bb80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:16.196 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.900+0000 7fcbc1bef700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fcbbc10c160 con 0x7fcbbc107c50 2024-04-25T00:23:16.196 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.900+0000 7fcbbb7fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcbbc107c50 0x7fcbbc19bb80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:16.196 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.900+0000 7fcbbb7fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcbbc107c50 0x7fcbbc19bb80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:52012/0 (socket says 172.21.15.32:52012) 2024-04-25T00:23:16.196 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.900+0000 7fcbbb7fe700 1 -- 172.21.15.32:0/2101866223 learned_addr learned my addr 172.21.15.32:0/2101866223 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:16.196 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.901+0000 7fcbbb7fe700 1 -- 172.21.15.32:0/2101866223 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fcbbc19c0c0 con 0x7fcbbc107c50 2024-04-25T00:23:16.196 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.901+0000 7fcbbb7fe700 1 --2- 172.21.15.32:0/2101866223 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcbbc107c50 0x7fcbbc19bb80 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7fcbb0009fb0 tx=0x7fcbb0011bf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:16.196 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.901+0000 7fcbb8ff9700 1 -- 172.21.15.32:0/2101866223 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fcbb00038b0 con 0x7fcbbc107c50 2024-04-25T00:23:16.197 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.901+0000 7fcbb8ff9700 1 -- 172.21.15.32:0/2101866223 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7fcbb0011e50 con 0x7fcbbc107c50 2024-04-25T00:23:16.197 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.901+0000 7fcbc1bef700 1 -- 172.21.15.32:0/2101866223 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fcbbc19c350 con 0x7fcbbc107c50 2024-04-25T00:23:16.197 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.901+0000 7fcbb8ff9700 1 -- 172.21.15.32:0/2101866223 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fcbb0019550 con 0x7fcbbc107c50 2024-04-25T00:23:16.197 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.901+0000 7fcbc1bef700 1 -- 172.21.15.32:0/2101866223 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fcbbc19f040 con 0x7fcbbc107c50 2024-04-25T00:23:16.197 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.902+0000 7fcbb8ff9700 1 -- 172.21.15.32:0/2101866223 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 5) ==== 46287+0+0 (secure 0 0 0) 0x7fcbb0019800 con 0x7fcbbc107c50 2024-04-25T00:23:16.197 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.902+0000 7fcbb8ff9700 1 --2- 172.21.15.32:0/2101866223 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7fcba4039050 0x7fcba403b510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:16.197 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.902+0000 7fcbb8ff9700 1 -- 172.21.15.32:0/2101866223 --> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x7fcba403bc20 con 0x7fcba4039050 2024-04-25T00:23:16.197 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.902+0000 7fcbb8ff9700 1 -- 172.21.15.32:0/2101866223 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7fcbb004d210 con 0x7fcbbc107c50 2024-04-25T00:23:16.197 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.902+0000 7fcbbaffd700 1 -- 172.21.15.32:0/2101866223 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7fcba4039050 msgr2=0x7fcba403b510 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/1637256323 2024-04-25T00:23:16.197 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:09.902+0000 7fcbbaffd700 1 --2- 172.21.15.32:0/2101866223 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7fcba4039050 0x7fcba403b510 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-25T00:23:16.197 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:10.103+0000 7fcbbaffd700 1 -- 172.21.15.32:0/2101866223 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7fcba4039050 msgr2=0x7fcba403b510 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/1637256323 2024-04-25T00:23:16.197 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:10.103+0000 7fcbbaffd700 1 --2- 172.21.15.32:0/2101866223 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7fcba4039050 0x7fcba403b510 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-04-25T00:23:16.197 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:10.504+0000 7fcbbaffd700 1 -- 172.21.15.32:0/2101866223 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7fcba4039050 msgr2=0x7fcba403b510 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/1637256323 2024-04-25T00:23:16.197 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:10.504+0000 7fcbbaffd700 1 --2- 172.21.15.32:0/2101866223 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7fcba4039050 0x7fcba403b510 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2024-04-25T00:23:16.197 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:11.305+0000 7fcbbaffd700 1 -- 172.21.15.32:0/2101866223 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7fcba4039050 msgr2=0x7fcba403b510 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/1637256323 2024-04-25T00:23:16.198 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:11.305+0000 7fcbbaffd700 1 --2- 172.21.15.32:0/2101866223 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7fcba4039050 0x7fcba403b510 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 1.600000 2024-04-25T00:23:16.198 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:12.906+0000 7fcbbaffd700 1 -- 172.21.15.32:0/2101866223 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7fcba4039050 msgr2=0x7fcba403b510 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/1637256323 2024-04-25T00:23:16.198 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:12.906+0000 7fcbbaffd700 1 --2- 172.21.15.32:0/2101866223 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7fcba4039050 0x7fcba403b510 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 3.200000 2024-04-25T00:23:16.198 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:14.615+0000 7fcbb8ff9700 1 -- 172.21.15.32:0/2101866223 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mgrmap(e 6) ==== 46054+0+0 (secure 0 0 0) 0x7fcbb0019b40 con 0x7fcbbc107c50 2024-04-25T00:23:16.198 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:14.615+0000 7fcbb8ff9700 1 -- 172.21.15.32:0/2101866223 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7fcba4039050 msgr2=0x7fcba403b510 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:23:16.198 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:14.615+0000 7fcbb8ff9700 1 --2- 172.21.15.32:0/2101866223 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7fcba4039050 0x7fcba403b510 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:16.198 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:15.616+0000 7fcbb8ff9700 1 -- 172.21.15.32:0/2101866223 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mgrmap(e 7) ==== 46181+0+0 (secure 0 0 0) 0x7fcbb0020ca0 con 0x7fcbbc107c50 2024-04-25T00:23:16.198 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:15.616+0000 7fcbb8ff9700 1 --2- 172.21.15.32:0/2101866223 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7fcba403c980 0x7fcba403ed70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:16.198 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:15.616+0000 7fcbb8ff9700 1 -- 172.21.15.32:0/2101866223 --> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x7fcbb0003bf0 con 0x7fcba403c980 2024-04-25T00:23:16.198 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:15.617+0000 7fcbbaffd700 1 --2- 172.21.15.32:0/2101866223 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7fcba403c980 0x7fcba403ed70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:16.198 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:15.617+0000 7fcbbaffd700 1 --2- 172.21.15.32:0/2101866223 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7fcba403c980 0x7fcba403ed70 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7fcbac003a10 tx=0x7fcbac0092b0 comp rx=0 tx=0).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:16.198 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:15.618+0000 7fcbb8ff9700 1 -- 172.21.15.32:0/2101866223 <== mgr.14116 v2:172.21.15.32:6800/4160192081 1 ==== command_reply(tid 0: 0 ) ==== 8+0+7774 (secure 0 0 0) 0x7fcbb0003bf0 con 0x7fcba403c980 2024-04-25T00:23:16.198 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:15.625+0000 7fcbc1bef700 1 -- 172.21.15.32:0/2101866223 --> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] -- command(tid 1: {"prefix": "mgr_status"}) -- 0x7fcbbc063890 con 0x7fcba403c980 2024-04-25T00:23:16.198 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:15.625+0000 7fcbb8ff9700 1 -- 172.21.15.32:0/2101866223 <== mgr.14116 v2:172.21.15.32:6800/4160192081 2 ==== command_reply(tid 1: 0 ) ==== 8+0+51 (secure 0 0 0) 0x7fcbbc063890 con 0x7fcba403c980 2024-04-25T00:23:16.199 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:15.625+0000 7fcbc1bef700 1 -- 172.21.15.32:0/2101866223 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7fcba403c980 msgr2=0x7fcba403ed70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:16.199 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:15.625+0000 7fcbc1bef700 1 --2- 172.21.15.32:0/2101866223 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7fcba403c980 0x7fcba403ed70 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7fcbac003a10 tx=0x7fcbac0092b0 comp rx=0 tx=0).stop 2024-04-25T00:23:16.199 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:15.625+0000 7fcbc1bef700 1 -- 172.21.15.32:0/2101866223 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcbbc107c50 msgr2=0x7fcbbc19bb80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:16.199 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:15.625+0000 7fcbc1bef700 1 --2- 172.21.15.32:0/2101866223 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcbbc107c50 0x7fcbbc19bb80 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7fcbb0009fb0 tx=0x7fcbb0011bf0 comp rx=0 tx=0).stop 2024-04-25T00:23:16.199 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:15.625+0000 7fcbc1bef700 1 -- 172.21.15.32:0/2101866223 shutdown_connections 2024-04-25T00:23:16.199 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:15.625+0000 7fcbc1bef700 1 --2- 172.21.15.32:0/2101866223 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7fcba403c980 0x7fcba403ed70 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:16.199 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:15.625+0000 7fcbc1bef700 1 --2- 172.21.15.32:0/2101866223 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcbbc107c50 0x7fcbbc19bb80 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:16.199 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:15.625+0000 7fcbc1bef700 1 --2- 172.21.15.32:0/2101866223 >> [v2:172.21.15.32:6800/1637256323,v1:172.21.15.32:6801/1637256323] conn(0x7fcba4039050 0x7fcba403b510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:16.199 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:15.625+0000 7fcbc1bef700 1 -- 172.21.15.32:0/2101866223 >> 172.21.15.32:0/2101866223 conn(0x7fcbbc103230 msgr2=0x7fcbbc103d90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:16.199 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:15.625+0000 7fcbc1bef700 1 -- 172.21.15.32:0/2101866223 shutdown_connections 2024-04-25T00:23:16.199 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:15.625+0000 7fcbc1bef700 1 -- 172.21.15.32:0/2101866223 wait complete. 2024-04-25T00:23:16.199 INFO:teuthology.orchestra.run.smithi032.stdout:mgr epoch 5 is available 2024-04-25T00:23:16.199 INFO:teuthology.orchestra.run.smithi032.stdout:Setting orchestrator backend to cephadm... 2024-04-25T00:23:16.876 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:16 smithi032 ceph-mon[29357]: from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-25T00:23:16.876 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:16 smithi032 ceph-mon[29357]: from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-25T00:23:16.876 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:16 smithi032 ceph-mon[29357]: [25/Apr/2024:00:23:16] ENGINE Bus STARTING 2024-04-25T00:23:16.876 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:16 smithi032 ceph-mon[29357]: [25/Apr/2024:00:23:16] ENGINE Serving on http://172.21.15.32:8765 2024-04-25T00:23:16.876 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:16 smithi032 ceph-mon[29357]: [25/Apr/2024:00:23:16] ENGINE Serving on https://172.21.15.32:7150 2024-04-25T00:23:16.876 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:16 smithi032 ceph-mon[29357]: [25/Apr/2024:00:23:16] ENGINE Bus STARTED 2024-04-25T00:23:16.876 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:16 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:23:17.424 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.639+0000 7eff3ace1700 1 Processor -- start 2024-04-25T00:23:17.424 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.639+0000 7eff3ace1700 1 -- start start 2024-04-25T00:23:17.424 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.639+0000 7eff3ace1700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff34107bd0 0x7eff34107ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:17.424 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.639+0000 7eff3ace1700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7eff341085c0 con 0x7eff34107bd0 2024-04-25T00:23:17.424 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.639+0000 7eff38a7d700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff34107bd0 0x7eff34107ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:17.424 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.640+0000 7eff38a7d700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff34107bd0 0x7eff34107ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40062/0 (socket says 172.21.15.32:40062) 2024-04-25T00:23:17.425 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.640+0000 7eff38a7d700 1 -- 172.21.15.32:0/3065930529 learned_addr learned my addr 172.21.15.32:0/3065930529 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:17.425 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.640+0000 7eff38a7d700 1 -- 172.21.15.32:0/3065930529 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7eff34108d80 con 0x7eff34107bd0 2024-04-25T00:23:17.425 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.640+0000 7eff38a7d700 1 --2- 172.21.15.32:0/3065930529 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff34107bd0 0x7eff34107ff0 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7eff2c009e80 tx=0x7eff2c00b330 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4f5fef68893f05b9 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:17.425 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.640+0000 7eff337fe700 1 -- 172.21.15.32:0/3065930529 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7eff2c017070 con 0x7eff34107bd0 2024-04-25T00:23:17.425 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.640+0000 7eff337fe700 1 -- 172.21.15.32:0/3065930529 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7eff2c00b8c0 con 0x7eff34107bd0 2024-04-25T00:23:17.425 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.640+0000 7eff337fe700 1 -- 172.21.15.32:0/3065930529 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7eff2c0045c0 con 0x7eff34107bd0 2024-04-25T00:23:17.425 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.641+0000 7eff3ace1700 1 -- 172.21.15.32:0/3065930529 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff34107bd0 msgr2=0x7eff34107ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:17.425 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.641+0000 7eff3ace1700 1 --2- 172.21.15.32:0/3065930529 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff34107bd0 0x7eff34107ff0 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7eff2c009e80 tx=0x7eff2c00b330 comp rx=0 tx=0).stop 2024-04-25T00:23:17.425 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.641+0000 7eff3ace1700 1 -- 172.21.15.32:0/3065930529 shutdown_connections 2024-04-25T00:23:17.425 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.641+0000 7eff3ace1700 1 --2- 172.21.15.32:0/3065930529 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff34107bd0 0x7eff34107ff0 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:17.425 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.641+0000 7eff3ace1700 1 -- 172.21.15.32:0/3065930529 >> 172.21.15.32:0/3065930529 conn(0x7eff34103150 msgr2=0x7eff341055b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:17.425 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.641+0000 7eff3ace1700 1 -- 172.21.15.32:0/3065930529 shutdown_connections 2024-04-25T00:23:17.425 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.641+0000 7eff3ace1700 1 -- 172.21.15.32:0/3065930529 wait complete. 2024-04-25T00:23:17.425 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.641+0000 7eff3ace1700 1 Processor -- start 2024-04-25T00:23:17.426 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.641+0000 7eff3ace1700 1 -- start start 2024-04-25T00:23:17.426 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.642+0000 7eff3ace1700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff34107bd0 0x7eff3419bb10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:17.426 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.642+0000 7eff3ace1700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7eff3410c130 con 0x7eff34107bd0 2024-04-25T00:23:17.426 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.642+0000 7eff38a7d700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff34107bd0 0x7eff3419bb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:17.426 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.642+0000 7eff38a7d700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff34107bd0 0x7eff3419bb10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40070/0 (socket says 172.21.15.32:40070) 2024-04-25T00:23:17.426 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.642+0000 7eff38a7d700 1 -- 172.21.15.32:0/269229599 learned_addr learned my addr 172.21.15.32:0/269229599 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:17.426 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.642+0000 7eff38a7d700 1 -- 172.21.15.32:0/269229599 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7eff3419c050 con 0x7eff34107bd0 2024-04-25T00:23:17.426 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.642+0000 7eff38a7d700 1 --2- 172.21.15.32:0/269229599 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff34107bd0 0x7eff3419bb10 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7eff2c0072a0 tx=0x7eff2c00bd50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:17.426 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.642+0000 7eff31ffb700 1 -- 172.21.15.32:0/269229599 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7eff2c020020 con 0x7eff34107bd0 2024-04-25T00:23:17.426 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.642+0000 7eff3ace1700 1 -- 172.21.15.32:0/269229599 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7eff3419c2e0 con 0x7eff34107bd0 2024-04-25T00:23:17.426 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.642+0000 7eff31ffb700 1 -- 172.21.15.32:0/269229599 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7eff2c003b50 con 0x7eff34107bd0 2024-04-25T00:23:17.426 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.642+0000 7eff31ffb700 1 -- 172.21.15.32:0/269229599 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7eff2c017040 con 0x7eff34107bd0 2024-04-25T00:23:17.426 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.643+0000 7eff3ace1700 1 -- 172.21.15.32:0/269229599 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7eff3419efd0 con 0x7eff34107bd0 2024-04-25T00:23:17.427 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.643+0000 7eff3ace1700 1 -- 172.21.15.32:0/269229599 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7eff34063890 con 0x7eff34107bd0 2024-04-25T00:23:17.427 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.643+0000 7eff31ffb700 1 -- 172.21.15.32:0/269229599 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 7) ==== 46181+0+0 (secure 0 0 0) 0x7eff2c023020 con 0x7eff34107bd0 2024-04-25T00:23:17.427 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.644+0000 7eff31ffb700 1 --2- 172.21.15.32:0/269229599 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7eff1c038ed0 0x7eff1c03b390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:17.427 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.644+0000 7eff31ffb700 1 -- 172.21.15.32:0/269229599 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7eff2c00f050 con 0x7eff34107bd0 2024-04-25T00:23:17.427 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.644+0000 7eff33fff700 1 --2- 172.21.15.32:0/269229599 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7eff1c038ed0 0x7eff1c03b390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:17.427 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.644+0000 7eff33fff700 1 --2- 172.21.15.32:0/269229599 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7eff1c038ed0 0x7eff1c03b390 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7eff28006fd0 tx=0x7eff28006e40 comp rx=0 tx=0).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:17.427 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.649+0000 7eff31ffb700 1 -- 172.21.15.32:0/269229599 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7eff2c052020 con 0x7eff34107bd0 2024-04-25T00:23:17.427 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.667+0000 7eff31ffb700 1 -- 172.21.15.32:0/269229599 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mgrmap(e 8) ==== 46287+0+0 (secure 0 0 0) 0x7eff2c012980 con 0x7eff34107bd0 2024-04-25T00:23:17.427 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.876+0000 7eff3ace1700 1 -- 172.21.15.32:0/269229599 --> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] -- mgr_command(tid 0: {"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}) -- 0x7eff341043a0 con 0x7eff1c038ed0 2024-04-25T00:23:17.427 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.880+0000 7eff31ffb700 1 -- 172.21.15.32:0/269229599 <== mgr.14116 v2:172.21.15.32:6800/4160192081 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+0 (secure 0 0 0) 0x7eff341043a0 con 0x7eff1c038ed0 2024-04-25T00:23:17.427 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.883+0000 7eff3ace1700 1 -- 172.21.15.32:0/269229599 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7eff1c038ed0 msgr2=0x7eff1c03b390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:17.427 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.883+0000 7eff3ace1700 1 --2- 172.21.15.32:0/269229599 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7eff1c038ed0 0x7eff1c03b390 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7eff28006fd0 tx=0x7eff28006e40 comp rx=0 tx=0).stop 2024-04-25T00:23:17.428 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.883+0000 7eff3ace1700 1 -- 172.21.15.32:0/269229599 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff34107bd0 msgr2=0x7eff3419bb10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:17.428 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.883+0000 7eff3ace1700 1 --2- 172.21.15.32:0/269229599 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff34107bd0 0x7eff3419bb10 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7eff2c0072a0 tx=0x7eff2c00bd50 comp rx=0 tx=0).stop 2024-04-25T00:23:17.428 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.883+0000 7eff3ace1700 1 -- 172.21.15.32:0/269229599 shutdown_connections 2024-04-25T00:23:17.428 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.883+0000 7eff3ace1700 1 --2- 172.21.15.32:0/269229599 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7eff1c038ed0 0x7eff1c03b390 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:17.428 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.883+0000 7eff3ace1700 1 --2- 172.21.15.32:0/269229599 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff34107bd0 0x7eff3419bb10 unknown :-1 s=CLOSED pgs=34 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:17.428 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.883+0000 7eff3ace1700 1 -- 172.21.15.32:0/269229599 >> 172.21.15.32:0/269229599 conn(0x7eff34103150 msgr2=0x7eff34103c90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:17.428 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.883+0000 7eff3ace1700 1 -- 172.21.15.32:0/269229599 shutdown_connections 2024-04-25T00:23:17.428 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:16.883+0000 7eff3ace1700 1 -- 172.21.15.32:0/269229599 wait complete. 2024-04-25T00:23:17.834 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:17 smithi032 ceph-mon[29357]: mgrmap e8: x(active, since 2s) 2024-04-25T00:23:17.834 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:17 smithi032 ceph-mon[29357]: from='client.14128 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:23:17.834 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:17 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' 2024-04-25T00:23:17.834 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:17 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:23:18.647 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout value unchanged 2024-04-25T00:23:18.647 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.867+0000 7fec7c6a8700 1 Processor -- start 2024-04-25T00:23:18.647 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.868+0000 7fec7c6a8700 1 -- start start 2024-04-25T00:23:18.647 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.868+0000 7fec7c6a8700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fec7407de50 0x7fec7407c300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:18.647 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.868+0000 7fec7c6a8700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fec7407c840 con 0x7fec7407de50 2024-04-25T00:23:18.647 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.868+0000 7fec7a444700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fec7407de50 0x7fec7407c300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:18.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.868+0000 7fec7a444700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fec7407de50 0x7fec7407c300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40084/0 (socket says 172.21.15.32:40084) 2024-04-25T00:23:18.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.868+0000 7fec7a444700 1 -- 172.21.15.32:0/1824642331 learned_addr learned my addr 172.21.15.32:0/1824642331 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:18.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.868+0000 7fec7a444700 1 -- 172.21.15.32:0/1824642331 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fec7407c9c0 con 0x7fec7407de50 2024-04-25T00:23:18.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.868+0000 7fec7a444700 1 --2- 172.21.15.32:0/1824642331 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fec7407de50 0x7fec7407c300 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7fec700098a0 tx=0x7fec70009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8d09a508b43f6a2b server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:18.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.869+0000 7fec79442700 1 -- 172.21.15.32:0/1824642331 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fec70004160 con 0x7fec7407de50 2024-04-25T00:23:18.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.869+0000 7fec79442700 1 -- 172.21.15.32:0/1824642331 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7fec70004300 con 0x7fec7407de50 2024-04-25T00:23:18.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.869+0000 7fec79442700 1 -- 172.21.15.32:0/1824642331 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fec70011b50 con 0x7fec7407de50 2024-04-25T00:23:18.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.869+0000 7fec7c6a8700 1 -- 172.21.15.32:0/1824642331 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fec7407de50 msgr2=0x7fec7407c300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:18.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.869+0000 7fec7c6a8700 1 --2- 172.21.15.32:0/1824642331 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fec7407de50 0x7fec7407c300 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7fec700098a0 tx=0x7fec70009bb0 comp rx=0 tx=0).stop 2024-04-25T00:23:18.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.869+0000 7fec7c6a8700 1 -- 172.21.15.32:0/1824642331 shutdown_connections 2024-04-25T00:23:18.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.869+0000 7fec7c6a8700 1 --2- 172.21.15.32:0/1824642331 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fec7407de50 0x7fec7407c300 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:18.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.869+0000 7fec7c6a8700 1 -- 172.21.15.32:0/1824642331 >> 172.21.15.32:0/1824642331 conn(0x7fec74103170 msgr2=0x7fec741055d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:18.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.869+0000 7fec7c6a8700 1 -- 172.21.15.32:0/1824642331 shutdown_connections 2024-04-25T00:23:18.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.869+0000 7fec7c6a8700 1 -- 172.21.15.32:0/1824642331 wait complete. 2024-04-25T00:23:18.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.870+0000 7fec7c6a8700 1 Processor -- start 2024-04-25T00:23:18.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.870+0000 7fec7c6a8700 1 -- start start 2024-04-25T00:23:18.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.870+0000 7fec7c6a8700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fec7407de50 0x7fec7419dd30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:18.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.870+0000 7fec7c6a8700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fec7410e1c0 con 0x7fec7407de50 2024-04-25T00:23:18.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.870+0000 7fec7a444700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fec7407de50 0x7fec7419dd30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:18.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.870+0000 7fec7a444700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fec7407de50 0x7fec7419dd30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40094/0 (socket says 172.21.15.32:40094) 2024-04-25T00:23:18.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.870+0000 7fec7a444700 1 -- 172.21.15.32:0/1776993229 learned_addr learned my addr 172.21.15.32:0/1776993229 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:18.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.871+0000 7fec7a444700 1 -- 172.21.15.32:0/1776993229 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fec7419e270 con 0x7fec7407de50 2024-04-25T00:23:18.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.871+0000 7fec7a444700 1 --2- 172.21.15.32:0/1776993229 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fec7407de50 0x7fec7419dd30 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fec70011670 tx=0x7fec70011b60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:18.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.871+0000 7fec677fe700 1 -- 172.21.15.32:0/1776993229 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fec700039c0 con 0x7fec7407de50 2024-04-25T00:23:18.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.871+0000 7fec7c6a8700 1 -- 172.21.15.32:0/1776993229 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fec7419e500 con 0x7fec7407de50 2024-04-25T00:23:18.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.871+0000 7fec677fe700 1 -- 172.21.15.32:0/1776993229 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7fec70011e50 con 0x7fec7407de50 2024-04-25T00:23:18.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.871+0000 7fec677fe700 1 -- 172.21.15.32:0/1776993229 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fec7001a5c0 con 0x7fec7407de50 2024-04-25T00:23:18.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.871+0000 7fec7c6a8700 1 -- 172.21.15.32:0/1776993229 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fec741a11f0 con 0x7fec7407de50 2024-04-25T00:23:18.650 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.872+0000 7fec677fe700 1 -- 172.21.15.32:0/1776993229 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 8) ==== 46287+0+0 (secure 0 0 0) 0x7fec7001a8c0 con 0x7fec7407de50 2024-04-25T00:23:18.650 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.872+0000 7fec7c6a8700 1 -- 172.21.15.32:0/1776993229 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fec74063890 con 0x7fec7407de50 2024-04-25T00:23:18.650 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.872+0000 7fec677fe700 1 --2- 172.21.15.32:0/1776993229 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7fec60039000 0x7fec6003b4c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:18.650 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.872+0000 7fec677fe700 1 -- 172.21.15.32:0/1776993229 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7fec7004d590 con 0x7fec7407de50 2024-04-25T00:23:18.650 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.872+0000 7fec79c43700 1 --2- 172.21.15.32:0/1776993229 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7fec60039000 0x7fec6003b4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:18.650 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.873+0000 7fec79c43700 1 --2- 172.21.15.32:0/1776993229 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7fec60039000 0x7fec6003b4c0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fec68006fd0 tx=0x7fec68006e40 comp rx=0 tx=0).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:18.650 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:17.877+0000 7fec677fe700 1 -- 172.21.15.32:0/1776993229 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fec7002d910 con 0x7fec7407de50 2024-04-25T00:23:18.650 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:18.088+0000 7fec7c6a8700 1 -- 172.21.15.32:0/1776993229 --> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] -- mgr_command(tid 0: {"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}) -- 0x7fec74104430 con 0x7fec60039000 2024-04-25T00:23:18.650 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:18.088+0000 7fec677fe700 1 -- 172.21.15.32:0/1776993229 <== mgr.14116 v2:172.21.15.32:6800/4160192081 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+16 (secure 0 0 0) 0x7fec74104430 con 0x7fec60039000 2024-04-25T00:23:18.650 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:18.090+0000 7fec7c6a8700 1 -- 172.21.15.32:0/1776993229 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7fec60039000 msgr2=0x7fec6003b4c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:18.651 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:18.090+0000 7fec7c6a8700 1 --2- 172.21.15.32:0/1776993229 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7fec60039000 0x7fec6003b4c0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fec68006fd0 tx=0x7fec68006e40 comp rx=0 tx=0).stop 2024-04-25T00:23:18.651 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:18.090+0000 7fec7c6a8700 1 -- 172.21.15.32:0/1776993229 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fec7407de50 msgr2=0x7fec7419dd30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:18.651 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:18.090+0000 7fec7c6a8700 1 --2- 172.21.15.32:0/1776993229 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fec7407de50 0x7fec7419dd30 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fec70011670 tx=0x7fec70011b60 comp rx=0 tx=0).stop 2024-04-25T00:23:18.651 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:18.090+0000 7fec7c6a8700 1 -- 172.21.15.32:0/1776993229 shutdown_connections 2024-04-25T00:23:18.651 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:18.090+0000 7fec7c6a8700 1 --2- 172.21.15.32:0/1776993229 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7fec60039000 0x7fec6003b4c0 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:18.651 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:18.090+0000 7fec7c6a8700 1 --2- 172.21.15.32:0/1776993229 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fec7407de50 0x7fec7419dd30 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:18.651 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:18.090+0000 7fec7c6a8700 1 -- 172.21.15.32:0/1776993229 >> 172.21.15.32:0/1776993229 conn(0x7fec74103170 msgr2=0x7fec74103d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:18.651 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:18.091+0000 7fec7c6a8700 1 -- 172.21.15.32:0/1776993229 shutdown_connections 2024-04-25T00:23:18.651 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:18.091+0000 7fec7c6a8700 1 -- 172.21.15.32:0/1776993229 wait complete. 2024-04-25T00:23:18.651 INFO:teuthology.orchestra.run.smithi032.stdout:Generating ssh key... 2024-04-25T00:23:18.940 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:18 smithi032 ceph-mon[29357]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:23:19.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:19 smithi032 ceph-mon[29357]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:23:19.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:19 smithi032 ceph-mon[29357]: Generating ssh key... 2024-04-25T00:23:20.418 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:20 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: Generating public/private rsa key pair. 2024-04-25T00:23:20.418 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:20 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: Your identification has been saved in /tmp/tmpd3tlr6s8/key. 2024-04-25T00:23:20.418 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:20 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: Your public key has been saved in /tmp/tmpd3tlr6s8/key.pub. 2024-04-25T00:23:20.418 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:20 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: The key fingerprint is: 2024-04-25T00:23:20.419 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:20 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: SHA256:D6hoGtyqDUqmrLvVEmHhAXZV7ECwLTYz+kx1lPAfW8k ceph-cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:23:20.419 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:20 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: The key's randomart image is: 2024-04-25T00:23:20.419 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:20 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: +---[RSA 3072]----+ 2024-04-25T00:23:20.419 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:20 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: |.ooo+++o. | 2024-04-25T00:23:20.419 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:20 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: |...oo.oo . . | 2024-04-25T00:23:20.419 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:20 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: | +B ooo . E | 2024-04-25T00:23:20.419 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:20 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: | .o.* .o. + | 2024-04-25T00:23:20.419 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:20 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: | ... . So | 2024-04-25T00:23:20.419 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:20 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: |. =+ . o | 2024-04-25T00:23:20.419 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:20 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: |o+=+o . | 2024-04-25T00:23:20.419 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:20 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: |*B.. | 2024-04-25T00:23:20.419 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:20 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: |@=. | 2024-04-25T00:23:20.420 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:20 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: +----[SHA256]-----+ 2024-04-25T00:23:20.686 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.047+0000 7f8152ef3700 1 Processor -- start 2024-04-25T00:23:20.686 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.047+0000 7f8152ef3700 1 -- start start 2024-04-25T00:23:20.686 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.048+0000 7f8152ef3700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f814c106ae0 0x7f814c106f00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:20.686 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.048+0000 7f8152ef3700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f814c1074d0 con 0x7f814c106ae0 2024-04-25T00:23:20.686 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.048+0000 7f8150c8f700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f814c106ae0 0x7f814c106f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:20.686 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.048+0000 7f8150c8f700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f814c106ae0 0x7f814c106f00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40106/0 (socket says 172.21.15.32:40106) 2024-04-25T00:23:20.686 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.048+0000 7f8150c8f700 1 -- 172.21.15.32:0/2327702510 learned_addr learned my addr 172.21.15.32:0/2327702510 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:20.686 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.048+0000 7f8150c8f700 1 -- 172.21.15.32:0/2327702510 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f814c107ce0 con 0x7f814c106ae0 2024-04-25T00:23:20.686 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.048+0000 7f8150c8f700 1 --2- 172.21.15.32:0/2327702510 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f814c106ae0 0x7f814c106f00 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f81400098a0 tx=0x7f8140009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cef78bc600181e05 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:20.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.049+0000 7f814b7fe700 1 -- 172.21.15.32:0/2327702510 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f8140004160 con 0x7f814c106ae0 2024-04-25T00:23:20.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.049+0000 7f814b7fe700 1 -- 172.21.15.32:0/2327702510 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f8140004300 con 0x7f814c106ae0 2024-04-25T00:23:20.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.049+0000 7f814b7fe700 1 -- 172.21.15.32:0/2327702510 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f8140011b50 con 0x7f814c106ae0 2024-04-25T00:23:20.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.049+0000 7f8152ef3700 1 -- 172.21.15.32:0/2327702510 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f814c106ae0 msgr2=0x7f814c106f00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:20.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.049+0000 7f8152ef3700 1 --2- 172.21.15.32:0/2327702510 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f814c106ae0 0x7f814c106f00 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f81400098a0 tx=0x7f8140009bb0 comp rx=0 tx=0).stop 2024-04-25T00:23:20.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.049+0000 7f8152ef3700 1 -- 172.21.15.32:0/2327702510 shutdown_connections 2024-04-25T00:23:20.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.049+0000 7f8152ef3700 1 --2- 172.21.15.32:0/2327702510 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f814c106ae0 0x7f814c106f00 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:20.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.049+0000 7f8152ef3700 1 -- 172.21.15.32:0/2327702510 >> 172.21.15.32:0/2327702510 conn(0x7f814c1020a0 msgr2=0x7f814c1044c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:20.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.049+0000 7f8152ef3700 1 -- 172.21.15.32:0/2327702510 shutdown_connections 2024-04-25T00:23:20.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.049+0000 7f8152ef3700 1 -- 172.21.15.32:0/2327702510 wait complete. 2024-04-25T00:23:20.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.050+0000 7f8152ef3700 1 Processor -- start 2024-04-25T00:23:20.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.050+0000 7f8152ef3700 1 -- start start 2024-04-25T00:23:20.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.050+0000 7f8152ef3700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f814c106ae0 0x7f814c19bba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:20.688 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.050+0000 7f8152ef3700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f814c10b090 con 0x7f814c106ae0 2024-04-25T00:23:20.688 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.050+0000 7f8150c8f700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f814c106ae0 0x7f814c19bba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:20.688 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.050+0000 7f8150c8f700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f814c106ae0 0x7f814c19bba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40114/0 (socket says 172.21.15.32:40114) 2024-04-25T00:23:20.688 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.050+0000 7f8150c8f700 1 -- 172.21.15.32:0/3669314515 learned_addr learned my addr 172.21.15.32:0/3669314515 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:20.688 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.051+0000 7f8150c8f700 1 -- 172.21.15.32:0/3669314515 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f814c19c0e0 con 0x7f814c106ae0 2024-04-25T00:23:20.688 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.051+0000 7f8150c8f700 1 --2- 172.21.15.32:0/3669314515 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f814c106ae0 0x7f814c19bba0 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f8140000c00 tx=0x7f8140011c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:20.688 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.051+0000 7f8149ffb700 1 -- 172.21.15.32:0/3669314515 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f8140003b20 con 0x7f814c106ae0 2024-04-25T00:23:20.688 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.051+0000 7f8149ffb700 1 -- 172.21.15.32:0/3669314515 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f8140003cc0 con 0x7f814c106ae0 2024-04-25T00:23:20.688 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.051+0000 7f8152ef3700 1 -- 172.21.15.32:0/3669314515 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f814c19c370 con 0x7f814c106ae0 2024-04-25T00:23:20.688 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.051+0000 7f8149ffb700 1 -- 172.21.15.32:0/3669314515 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f8140024910 con 0x7f814c106ae0 2024-04-25T00:23:20.688 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.051+0000 7f8152ef3700 1 -- 172.21.15.32:0/3669314515 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f814c19f060 con 0x7f814c106ae0 2024-04-25T00:23:20.688 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.052+0000 7f8149ffb700 1 -- 172.21.15.32:0/3669314515 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 8) ==== 46287+0+0 (secure 0 0 0) 0x7f8140022020 con 0x7f814c106ae0 2024-04-25T00:23:20.689 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.052+0000 7f8152ef3700 1 -- 172.21.15.32:0/3669314515 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f814c063890 con 0x7f814c106ae0 2024-04-25T00:23:20.689 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.052+0000 7f8149ffb700 1 --2- 172.21.15.32:0/3669314515 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f813c0390a0 0x7f813c03b560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:20.689 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.052+0000 7f8149ffb700 1 -- 172.21.15.32:0/3669314515 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7f8140011e00 con 0x7f814c106ae0 2024-04-25T00:23:20.689 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.052+0000 7f814bfff700 1 --2- 172.21.15.32:0/3669314515 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f813c0390a0 0x7f813c03b560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:20.689 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.053+0000 7f814bfff700 1 --2- 172.21.15.32:0/3669314515 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f813c0390a0 0x7f813c03b560 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f8138006fd0 tx=0x7f8138006e40 comp rx=0 tx=0).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:20.689 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.057+0000 7f8149ffb700 1 -- 172.21.15.32:0/3669314515 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f8140017ca0 con 0x7f814c106ae0 2024-04-25T00:23:20.689 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:19.263+0000 7f8152ef3700 1 -- 172.21.15.32:0/3669314515 --> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] -- mgr_command(tid 0: {"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}) -- 0x7f814c0008d0 con 0x7f813c0390a0 2024-04-25T00:23:20.689 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:20.099+0000 7f8149ffb700 1 -- 172.21.15.32:0/3669314515 <== mgr.14116 v2:172.21.15.32:6800/4160192081 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+0 (secure 0 0 0) 0x7f814c0008d0 con 0x7f813c0390a0 2024-04-25T00:23:20.689 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:20.101+0000 7f8152ef3700 1 -- 172.21.15.32:0/3669314515 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f813c0390a0 msgr2=0x7f813c03b560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:20.689 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:20.101+0000 7f8152ef3700 1 --2- 172.21.15.32:0/3669314515 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f813c0390a0 0x7f813c03b560 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f8138006fd0 tx=0x7f8138006e40 comp rx=0 tx=0).stop 2024-04-25T00:23:20.689 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:20.101+0000 7f8152ef3700 1 -- 172.21.15.32:0/3669314515 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f814c106ae0 msgr2=0x7f814c19bba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:20.690 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:20.101+0000 7f8152ef3700 1 --2- 172.21.15.32:0/3669314515 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f814c106ae0 0x7f814c19bba0 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f8140000c00 tx=0x7f8140011c10 comp rx=0 tx=0).stop 2024-04-25T00:23:20.690 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:20.101+0000 7f8152ef3700 1 -- 172.21.15.32:0/3669314515 shutdown_connections 2024-04-25T00:23:20.690 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:20.101+0000 7f8152ef3700 1 --2- 172.21.15.32:0/3669314515 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f813c0390a0 0x7f813c03b560 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:20.690 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:20.101+0000 7f8152ef3700 1 --2- 172.21.15.32:0/3669314515 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f814c106ae0 0x7f814c19bba0 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:20.690 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:20.101+0000 7f8152ef3700 1 -- 172.21.15.32:0/3669314515 >> 172.21.15.32:0/3669314515 conn(0x7f814c1020a0 msgr2=0x7f814c102f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:20.690 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:20.101+0000 7f8152ef3700 1 -- 172.21.15.32:0/3669314515 shutdown_connections 2024-04-25T00:23:20.690 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:20.101+0000 7f8152ef3700 1 -- 172.21.15.32:0/3669314515 wait complete. 2024-04-25T00:23:21.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:21 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' 2024-04-25T00:23:21.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:21 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' 2024-04-25T00:23:21.885 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDbpksuQENO7qVQyOf4/NPXGtN/S59yLoR/VLuJhl6gtPW2+mwrirhW3lOvbVCzMhpcBOeyLdEBmEri3c8xP4rb1vp4KXDAOjBcMdHSIxvf677Q1276d/ouFeaCwDvMx1GQ4KjCs6XPjMMWf1FyKp5But6ES6iZ3dmNYpEQCdX7UPhTPFuLZ7HihF88x5vTGM9Dz7W1js3lU+dlHb6gY4LrQPiElUF885/JYfG8MmG/8WULjzmPTsvi5o249ihqbGgjOZyhl5K4uUdA3U+M/LxrnkpCsT2pEF7Iq+3gpfSzkxrU6ZT6yK/gPXY2yMfLcEVintsCrxhT5W3ueFBXUCcaLgAoORDod/GleFgV+K1VZ89z0vCEKw8zucIqhTtWxgsgKaNdu7uesusu5rigVVAE8y9RlLj0WQquE4G/x/Wg+MfRZptvlTbq7xlsqhOZVTpW0ZWewipALhTaq9vT17jdWnhIOzGSDscg0Enz4MiLRcWbr7ICEeUxBzQapchBj0c= ceph-cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:23:21.885 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.099+0000 7f23256f2700 1 Processor -- start 2024-04-25T00:23:21.885 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.099+0000 7f23256f2700 1 -- start start 2024-04-25T00:23:21.885 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.099+0000 7f23256f2700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2320108690 0x7f232010aa80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:21.885 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.099+0000 7f23256f2700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f232007be90 con 0x7f2320108690 2024-04-25T00:23:21.885 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.099+0000 7f231effd700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2320108690 0x7f232010aa80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:21.885 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.100+0000 7f231effd700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2320108690 0x7f232010aa80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40124/0 (socket says 172.21.15.32:40124) 2024-04-25T00:23:21.885 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.100+0000 7f231effd700 1 -- 172.21.15.32:0/909835386 learned_addr learned my addr 172.21.15.32:0/909835386 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:21.885 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.100+0000 7f231effd700 1 -- 172.21.15.32:0/909835386 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f232010afc0 con 0x7f2320108690 2024-04-25T00:23:21.886 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.100+0000 7f231effd700 1 --2- 172.21.15.32:0/909835386 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2320108690 0x7f232010aa80 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f23100098a0 tx=0x7f2310009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7f26a6ae0879449d server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:21.886 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.100+0000 7f231dffb700 1 -- 172.21.15.32:0/909835386 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f2310004160 con 0x7f2320108690 2024-04-25T00:23:21.886 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.100+0000 7f231dffb700 1 -- 172.21.15.32:0/909835386 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f2310004300 con 0x7f2320108690 2024-04-25T00:23:21.886 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.100+0000 7f231dffb700 1 -- 172.21.15.32:0/909835386 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f2310011b50 con 0x7f2320108690 2024-04-25T00:23:21.886 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.101+0000 7f23256f2700 1 -- 172.21.15.32:0/909835386 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2320108690 msgr2=0x7f232010aa80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:21.886 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.101+0000 7f23256f2700 1 --2- 172.21.15.32:0/909835386 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2320108690 0x7f232010aa80 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f23100098a0 tx=0x7f2310009bb0 comp rx=0 tx=0).stop 2024-04-25T00:23:21.886 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.101+0000 7f23256f2700 1 -- 172.21.15.32:0/909835386 shutdown_connections 2024-04-25T00:23:21.886 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.101+0000 7f23256f2700 1 --2- 172.21.15.32:0/909835386 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2320108690 0x7f232010aa80 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:21.886 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.101+0000 7f23256f2700 1 -- 172.21.15.32:0/909835386 >> 172.21.15.32:0/909835386 conn(0x7f23201020a0 msgr2=0x7f23201044c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:21.886 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.101+0000 7f23256f2700 1 -- 172.21.15.32:0/909835386 shutdown_connections 2024-04-25T00:23:21.886 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.101+0000 7f23256f2700 1 -- 172.21.15.32:0/909835386 wait complete. 2024-04-25T00:23:21.886 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.101+0000 7f23256f2700 1 Processor -- start 2024-04-25T00:23:21.886 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.102+0000 7f23256f2700 1 -- start start 2024-04-25T00:23:21.887 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.102+0000 7f23256f2700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2320108690 0x7f232019bb70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:21.887 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.102+0000 7f23256f2700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f232010bfe0 con 0x7f2320108690 2024-04-25T00:23:21.887 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.102+0000 7f231effd700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2320108690 0x7f232019bb70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:21.887 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.102+0000 7f231effd700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2320108690 0x7f232019bb70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40128/0 (socket says 172.21.15.32:40128) 2024-04-25T00:23:21.887 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.102+0000 7f231effd700 1 -- 172.21.15.32:0/2519171274 learned_addr learned my addr 172.21.15.32:0/2519171274 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:21.887 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.102+0000 7f231effd700 1 -- 172.21.15.32:0/2519171274 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f232019c0b0 con 0x7f2320108690 2024-04-25T00:23:21.887 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.102+0000 7f231effd700 1 --2- 172.21.15.32:0/2519171274 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2320108690 0x7f232019bb70 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f2310011670 tx=0x7f2310011b60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:21.887 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.102+0000 7f2307fff700 1 -- 172.21.15.32:0/2519171274 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f23100039c0 con 0x7f2320108690 2024-04-25T00:23:21.887 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.103+0000 7f2307fff700 1 -- 172.21.15.32:0/2519171274 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f2310011e50 con 0x7f2320108690 2024-04-25T00:23:21.887 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.103+0000 7f23256f2700 1 -- 172.21.15.32:0/2519171274 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f232019c340 con 0x7f2320108690 2024-04-25T00:23:21.887 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.103+0000 7f2307fff700 1 -- 172.21.15.32:0/2519171274 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f231001a5c0 con 0x7f2320108690 2024-04-25T00:23:21.887 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.103+0000 7f23256f2700 1 -- 172.21.15.32:0/2519171274 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f232019f030 con 0x7f2320108690 2024-04-25T00:23:21.887 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.103+0000 7f2307fff700 1 -- 172.21.15.32:0/2519171274 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 8) ==== 46287+0+0 (secure 0 0 0) 0x7f231001a870 con 0x7f2320108690 2024-04-25T00:23:21.887 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.104+0000 7f23256f2700 1 -- 172.21.15.32:0/2519171274 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2320063890 con 0x7f2320108690 2024-04-25T00:23:21.888 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.104+0000 7f2307fff700 1 --2- 172.21.15.32:0/2519171274 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f2308039050 0x7f230803b510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:21.888 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.104+0000 7f2307fff700 1 -- 172.21.15.32:0/2519171274 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7f231004d310 con 0x7f2320108690 2024-04-25T00:23:21.888 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.104+0000 7f231e7fc700 1 --2- 172.21.15.32:0/2519171274 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f2308039050 0x7f230803b510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:21.888 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.104+0000 7f231e7fc700 1 --2- 172.21.15.32:0/2519171274 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f2308039050 0x7f230803b510 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f2314006fd0 tx=0x7f2314006e40 comp rx=0 tx=0).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:21.888 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.109+0000 7f2307fff700 1 -- 172.21.15.32:0/2519171274 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f2310019c70 con 0x7f2320108690 2024-04-25T00:23:21.888 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.318+0000 7f23256f2700 1 -- 172.21.15.32:0/2519171274 --> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] -- mgr_command(tid 0: {"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}) -- 0x7f23201032c0 con 0x7f2308039050 2024-04-25T00:23:21.888 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.319+0000 7f2307fff700 1 -- 172.21.15.32:0/2519171274 <== mgr.14116 v2:172.21.15.32:6800/4160192081 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+595 (secure 0 0 0) 0x7f23201032c0 con 0x7f2308039050 2024-04-25T00:23:21.888 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.321+0000 7f23256f2700 1 -- 172.21.15.32:0/2519171274 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f2308039050 msgr2=0x7f230803b510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:21.888 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.321+0000 7f23256f2700 1 --2- 172.21.15.32:0/2519171274 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f2308039050 0x7f230803b510 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f2314006fd0 tx=0x7f2314006e40 comp rx=0 tx=0).stop 2024-04-25T00:23:21.888 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.321+0000 7f23256f2700 1 -- 172.21.15.32:0/2519171274 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2320108690 msgr2=0x7f232019bb70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:21.888 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.321+0000 7f23256f2700 1 --2- 172.21.15.32:0/2519171274 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2320108690 0x7f232019bb70 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f2310011670 tx=0x7f2310011b60 comp rx=0 tx=0).stop 2024-04-25T00:23:21.888 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.321+0000 7f23256f2700 1 -- 172.21.15.32:0/2519171274 shutdown_connections 2024-04-25T00:23:21.888 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.321+0000 7f23256f2700 1 --2- 172.21.15.32:0/2519171274 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f2308039050 0x7f230803b510 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:21.889 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.321+0000 7f23256f2700 1 --2- 172.21.15.32:0/2519171274 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2320108690 0x7f232019bb70 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:21.889 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.321+0000 7f23256f2700 1 -- 172.21.15.32:0/2519171274 >> 172.21.15.32:0/2519171274 conn(0x7f23201020a0 msgr2=0x7f2320102bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:21.889 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.321+0000 7f23256f2700 1 -- 172.21.15.32:0/2519171274 shutdown_connections 2024-04-25T00:23:21.889 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:21.321+0000 7f23256f2700 1 -- 172.21.15.32:0/2519171274 wait complete. 2024-04-25T00:23:21.889 INFO:teuthology.orchestra.run.smithi032.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-25T00:23:21.889 INFO:teuthology.orchestra.run.smithi032.stdout:Adding key to root@localhost authorized_keys... 2024-04-25T00:23:21.889 INFO:teuthology.orchestra.run.smithi032.stdout:Adding host smithi032... 2024-04-25T00:23:22.227 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:22 smithi032 ceph-mon[29357]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:23:23.174 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:23 smithi032 ceph-mon[29357]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi032", "addr": "172.21.15.32", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:23:24.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:24 smithi032 ceph-mon[29357]: Deploying cephadm binary to smithi032 2024-04-25T00:23:27.348 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout Added host 'smithi032' with addr '172.21.15.32' 2024-04-25T00:23:27.349 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.343+0000 7f97ac148700 1 Processor -- start 2024-04-25T00:23:27.349 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.343+0000 7f97ac148700 1 -- start start 2024-04-25T00:23:27.349 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.344+0000 7f97ac148700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f97a407de00 0x7f97a407c300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:27.349 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.344+0000 7f97ac148700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f97a407c840 con 0x7f97a407de00 2024-04-25T00:23:27.349 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.344+0000 7f97a9ee4700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f97a407de00 0x7f97a407c300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:27.349 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.344+0000 7f97a9ee4700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f97a407de00 0x7f97a407c300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:59630/0 (socket says 172.21.15.32:59630) 2024-04-25T00:23:27.349 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.344+0000 7f97a9ee4700 1 -- 172.21.15.32:0/3023748197 learned_addr learned my addr 172.21.15.32:0/3023748197 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:27.349 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.344+0000 7f97a9ee4700 1 -- 172.21.15.32:0/3023748197 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f97a407c9c0 con 0x7f97a407de00 2024-04-25T00:23:27.349 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.344+0000 7f97a9ee4700 1 --2- 172.21.15.32:0/3023748197 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f97a407de00 0x7f97a407c300 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f9794009e80 tx=0x7f979400b330 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f5ed68889321df5a server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:27.349 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.344+0000 7f97a8ee2700 1 -- 172.21.15.32:0/3023748197 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f9794017070 con 0x7f97a407de00 2024-04-25T00:23:27.349 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.344+0000 7f97a8ee2700 1 -- 172.21.15.32:0/3023748197 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f979400b8c0 con 0x7f97a407de00 2024-04-25T00:23:27.349 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.344+0000 7f97a8ee2700 1 -- 172.21.15.32:0/3023748197 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f97940045c0 con 0x7f97a407de00 2024-04-25T00:23:27.350 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.345+0000 7f97ac148700 1 -- 172.21.15.32:0/3023748197 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f97a407de00 msgr2=0x7f97a407c300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:27.350 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.345+0000 7f97ac148700 1 --2- 172.21.15.32:0/3023748197 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f97a407de00 0x7f97a407c300 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f9794009e80 tx=0x7f979400b330 comp rx=0 tx=0).stop 2024-04-25T00:23:27.350 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.345+0000 7f97ac148700 1 -- 172.21.15.32:0/3023748197 shutdown_connections 2024-04-25T00:23:27.350 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.345+0000 7f97ac148700 1 --2- 172.21.15.32:0/3023748197 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f97a407de00 0x7f97a407c300 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:27.350 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.345+0000 7f97ac148700 1 -- 172.21.15.32:0/3023748197 >> 172.21.15.32:0/3023748197 conn(0x7f97a4103230 msgr2=0x7f97a4105650 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:27.350 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.345+0000 7f97ac148700 1 -- 172.21.15.32:0/3023748197 shutdown_connections 2024-04-25T00:23:27.350 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.345+0000 7f97ac148700 1 -- 172.21.15.32:0/3023748197 wait complete. 2024-04-25T00:23:27.350 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.346+0000 7f97ac148700 1 Processor -- start 2024-04-25T00:23:27.350 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.346+0000 7f97ac148700 1 -- start start 2024-04-25T00:23:27.350 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.346+0000 7f97ac148700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f97a407de00 0x7f97a419dd50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:27.350 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.346+0000 7f97ac148700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f97a410e1c0 con 0x7f97a407de00 2024-04-25T00:23:27.350 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.346+0000 7f97a9ee4700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f97a407de00 0x7f97a419dd50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:27.350 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.346+0000 7f97a9ee4700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f97a407de00 0x7f97a419dd50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:59646/0 (socket says 172.21.15.32:59646) 2024-04-25T00:23:27.351 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.346+0000 7f97a9ee4700 1 -- 172.21.15.32:0/637278596 learned_addr learned my addr 172.21.15.32:0/637278596 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:27.351 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.346+0000 7f97a9ee4700 1 -- 172.21.15.32:0/637278596 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f97a419e290 con 0x7f97a407de00 2024-04-25T00:23:27.351 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.347+0000 7f97a9ee4700 1 --2- 172.21.15.32:0/637278596 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f97a407de00 0x7f97a419dd50 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f9794009e50 tx=0x7f979400bd30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:27.351 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.347+0000 7f97a2ffd700 1 -- 172.21.15.32:0/637278596 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f9794020020 con 0x7f97a407de00 2024-04-25T00:23:27.351 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.347+0000 7f97a2ffd700 1 -- 172.21.15.32:0/637278596 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f9794003b50 con 0x7f97a407de00 2024-04-25T00:23:27.351 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.347+0000 7f97ac148700 1 -- 172.21.15.32:0/637278596 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f97a419e520 con 0x7f97a407de00 2024-04-25T00:23:27.351 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.347+0000 7f97a2ffd700 1 -- 172.21.15.32:0/637278596 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f9794017040 con 0x7f97a407de00 2024-04-25T00:23:27.351 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.347+0000 7f97ac148700 1 -- 172.21.15.32:0/637278596 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f97a41a11b0 con 0x7f97a407de00 2024-04-25T00:23:27.351 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.348+0000 7f97a2ffd700 1 -- 172.21.15.32:0/637278596 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 8) ==== 46287+0+0 (secure 0 0 0) 0x7f9794023020 con 0x7f97a407de00 2024-04-25T00:23:27.351 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.348+0000 7f97a2ffd700 1 --2- 172.21.15.32:0/637278596 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f978c039050 0x7f978c03b510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:27.351 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.348+0000 7f97a2ffd700 1 -- 172.21.15.32:0/637278596 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7f979400f050 con 0x7f97a407de00 2024-04-25T00:23:27.351 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.348+0000 7f97a96e3700 1 --2- 172.21.15.32:0/637278596 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f978c039050 0x7f978c03b510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:27.351 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.348+0000 7f97ac148700 1 -- 172.21.15.32:0/637278596 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f97a4063890 con 0x7f97a407de00 2024-04-25T00:23:27.351 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.349+0000 7f97a96e3700 1 --2- 172.21.15.32:0/637278596 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f978c039050 0x7f978c03b510 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f9798006fd0 tx=0x7f9798006e40 comp rx=0 tx=0).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:27.352 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.353+0000 7f97a2ffd700 1 -- 172.21.15.32:0/637278596 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f979402a020 con 0x7f97a407de00 2024-04-25T00:23:27.352 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:22.566+0000 7f97ac148700 1 -- 172.21.15.32:0/637278596 --> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] -- mgr_command(tid 0: {"prefix": "orch host add", "hostname": "smithi032", "addr": "172.21.15.32", "target": ["mon-mgr", ""]}) -- 0x7f97a4104440 con 0x7f978c039050 2024-04-25T00:23:27.352 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:26.777+0000 7f97a2ffd700 1 -- 172.21.15.32:0/637278596 <== mgr.14116 v2:172.21.15.32:6800/4160192081 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+48 (secure 0 0 0) 0x7f97a4104440 con 0x7f978c039050 2024-04-25T00:23:27.352 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:26.780+0000 7f97ac148700 1 -- 172.21.15.32:0/637278596 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f978c039050 msgr2=0x7f978c03b510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:27.352 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:26.780+0000 7f97ac148700 1 --2- 172.21.15.32:0/637278596 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f978c039050 0x7f978c03b510 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f9798006fd0 tx=0x7f9798006e40 comp rx=0 tx=0).stop 2024-04-25T00:23:27.352 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:26.780+0000 7f97ac148700 1 -- 172.21.15.32:0/637278596 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f97a407de00 msgr2=0x7f97a419dd50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:27.352 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:26.780+0000 7f97ac148700 1 --2- 172.21.15.32:0/637278596 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f97a407de00 0x7f97a419dd50 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f9794009e50 tx=0x7f979400bd30 comp rx=0 tx=0).stop 2024-04-25T00:23:27.352 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:26.780+0000 7f97ac148700 1 -- 172.21.15.32:0/637278596 shutdown_connections 2024-04-25T00:23:27.352 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:26.780+0000 7f97ac148700 1 --2- 172.21.15.32:0/637278596 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f978c039050 0x7f978c03b510 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:27.352 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:26.780+0000 7f97ac148700 1 --2- 172.21.15.32:0/637278596 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f97a407de00 0x7f97a419dd50 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:27.352 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:26.780+0000 7f97ac148700 1 -- 172.21.15.32:0/637278596 >> 172.21.15.32:0/637278596 conn(0x7f97a4103230 msgr2=0x7f97a4103d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:27.352 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:26.780+0000 7f97ac148700 1 -- 172.21.15.32:0/637278596 shutdown_connections 2024-04-25T00:23:27.352 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:26.780+0000 7f97ac148700 1 -- 172.21.15.32:0/637278596 wait complete. 2024-04-25T00:23:27.352 INFO:teuthology.orchestra.run.smithi032.stdout:Deploying unmanaged mon service... 2024-04-25T00:23:27.980 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:27 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' 2024-04-25T00:23:27.980 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:27 smithi032 ceph-mon[29357]: Added host smithi032 2024-04-25T00:23:27.980 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:27 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:23:29.329 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:29 smithi032 ceph-mon[29357]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:23:29.329 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:29 smithi032 ceph-mon[29357]: Saving service mon spec with placement count:5 2024-04-25T00:23:29.329 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:29 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' 2024-04-25T00:23:29.597 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-04-25T00:23:29.597 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.815+0000 7fc9abe77700 1 Processor -- start 2024-04-25T00:23:29.597 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.815+0000 7fc9abe77700 1 -- start start 2024-04-25T00:23:29.597 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.815+0000 7fc9abe77700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc9a4107bf0 0x7fc9a4108010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:29.597 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.815+0000 7fc9abe77700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fc9a41085e0 con 0x7fc9a4107bf0 2024-04-25T00:23:29.597 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.816+0000 7fc9a9c13700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc9a4107bf0 0x7fc9a4108010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:29.597 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.816+0000 7fc9a9c13700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc9a4107bf0 0x7fc9a4108010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:59658/0 (socket says 172.21.15.32:59658) 2024-04-25T00:23:29.597 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.816+0000 7fc9a9c13700 1 -- 172.21.15.32:0/1942760309 learned_addr learned my addr 172.21.15.32:0/1942760309 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:29.598 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.816+0000 7fc9a9c13700 1 -- 172.21.15.32:0/1942760309 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc9a4108da0 con 0x7fc9a4107bf0 2024-04-25T00:23:29.598 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.816+0000 7fc9a9c13700 1 --2- 172.21.15.32:0/1942760309 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc9a4107bf0 0x7fc9a4108010 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7fc9940098a0 tx=0x7fc994009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=32a47ab54e571036 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:29.598 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.816+0000 7fc9a8c11700 1 -- 172.21.15.32:0/1942760309 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fc994004160 con 0x7fc9a4107bf0 2024-04-25T00:23:29.598 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.816+0000 7fc9a8c11700 1 -- 172.21.15.32:0/1942760309 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7fc994004300 con 0x7fc9a4107bf0 2024-04-25T00:23:29.598 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.816+0000 7fc9a8c11700 1 -- 172.21.15.32:0/1942760309 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fc994011b50 con 0x7fc9a4107bf0 2024-04-25T00:23:29.598 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.817+0000 7fc9abe77700 1 -- 172.21.15.32:0/1942760309 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc9a4107bf0 msgr2=0x7fc9a4108010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:29.598 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.817+0000 7fc9abe77700 1 --2- 172.21.15.32:0/1942760309 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc9a4107bf0 0x7fc9a4108010 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7fc9940098a0 tx=0x7fc994009bb0 comp rx=0 tx=0).stop 2024-04-25T00:23:29.598 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.817+0000 7fc9abe77700 1 -- 172.21.15.32:0/1942760309 shutdown_connections 2024-04-25T00:23:29.598 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.817+0000 7fc9abe77700 1 --2- 172.21.15.32:0/1942760309 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc9a4107bf0 0x7fc9a4108010 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:29.598 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.817+0000 7fc9abe77700 1 -- 172.21.15.32:0/1942760309 >> 172.21.15.32:0/1942760309 conn(0x7fc9a4103170 msgr2=0x7fc9a41055d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:29.598 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.817+0000 7fc9abe77700 1 -- 172.21.15.32:0/1942760309 shutdown_connections 2024-04-25T00:23:29.598 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.817+0000 7fc9abe77700 1 -- 172.21.15.32:0/1942760309 wait complete. 2024-04-25T00:23:29.598 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.817+0000 7fc9abe77700 1 Processor -- start 2024-04-25T00:23:29.599 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.817+0000 7fc9abe77700 1 -- start start 2024-04-25T00:23:29.599 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.818+0000 7fc9abe77700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc9a4107bf0 0x7fc9a419bb70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:29.599 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.818+0000 7fc9abe77700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fc9a410c150 con 0x7fc9a4107bf0 2024-04-25T00:23:29.599 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.818+0000 7fc9a9c13700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc9a4107bf0 0x7fc9a419bb70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:29.599 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.818+0000 7fc9a9c13700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc9a4107bf0 0x7fc9a419bb70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:59662/0 (socket says 172.21.15.32:59662) 2024-04-25T00:23:29.599 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.818+0000 7fc9a9c13700 1 -- 172.21.15.32:0/4215191175 learned_addr learned my addr 172.21.15.32:0/4215191175 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:29.599 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.818+0000 7fc9a9c13700 1 -- 172.21.15.32:0/4215191175 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc9a419c0b0 con 0x7fc9a4107bf0 2024-04-25T00:23:29.599 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.818+0000 7fc9a9c13700 1 --2- 172.21.15.32:0/4215191175 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc9a4107bf0 0x7fc9a419bb70 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7fc994011670 tx=0x7fc994011b60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:29.599 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.818+0000 7fc9a2ffd700 1 -- 172.21.15.32:0/4215191175 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fc9940039c0 con 0x7fc9a4107bf0 2024-04-25T00:23:29.599 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.818+0000 7fc9a2ffd700 1 -- 172.21.15.32:0/4215191175 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7fc994011e50 con 0x7fc9a4107bf0 2024-04-25T00:23:29.599 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.818+0000 7fc9abe77700 1 -- 172.21.15.32:0/4215191175 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc9a419c340 con 0x7fc9a4107bf0 2024-04-25T00:23:29.599 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.818+0000 7fc9a2ffd700 1 -- 172.21.15.32:0/4215191175 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fc99401a5c0 con 0x7fc9a4107bf0 2024-04-25T00:23:29.599 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.818+0000 7fc9abe77700 1 -- 172.21.15.32:0/4215191175 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc9a419f030 con 0x7fc9a4107bf0 2024-04-25T00:23:29.599 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.819+0000 7fc9a2ffd700 1 -- 172.21.15.32:0/4215191175 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 8) ==== 46287+0+0 (secure 0 0 0) 0x7fc994024430 con 0x7fc9a4107bf0 2024-04-25T00:23:29.600 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.819+0000 7fc9a2ffd700 1 --2- 172.21.15.32:0/4215191175 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7fc98c039050 0x7fc98c03b510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:29.600 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.819+0000 7fc9a2ffd700 1 -- 172.21.15.32:0/4215191175 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7fc99404d4f0 con 0x7fc9a4107bf0 2024-04-25T00:23:29.600 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.820+0000 7fc9a9412700 1 --2- 172.21.15.32:0/4215191175 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7fc98c039050 0x7fc98c03b510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:29.600 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.820+0000 7fc9abe77700 1 -- 172.21.15.32:0/4215191175 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc9a4063890 con 0x7fc9a4107bf0 2024-04-25T00:23:29.600 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.820+0000 7fc9a9412700 1 --2- 172.21.15.32:0/4215191175 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7fc98c039050 0x7fc98c03b510 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7fc998006fd0 tx=0x7fc998006e40 comp rx=0 tx=0).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:29.600 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:27.825+0000 7fc9a2ffd700 1 -- 172.21.15.32:0/4215191175 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fc99402d910 con 0x7fc9a4107bf0 2024-04-25T00:23:29.600 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:28.052+0000 7fc9abe77700 1 -- 172.21.15.32:0/4215191175 --> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}) -- 0x7fc9a4104400 con 0x7fc98c039050 2024-04-25T00:23:29.600 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:28.056+0000 7fc9a2ffd700 1 -- 172.21.15.32:0/4215191175 <== mgr.14116 v2:172.21.15.32:6800/4160192081 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+24 (secure 0 0 0) 0x7fc9a4104400 con 0x7fc98c039050 2024-04-25T00:23:29.600 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:28.058+0000 7fc9abe77700 1 -- 172.21.15.32:0/4215191175 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7fc98c039050 msgr2=0x7fc98c03b510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:29.600 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:28.058+0000 7fc9abe77700 1 --2- 172.21.15.32:0/4215191175 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7fc98c039050 0x7fc98c03b510 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7fc998006fd0 tx=0x7fc998006e40 comp rx=0 tx=0).stop 2024-04-25T00:23:29.600 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:28.058+0000 7fc9abe77700 1 -- 172.21.15.32:0/4215191175 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc9a4107bf0 msgr2=0x7fc9a419bb70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:29.600 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:28.058+0000 7fc9abe77700 1 --2- 172.21.15.32:0/4215191175 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc9a4107bf0 0x7fc9a419bb70 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7fc994011670 tx=0x7fc994011b60 comp rx=0 tx=0).stop 2024-04-25T00:23:29.600 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:28.058+0000 7fc9abe77700 1 -- 172.21.15.32:0/4215191175 shutdown_connections 2024-04-25T00:23:29.600 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:28.058+0000 7fc9abe77700 1 --2- 172.21.15.32:0/4215191175 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7fc98c039050 0x7fc98c03b510 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:29.600 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:28.058+0000 7fc9abe77700 1 --2- 172.21.15.32:0/4215191175 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc9a4107bf0 0x7fc9a419bb70 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:29.601 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:28.058+0000 7fc9abe77700 1 -- 172.21.15.32:0/4215191175 >> 172.21.15.32:0/4215191175 conn(0x7fc9a4103170 msgr2=0x7fc9a4103cf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:29.601 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:28.059+0000 7fc9abe77700 1 -- 172.21.15.32:0/4215191175 shutdown_connections 2024-04-25T00:23:29.601 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:28.059+0000 7fc9abe77700 1 -- 172.21.15.32:0/4215191175 wait complete. 2024-04-25T00:23:29.601 INFO:teuthology.orchestra.run.smithi032.stdout:Deploying unmanaged mgr service... 2024-04-25T00:23:30.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:30 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' 2024-04-25T00:23:30.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:30 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' 2024-04-25T00:23:30.873 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-04-25T00:23:30.874 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.062+0000 7f2a087dd700 1 Processor -- start 2024-04-25T00:23:30.874 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.062+0000 7f2a087dd700 1 -- start start 2024-04-25T00:23:30.874 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.063+0000 7f2a087dd700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2a00107c50 0x7f2a00108070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:30.874 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.063+0000 7f2a087dd700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f2a00108640 con 0x7f2a00107c50 2024-04-25T00:23:30.874 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.063+0000 7f2a06579700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2a00107c50 0x7f2a00108070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:30.874 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.063+0000 7f2a06579700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2a00107c50 0x7f2a00108070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:59670/0 (socket says 172.21.15.32:59670) 2024-04-25T00:23:30.874 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.063+0000 7f2a06579700 1 -- 172.21.15.32:0/329959735 learned_addr learned my addr 172.21.15.32:0/329959735 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:30.874 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.063+0000 7f2a06579700 1 -- 172.21.15.32:0/329959735 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2a00108db0 con 0x7f2a00107c50 2024-04-25T00:23:30.874 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.063+0000 7f2a06579700 1 --2- 172.21.15.32:0/329959735 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2a00107c50 0x7f2a00108070 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f29f40098a0 tx=0x7f29f4009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e2ff07011e514444 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:30.874 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.063+0000 7f2a05577700 1 -- 172.21.15.32:0/329959735 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f29f4004160 con 0x7f2a00107c50 2024-04-25T00:23:30.874 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.063+0000 7f2a05577700 1 -- 172.21.15.32:0/329959735 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f29f4004300 con 0x7f2a00107c50 2024-04-25T00:23:30.874 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.063+0000 7f2a05577700 1 -- 172.21.15.32:0/329959735 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f29f4011b20 con 0x7f2a00107c50 2024-04-25T00:23:30.874 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.064+0000 7f2a087dd700 1 -- 172.21.15.32:0/329959735 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2a00107c50 msgr2=0x7f2a00108070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:30.875 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.064+0000 7f2a087dd700 1 --2- 172.21.15.32:0/329959735 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2a00107c50 0x7f2a00108070 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f29f40098a0 tx=0x7f29f4009bb0 comp rx=0 tx=0).stop 2024-04-25T00:23:30.875 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.064+0000 7f2a087dd700 1 -- 172.21.15.32:0/329959735 shutdown_connections 2024-04-25T00:23:30.875 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.064+0000 7f2a087dd700 1 --2- 172.21.15.32:0/329959735 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2a00107c50 0x7f2a00108070 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:30.875 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.064+0000 7f2a087dd700 1 -- 172.21.15.32:0/329959735 >> 172.21.15.32:0/329959735 conn(0x7f2a00103230 msgr2=0x7f2a00105650 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:30.875 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.064+0000 7f2a087dd700 1 -- 172.21.15.32:0/329959735 shutdown_connections 2024-04-25T00:23:30.875 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.064+0000 7f2a087dd700 1 -- 172.21.15.32:0/329959735 wait complete. 2024-04-25T00:23:30.875 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.065+0000 7f2a087dd700 1 Processor -- start 2024-04-25T00:23:30.875 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.065+0000 7f2a087dd700 1 -- start start 2024-04-25T00:23:30.875 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.065+0000 7f2a087dd700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2a00107c50 0x7f2a0007db90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:30.875 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.065+0000 7f2a087dd700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f2a0010c160 con 0x7f2a00107c50 2024-04-25T00:23:30.875 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.065+0000 7f2a06579700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2a00107c50 0x7f2a0007db90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:30.875 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.065+0000 7f2a06579700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2a00107c50 0x7f2a0007db90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:59672/0 (socket says 172.21.15.32:59672) 2024-04-25T00:23:30.875 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.065+0000 7f2a06579700 1 -- 172.21.15.32:0/2261716021 learned_addr learned my addr 172.21.15.32:0/2261716021 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:30.875 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.065+0000 7f2a06579700 1 -- 172.21.15.32:0/2261716021 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2a0007e0d0 con 0x7f2a00107c50 2024-04-25T00:23:30.875 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.066+0000 7f2a06579700 1 --2- 172.21.15.32:0/2261716021 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2a00107c50 0x7f2a0007db90 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f29f40116a0 tx=0x7f29f4011ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:30.876 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.066+0000 7f29f37fe700 1 -- 172.21.15.32:0/2261716021 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f29f4003a40 con 0x7f2a00107c50 2024-04-25T00:23:30.876 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.066+0000 7f29f37fe700 1 -- 172.21.15.32:0/2261716021 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f29f4003be0 con 0x7f2a00107c50 2024-04-25T00:23:30.876 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.066+0000 7f2a087dd700 1 -- 172.21.15.32:0/2261716021 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2a0007c2c0 con 0x7f2a00107c50 2024-04-25T00:23:30.876 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.066+0000 7f29f37fe700 1 -- 172.21.15.32:0/2261716021 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f29f401a720 con 0x7f2a00107c50 2024-04-25T00:23:30.876 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.066+0000 7f2a087dd700 1 -- 172.21.15.32:0/2261716021 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2a0007c780 con 0x7f2a00107c50 2024-04-25T00:23:30.876 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.067+0000 7f29f37fe700 1 -- 172.21.15.32:0/2261716021 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 8) ==== 46287+0+0 (secure 0 0 0) 0x7f29f4024430 con 0x7f2a00107c50 2024-04-25T00:23:30.876 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.067+0000 7f2a087dd700 1 -- 172.21.15.32:0/2261716021 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2a00063890 con 0x7f2a00107c50 2024-04-25T00:23:30.876 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.067+0000 7f29f37fe700 1 --2- 172.21.15.32:0/2261716021 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f29ec039000 0x7f29ec03b4c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:30.876 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.067+0000 7f29f37fe700 1 -- 172.21.15.32:0/2261716021 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7f29f404d720 con 0x7f2a00107c50 2024-04-25T00:23:30.876 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.067+0000 7f2a05d78700 1 --2- 172.21.15.32:0/2261716021 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f29ec039000 0x7f29ec03b4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:30.876 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.068+0000 7f2a05d78700 1 --2- 172.21.15.32:0/2261716021 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f29ec039000 0x7f29ec03b4c0 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f29fc006fd0 tx=0x7f29fc006e40 comp rx=0 tx=0).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:30.876 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.072+0000 7f29f37fe700 1 -- 172.21.15.32:0/2261716021 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f29f4011e20 con 0x7f2a00107c50 2024-04-25T00:23:30.876 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.310+0000 7f2a087dd700 1 -- 172.21.15.32:0/2261716021 --> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}) -- 0x7f2a0007cf60 con 0x7f29ec039000 2024-04-25T00:23:30.876 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.315+0000 7f29f37fe700 1 -- 172.21.15.32:0/2261716021 <== mgr.14116 v2:172.21.15.32:6800/4160192081 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+24 (secure 0 0 0) 0x7f2a0007cf60 con 0x7f29ec039000 2024-04-25T00:23:30.877 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.317+0000 7f2a087dd700 1 -- 172.21.15.32:0/2261716021 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f29ec039000 msgr2=0x7f29ec03b4c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:30.877 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.317+0000 7f2a087dd700 1 --2- 172.21.15.32:0/2261716021 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f29ec039000 0x7f29ec03b4c0 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f29fc006fd0 tx=0x7f29fc006e40 comp rx=0 tx=0).stop 2024-04-25T00:23:30.877 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.317+0000 7f2a087dd700 1 -- 172.21.15.32:0/2261716021 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2a00107c50 msgr2=0x7f2a0007db90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:30.877 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.317+0000 7f2a087dd700 1 --2- 172.21.15.32:0/2261716021 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2a00107c50 0x7f2a0007db90 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f29f40116a0 tx=0x7f29f4011ba0 comp rx=0 tx=0).stop 2024-04-25T00:23:30.877 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.317+0000 7f2a087dd700 1 -- 172.21.15.32:0/2261716021 shutdown_connections 2024-04-25T00:23:30.877 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.317+0000 7f2a087dd700 1 --2- 172.21.15.32:0/2261716021 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f29ec039000 0x7f29ec03b4c0 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:30.877 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.317+0000 7f2a087dd700 1 --2- 172.21.15.32:0/2261716021 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2a00107c50 0x7f2a0007db90 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:30.877 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.317+0000 7f2a087dd700 1 -- 172.21.15.32:0/2261716021 >> 172.21.15.32:0/2261716021 conn(0x7f2a00103230 msgr2=0x7f2a00103cf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:30.877 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.317+0000 7f2a087dd700 1 -- 172.21.15.32:0/2261716021 shutdown_connections 2024-04-25T00:23:30.877 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:30.317+0000 7f2a087dd700 1 -- 172.21.15.32:0/2261716021 wait complete. 2024-04-25T00:23:31.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:31 smithi032 ceph-mon[29357]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:23:31.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:31 smithi032 ceph-mon[29357]: Saving service mgr spec with placement count:2 2024-04-25T00:23:31.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:31 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' 2024-04-25T00:23:32.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:32 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3269189145' entity='client.admin' 2024-04-25T00:23:32.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:32 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' 2024-04-25T00:23:32.762 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.425+0000 7f0b7be1f700 1 Processor -- start 2024-04-25T00:23:32.762 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.425+0000 7f0b7be1f700 1 -- start start 2024-04-25T00:23:32.762 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.425+0000 7f0b7be1f700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0b741064c0 0x7f0b741068e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:32.762 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.425+0000 7f0b7be1f700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f0b7407be50 con 0x7f0b741064c0 2024-04-25T00:23:32.762 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.425+0000 7f0b79bbb700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0b741064c0 0x7f0b741068e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:32.762 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.425+0000 7f0b79bbb700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0b741064c0 0x7f0b741068e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:59688/0 (socket says 172.21.15.32:59688) 2024-04-25T00:23:32.762 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.425+0000 7f0b79bbb700 1 -- 172.21.15.32:0/1516482868 learned_addr learned my addr 172.21.15.32:0/1516482868 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:32.762 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.426+0000 7f0b79bbb700 1 -- 172.21.15.32:0/1516482868 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0b74106e20 con 0x7f0b741064c0 2024-04-25T00:23:32.762 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.426+0000 7f0b79bbb700 1 --2- 172.21.15.32:0/1516482868 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0b741064c0 0x7f0b741068e0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f0b700098a0 tx=0x7f0b70009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e97cf827711fb38e server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:32.762 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.426+0000 7f0b78bb9700 1 -- 172.21.15.32:0/1516482868 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0b70004160 con 0x7f0b741064c0 2024-04-25T00:23:32.763 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.426+0000 7f0b78bb9700 1 -- 172.21.15.32:0/1516482868 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0b70004300 con 0x7f0b741064c0 2024-04-25T00:23:32.763 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.426+0000 7f0b78bb9700 1 -- 172.21.15.32:0/1516482868 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0b70011b20 con 0x7f0b741064c0 2024-04-25T00:23:32.763 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.427+0000 7f0b7be1f700 1 -- 172.21.15.32:0/1516482868 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0b741064c0 msgr2=0x7f0b741068e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:32.763 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.427+0000 7f0b7be1f700 1 --2- 172.21.15.32:0/1516482868 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0b741064c0 0x7f0b741068e0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f0b700098a0 tx=0x7f0b70009bb0 comp rx=0 tx=0).stop 2024-04-25T00:23:32.763 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.427+0000 7f0b7be1f700 1 -- 172.21.15.32:0/1516482868 shutdown_connections 2024-04-25T00:23:32.763 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.427+0000 7f0b7be1f700 1 --2- 172.21.15.32:0/1516482868 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0b741064c0 0x7f0b741068e0 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:32.763 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.427+0000 7f0b7be1f700 1 -- 172.21.15.32:0/1516482868 >> 172.21.15.32:0/1516482868 conn(0x7f0b741020a0 msgr2=0x7f0b741044c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:32.763 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.427+0000 7f0b7be1f700 1 -- 172.21.15.32:0/1516482868 shutdown_connections 2024-04-25T00:23:32.763 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.427+0000 7f0b7be1f700 1 -- 172.21.15.32:0/1516482868 wait complete. 2024-04-25T00:23:32.763 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.428+0000 7f0b7be1f700 1 Processor -- start 2024-04-25T00:23:32.763 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.428+0000 7f0b7be1f700 1 -- start start 2024-04-25T00:23:32.763 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.428+0000 7f0b7be1f700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0b741064c0 0x7f0b7419dcd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:32.764 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.428+0000 7f0b7be1f700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f0b74107ef0 con 0x7f0b741064c0 2024-04-25T00:23:32.764 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.428+0000 7f0b79bbb700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0b741064c0 0x7f0b7419dcd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:32.764 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.428+0000 7f0b79bbb700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0b741064c0 0x7f0b7419dcd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:59704/0 (socket says 172.21.15.32:59704) 2024-04-25T00:23:32.764 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.428+0000 7f0b79bbb700 1 -- 172.21.15.32:0/3269189145 learned_addr learned my addr 172.21.15.32:0/3269189145 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:32.764 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.428+0000 7f0b79bbb700 1 -- 172.21.15.32:0/3269189145 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0b7419e210 con 0x7f0b741064c0 2024-04-25T00:23:32.764 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.428+0000 7f0b79bbb700 1 --2- 172.21.15.32:0/3269189145 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0b741064c0 0x7f0b7419dcd0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f0b700117e0 tx=0x7f0b70011c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:32.764 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.429+0000 7f0b6affd700 1 -- 172.21.15.32:0/3269189145 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0b70003bf0 con 0x7f0b741064c0 2024-04-25T00:23:32.764 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.429+0000 7f0b6affd700 1 -- 172.21.15.32:0/3269189145 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0b7001a910 con 0x7f0b741064c0 2024-04-25T00:23:32.764 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.429+0000 7f0b7be1f700 1 -- 172.21.15.32:0/3269189145 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0b7419e4a0 con 0x7f0b741064c0 2024-04-25T00:23:32.764 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.429+0000 7f0b6affd700 1 -- 172.21.15.32:0/3269189145 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0b70024970 con 0x7f0b741064c0 2024-04-25T00:23:32.764 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.429+0000 7f0b7be1f700 1 -- 172.21.15.32:0/3269189145 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0b7419e900 con 0x7f0b741064c0 2024-04-25T00:23:32.764 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.430+0000 7f0b6affd700 1 -- 172.21.15.32:0/3269189145 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 8) ==== 46287+0+0 (secure 0 0 0) 0x7f0b70022020 con 0x7f0b741064c0 2024-04-25T00:23:32.764 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.430+0000 7f0b6affd700 1 --2- 172.21.15.32:0/3269189145 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f0b60039000 0x7f0b6003b4c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:32.764 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.430+0000 7f0b7be1f700 1 -- 172.21.15.32:0/3269189145 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0b74063890 con 0x7f0b741064c0 2024-04-25T00:23:32.764 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.430+0000 7f0b6affd700 1 -- 172.21.15.32:0/3269189145 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7f0b7004d510 con 0x7f0b741064c0 2024-04-25T00:23:32.765 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.430+0000 7f0b793ba700 1 --2- 172.21.15.32:0/3269189145 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f0b60039000 0x7f0b6003b4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:32.765 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.431+0000 7f0b793ba700 1 --2- 172.21.15.32:0/3269189145 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f0b60039000 0x7f0b6003b4c0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f0b64006fd0 tx=0x7f0b64006e40 comp rx=0 tx=0).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:32.765 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.437+0000 7f0b6affd700 1 -- 172.21.15.32:0/3269189145 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f0b70011e20 con 0x7f0b741064c0 2024-04-25T00:23:32.765 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.656+0000 7f0b7be1f700 1 -- 172.21.15.32:0/3269189145 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command([{prefix=config set, name=mgr/cephadm/container_init}] v 0) -- 0x7f0b74062e40 con 0x7f0b741064c0 2024-04-25T00:23:32.765 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.659+0000 7f0b6affd700 1 -- 172.21.15.32:0/3269189145 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/cephadm/container_init}]=0 v7)=0 v7) ==== 142+0+0 (secure 0 0 0) 0x7f0b7002cde0 con 0x7f0b741064c0 2024-04-25T00:23:32.765 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.661+0000 7f0b7be1f700 1 -- 172.21.15.32:0/3269189145 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f0b60039000 msgr2=0x7f0b6003b4c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:32.765 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.661+0000 7f0b7be1f700 1 --2- 172.21.15.32:0/3269189145 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f0b60039000 0x7f0b6003b4c0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f0b64006fd0 tx=0x7f0b64006e40 comp rx=0 tx=0).stop 2024-04-25T00:23:32.765 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.661+0000 7f0b7be1f700 1 -- 172.21.15.32:0/3269189145 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0b741064c0 msgr2=0x7f0b7419dcd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:32.765 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.661+0000 7f0b7be1f700 1 --2- 172.21.15.32:0/3269189145 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0b741064c0 0x7f0b7419dcd0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f0b700117e0 tx=0x7f0b70011c10 comp rx=0 tx=0).stop 2024-04-25T00:23:32.765 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.661+0000 7f0b7be1f700 1 -- 172.21.15.32:0/3269189145 shutdown_connections 2024-04-25T00:23:32.765 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.661+0000 7f0b7be1f700 1 --2- 172.21.15.32:0/3269189145 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f0b60039000 0x7f0b6003b4c0 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:32.765 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.661+0000 7f0b7be1f700 1 --2- 172.21.15.32:0/3269189145 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0b741064c0 0x7f0b7419dcd0 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:32.765 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.661+0000 7f0b7be1f700 1 -- 172.21.15.32:0/3269189145 >> 172.21.15.32:0/3269189145 conn(0x7f0b741020a0 msgr2=0x7f0b74194930 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:32.765 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.661+0000 7f0b7be1f700 1 -- 172.21.15.32:0/3269189145 shutdown_connections 2024-04-25T00:23:32.766 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:31.661+0000 7f0b7be1f700 1 -- 172.21.15.32:0/3269189145 wait complete. 2024-04-25T00:23:34.050 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.244+0000 7f0c874c9700 1 Processor -- start 2024-04-25T00:23:34.050 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.244+0000 7f0c874c9700 1 -- start start 2024-04-25T00:23:34.050 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.244+0000 7f0c874c9700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0c80106ab0 0x7f0c80106ed0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:34.050 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.244+0000 7f0c874c9700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f0c801074a0 con 0x7f0c80106ab0 2024-04-25T00:23:34.051 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.245+0000 7f0c85265700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0c80106ab0 0x7f0c80106ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:34.051 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.245+0000 7f0c85265700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0c80106ab0 0x7f0c80106ed0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40698/0 (socket says 172.21.15.32:40698) 2024-04-25T00:23:34.051 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.245+0000 7f0c85265700 1 -- 172.21.15.32:0/4275646688 learned_addr learned my addr 172.21.15.32:0/4275646688 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:34.051 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.245+0000 7f0c85265700 1 -- 172.21.15.32:0/4275646688 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0c80107cb0 con 0x7f0c80106ab0 2024-04-25T00:23:34.051 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.245+0000 7f0c85265700 1 --2- 172.21.15.32:0/4275646688 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0c80106ab0 0x7f0c80106ed0 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f0c78009e80 tx=0x7f0c7800b330 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f97248619616a2b0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:34.051 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.245+0000 7f0c7ffff700 1 -- 172.21.15.32:0/4275646688 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0c78017070 con 0x7f0c80106ab0 2024-04-25T00:23:34.051 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.245+0000 7f0c7ffff700 1 -- 172.21.15.32:0/4275646688 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0c7800b8c0 con 0x7f0c80106ab0 2024-04-25T00:23:34.051 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.245+0000 7f0c7ffff700 1 -- 172.21.15.32:0/4275646688 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0c780045c0 con 0x7f0c80106ab0 2024-04-25T00:23:34.051 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.246+0000 7f0c874c9700 1 -- 172.21.15.32:0/4275646688 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0c80106ab0 msgr2=0x7f0c80106ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:34.051 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.246+0000 7f0c874c9700 1 --2- 172.21.15.32:0/4275646688 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0c80106ab0 0x7f0c80106ed0 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f0c78009e80 tx=0x7f0c7800b330 comp rx=0 tx=0).stop 2024-04-25T00:23:34.051 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.246+0000 7f0c874c9700 1 -- 172.21.15.32:0/4275646688 shutdown_connections 2024-04-25T00:23:34.051 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.246+0000 7f0c874c9700 1 --2- 172.21.15.32:0/4275646688 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0c80106ab0 0x7f0c80106ed0 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:34.051 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.246+0000 7f0c874c9700 1 -- 172.21.15.32:0/4275646688 >> 172.21.15.32:0/4275646688 conn(0x7f0c80102070 msgr2=0x7f0c80104490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:34.051 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.246+0000 7f0c874c9700 1 -- 172.21.15.32:0/4275646688 shutdown_connections 2024-04-25T00:23:34.052 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.246+0000 7f0c874c9700 1 -- 172.21.15.32:0/4275646688 wait complete. 2024-04-25T00:23:34.052 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.247+0000 7f0c874c9700 1 Processor -- start 2024-04-25T00:23:34.052 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.247+0000 7f0c874c9700 1 -- start start 2024-04-25T00:23:34.052 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.247+0000 7f0c874c9700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0c80106ab0 0x7f0c8019bb80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:34.052 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.247+0000 7f0c874c9700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f0c8010b060 con 0x7f0c80106ab0 2024-04-25T00:23:34.052 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.247+0000 7f0c85265700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0c80106ab0 0x7f0c8019bb80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:34.052 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.247+0000 7f0c85265700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0c80106ab0 0x7f0c8019bb80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40708/0 (socket says 172.21.15.32:40708) 2024-04-25T00:23:34.052 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.247+0000 7f0c85265700 1 -- 172.21.15.32:0/453339905 learned_addr learned my addr 172.21.15.32:0/453339905 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:34.052 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.247+0000 7f0c85265700 1 -- 172.21.15.32:0/453339905 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0c8019c0c0 con 0x7f0c80106ab0 2024-04-25T00:23:34.052 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.247+0000 7f0c85265700 1 --2- 172.21.15.32:0/453339905 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0c80106ab0 0x7f0c8019bb80 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f0c7800bca0 tx=0x7f0c7800bd80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:34.052 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.247+0000 7f0c7e7fc700 1 -- 172.21.15.32:0/453339905 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0c78020020 con 0x7f0c80106ab0 2024-04-25T00:23:34.052 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.248+0000 7f0c7e7fc700 1 -- 172.21.15.32:0/453339905 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0c78003d90 con 0x7f0c80106ab0 2024-04-25T00:23:34.052 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.248+0000 7f0c874c9700 1 -- 172.21.15.32:0/453339905 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0c8019c350 con 0x7f0c80106ab0 2024-04-25T00:23:34.052 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.248+0000 7f0c7e7fc700 1 -- 172.21.15.32:0/453339905 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0c78017040 con 0x7f0c80106ab0 2024-04-25T00:23:34.053 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.248+0000 7f0c874c9700 1 -- 172.21.15.32:0/453339905 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0c8019f040 con 0x7f0c80106ab0 2024-04-25T00:23:34.053 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.248+0000 7f0c7e7fc700 1 -- 172.21.15.32:0/453339905 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 8) ==== 46287+0+0 (secure 0 0 0) 0x7f0c78023020 con 0x7f0c80106ab0 2024-04-25T00:23:34.053 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.249+0000 7f0c7e7fc700 1 --2- 172.21.15.32:0/453339905 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f0c68039000 0x7f0c6803b4c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:34.053 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.249+0000 7f0c874c9700 1 -- 172.21.15.32:0/453339905 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0c80063890 con 0x7f0c80106ab0 2024-04-25T00:23:34.053 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.249+0000 7f0c7e7fc700 1 -- 172.21.15.32:0/453339905 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7f0c7800f050 con 0x7f0c80106ab0 2024-04-25T00:23:34.053 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.249+0000 7f0c84a64700 1 --2- 172.21.15.32:0/453339905 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f0c68039000 0x7f0c6803b4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:34.053 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.249+0000 7f0c84a64700 1 --2- 172.21.15.32:0/453339905 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f0c68039000 0x7f0c6803b4c0 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f0c74006fd0 tx=0x7f0c74006e40 comp rx=0 tx=0).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:34.053 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.254+0000 7f0c7e7fc700 1 -- 172.21.15.32:0/453339905 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f0c7802a020 con 0x7f0c80106ab0 2024-04-25T00:23:34.053 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.465+0000 7f0c874c9700 1 -- 172.21.15.32:0/453339905 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command([{prefix=config set, name=mgr/dashboard/ssl_server_port}] v 0) -- 0x7f0c8002ddf0 con 0x7f0c80106ab0 2024-04-25T00:23:34.053 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.468+0000 7f0c7e7fc700 1 -- 172.21.15.32:0/453339905 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/dashboard/ssl_server_port}]=0 v8)=0 v8) ==== 130+0+0 (secure 0 0 0) 0x7f0c7802d350 con 0x7f0c80106ab0 2024-04-25T00:23:34.053 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.471+0000 7f0c874c9700 1 -- 172.21.15.32:0/453339905 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f0c68039000 msgr2=0x7f0c6803b4c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:34.053 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.471+0000 7f0c874c9700 1 --2- 172.21.15.32:0/453339905 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f0c68039000 0x7f0c6803b4c0 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f0c74006fd0 tx=0x7f0c74006e40 comp rx=0 tx=0).stop 2024-04-25T00:23:34.053 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.471+0000 7f0c874c9700 1 -- 172.21.15.32:0/453339905 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0c80106ab0 msgr2=0x7f0c8019bb80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:34.053 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.471+0000 7f0c874c9700 1 --2- 172.21.15.32:0/453339905 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0c80106ab0 0x7f0c8019bb80 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f0c7800bca0 tx=0x7f0c7800bd80 comp rx=0 tx=0).stop 2024-04-25T00:23:34.054 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.471+0000 7f0c874c9700 1 -- 172.21.15.32:0/453339905 shutdown_connections 2024-04-25T00:23:34.054 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.471+0000 7f0c874c9700 1 --2- 172.21.15.32:0/453339905 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f0c68039000 0x7f0c6803b4c0 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:34.054 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.471+0000 7f0c874c9700 1 --2- 172.21.15.32:0/453339905 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0c80106ab0 0x7f0c8019bb80 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:34.054 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.471+0000 7f0c874c9700 1 -- 172.21.15.32:0/453339905 >> 172.21.15.32:0/453339905 conn(0x7f0c80102070 msgr2=0x7f0c80102ef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:34.054 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.471+0000 7f0c874c9700 1 -- 172.21.15.32:0/453339905 shutdown_connections 2024-04-25T00:23:34.054 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:33.471+0000 7f0c874c9700 1 -- 172.21.15.32:0/453339905 wait complete. 2024-04-25T00:23:34.054 INFO:teuthology.orchestra.run.smithi032.stdout:Enabling the dashboard module... 2024-04-25T00:23:34.679 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:34 smithi032 ceph-mon[29357]: from='mgr.14116 172.21.15.32:0/3540459056' entity='mgr.x' 2024-04-25T00:23:34.679 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:34 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/453339905' entity='client.admin' 2024-04-25T00:23:35.963 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:35 smithi032 ceph-mon[29357]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:23:35.963 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:35 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/845393168' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-25T00:23:35.963 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:35 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: ignoring --setuser ceph since I am not root 2024-04-25T00:23:35.963 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:35 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: ignoring --setgroup ceph since I am not root 2024-04-25T00:23:35.963 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:35 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:35.808+0000 7f97a48ad200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-25T00:23:36.248 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:36 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:36.002+0000 7f97a48ad200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-25T00:23:36.248 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:36 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:36.109+0000 7f97a48ad200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-25T00:23:36.296 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.439+0000 7f11767bf700 1 Processor -- start 2024-04-25T00:23:36.296 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.439+0000 7f11767bf700 1 -- start start 2024-04-25T00:23:36.297 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.439+0000 7f11767bf700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1170105e90 0x7f11701062b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:36.297 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.439+0000 7f11767bf700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f1170106880 con 0x7f1170105e90 2024-04-25T00:23:36.297 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.440+0000 7f116ffff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1170105e90 0x7f11701062b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:36.297 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.440+0000 7f116ffff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1170105e90 0x7f11701062b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40712/0 (socket says 172.21.15.32:40712) 2024-04-25T00:23:36.297 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.440+0000 7f116ffff700 1 -- 172.21.15.32:0/2672118330 learned_addr learned my addr 172.21.15.32:0/2672118330 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:36.297 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.440+0000 7f116ffff700 1 -- 172.21.15.32:0/2672118330 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1170106a00 con 0x7f1170105e90 2024-04-25T00:23:36.297 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.440+0000 7f116ffff700 1 --2- 172.21.15.32:0/2672118330 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1170105e90 0x7f11701062b0 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f11600098a0 tx=0x7f1160009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3e3d3e7c54ae88a1 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:36.297 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.440+0000 7f116effd700 1 -- 172.21.15.32:0/2672118330 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f1160004160 con 0x7f1170105e90 2024-04-25T00:23:36.297 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.440+0000 7f116effd700 1 -- 172.21.15.32:0/2672118330 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1160004300 con 0x7f1170105e90 2024-04-25T00:23:36.297 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.441+0000 7f116effd700 1 -- 172.21.15.32:0/2672118330 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f1160011b20 con 0x7f1170105e90 2024-04-25T00:23:36.297 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.441+0000 7f11767bf700 1 -- 172.21.15.32:0/2672118330 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1170105e90 msgr2=0x7f11701062b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:36.297 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.441+0000 7f11767bf700 1 --2- 172.21.15.32:0/2672118330 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1170105e90 0x7f11701062b0 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f11600098a0 tx=0x7f1160009bb0 comp rx=0 tx=0).stop 2024-04-25T00:23:36.297 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.441+0000 7f11767bf700 1 -- 172.21.15.32:0/2672118330 shutdown_connections 2024-04-25T00:23:36.297 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.441+0000 7f11767bf700 1 --2- 172.21.15.32:0/2672118330 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1170105e90 0x7f11701062b0 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:36.298 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.441+0000 7f11767bf700 1 -- 172.21.15.32:0/2672118330 >> 172.21.15.32:0/2672118330 conn(0x7f1170101830 msgr2=0x7f1170103c50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:36.298 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.441+0000 7f11767bf700 1 -- 172.21.15.32:0/2672118330 shutdown_connections 2024-04-25T00:23:36.298 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.441+0000 7f11767bf700 1 -- 172.21.15.32:0/2672118330 wait complete. 2024-04-25T00:23:36.298 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.442+0000 7f11767bf700 1 Processor -- start 2024-04-25T00:23:36.298 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.442+0000 7f11767bf700 1 -- start start 2024-04-25T00:23:36.298 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.442+0000 7f11767bf700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1170105e90 0x7f117019bb90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:36.298 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.442+0000 7f11767bf700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f117010c140 con 0x7f1170105e90 2024-04-25T00:23:36.298 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.442+0000 7f116ffff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1170105e90 0x7f117019bb90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:36.298 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.442+0000 7f116ffff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1170105e90 0x7f117019bb90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40716/0 (socket says 172.21.15.32:40716) 2024-04-25T00:23:36.298 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.442+0000 7f116ffff700 1 -- 172.21.15.32:0/845393168 learned_addr learned my addr 172.21.15.32:0/845393168 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:36.298 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.442+0000 7f116ffff700 1 -- 172.21.15.32:0/845393168 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f117019c0d0 con 0x7f1170105e90 2024-04-25T00:23:36.298 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.443+0000 7f116ffff700 1 --2- 172.21.15.32:0/845393168 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1170105e90 0x7f117019bb90 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f11600117e0 tx=0x7f1160011c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:36.298 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.443+0000 7f116d7fa700 1 -- 172.21.15.32:0/845393168 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f1160003bf0 con 0x7f1170105e90 2024-04-25T00:23:36.298 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.443+0000 7f116d7fa700 1 -- 172.21.15.32:0/845393168 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f116001a910 con 0x7f1170105e90 2024-04-25T00:23:36.299 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.443+0000 7f11767bf700 1 -- 172.21.15.32:0/845393168 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f117019c360 con 0x7f1170105e90 2024-04-25T00:23:36.299 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.443+0000 7f116d7fa700 1 -- 172.21.15.32:0/845393168 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f11600249a0 con 0x7f1170105e90 2024-04-25T00:23:36.299 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.443+0000 7f11767bf700 1 -- 172.21.15.32:0/845393168 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f117019c7c0 con 0x7f1170105e90 2024-04-25T00:23:36.299 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.444+0000 7f116d7fa700 1 -- 172.21.15.32:0/845393168 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 8) ==== 46287+0+0 (secure 0 0 0) 0x7f1160022020 con 0x7f1170105e90 2024-04-25T00:23:36.299 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.444+0000 7f116d7fa700 1 --2- 172.21.15.32:0/845393168 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f1158039050 0x7f115803b510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:36.299 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.444+0000 7f116d7fa700 1 -- 172.21.15.32:0/845393168 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7f116001a600 con 0x7f1170105e90 2024-04-25T00:23:36.299 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.444+0000 7f11767bf700 1 -- 172.21.15.32:0/845393168 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1170063890 con 0x7f1170105e90 2024-04-25T00:23:36.299 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.444+0000 7f116f7fe700 1 --2- 172.21.15.32:0/845393168 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f1158039050 0x7f115803b510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:36.299 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.444+0000 7f116f7fe700 1 --2- 172.21.15.32:0/845393168 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f1158039050 0x7f115803b510 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f1164006fd0 tx=0x7f1164006e40 comp rx=0 tx=0).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:36.299 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.449+0000 7f116d7fa700 1 -- 172.21.15.32:0/845393168 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f1160011e20 con 0x7f1170105e90 2024-04-25T00:23:36.299 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:34.731+0000 7f11767bf700 1 -- 172.21.15.32:0/845393168 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "mgr module enable", "module": "dashboard"} v 0) -- 0x7f117019f170 con 0x7f1170105e90 2024-04-25T00:23:36.299 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:35.623+0000 7f116d7fa700 1 -- 172.21.15.32:0/845393168 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "mgr module enable", "module": "dashboard"}]=0 v9) ==== 88+0+0 (secure 0 0 0) 0x7f11600172b0 con 0x7f1170105e90 2024-04-25T00:23:36.299 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:35.625+0000 7f116d7fa700 1 -- 172.21.15.32:0/845393168 <== mon.0 v2:172.21.15.32:3300/0 8 ==== mgrmap(e 9) ==== 46300+0+0 (secure 0 0 0) 0x7f116002d910 con 0x7f1170105e90 2024-04-25T00:23:36.299 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:35.627+0000 7f11767bf700 1 -- 172.21.15.32:0/845393168 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f1158039050 msgr2=0x7f115803b510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:36.299 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:35.627+0000 7f11767bf700 1 --2- 172.21.15.32:0/845393168 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f1158039050 0x7f115803b510 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f1164006fd0 tx=0x7f1164006e40 comp rx=0 tx=0).stop 2024-04-25T00:23:36.300 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:35.627+0000 7f11767bf700 1 -- 172.21.15.32:0/845393168 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1170105e90 msgr2=0x7f117019bb90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:36.300 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:35.627+0000 7f11767bf700 1 --2- 172.21.15.32:0/845393168 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1170105e90 0x7f117019bb90 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f11600117e0 tx=0x7f1160011c10 comp rx=0 tx=0).stop 2024-04-25T00:23:36.300 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:35.627+0000 7f11767bf700 1 -- 172.21.15.32:0/845393168 shutdown_connections 2024-04-25T00:23:36.300 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:35.627+0000 7f11767bf700 1 --2- 172.21.15.32:0/845393168 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f1158039050 0x7f115803b510 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:36.300 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:35.627+0000 7f11767bf700 1 --2- 172.21.15.32:0/845393168 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1170105e90 0x7f117019bb90 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:36.300 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:35.627+0000 7f11767bf700 1 -- 172.21.15.32:0/845393168 >> 172.21.15.32:0/845393168 conn(0x7f1170101830 msgr2=0x7f11701022f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:36.300 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:35.628+0000 7f11767bf700 1 -- 172.21.15.32:0/845393168 shutdown_connections 2024-04-25T00:23:36.300 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:35.628+0000 7f11767bf700 1 -- 172.21.15.32:0/845393168 wait complete. 2024-04-25T00:23:36.560 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:36 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:36.438+0000 7f97a48ad200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-25T00:23:36.560 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:36 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:36.526+0000 7f97a48ad200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-25T00:23:36.839 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:36 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/845393168' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-25T00:23:36.839 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:36 smithi032 ceph-mon[29357]: mgrmap e9: x(active, since 21s) 2024-04-25T00:23:37.645 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout { 2024-04-25T00:23:37.645 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-04-25T00:23:37.646 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-25T00:23:37.646 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "active_name": "x", 2024-04-25T00:23:37.646 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-25T00:23:37.646 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout } 2024-04-25T00:23:37.646 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.771+0000 7f6d25d7c700 1 Processor -- start 2024-04-25T00:23:37.646 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.771+0000 7f6d25d7c700 1 -- start start 2024-04-25T00:23:37.646 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.772+0000 7f6d25d7c700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6d2007de00 0x7f6d2007c300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:37.646 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.772+0000 7f6d25d7c700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f6d2007c840 con 0x7f6d2007de00 2024-04-25T00:23:37.646 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.772+0000 7f6d1f7fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6d2007de00 0x7f6d2007c300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:37.646 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.772+0000 7f6d1f7fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6d2007de00 0x7f6d2007c300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40740/0 (socket says 172.21.15.32:40740) 2024-04-25T00:23:37.646 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.772+0000 7f6d1f7fe700 1 -- 172.21.15.32:0/2030232614 learned_addr learned my addr 172.21.15.32:0/2030232614 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:37.646 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.772+0000 7f6d1f7fe700 1 -- 172.21.15.32:0/2030232614 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6d2007c9c0 con 0x7f6d2007de00 2024-04-25T00:23:37.646 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.772+0000 7f6d1f7fe700 1 --2- 172.21.15.32:0/2030232614 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6d2007de00 0x7f6d2007c300 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f6d100098a0 tx=0x7f6d10009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1658dc45ee67d6cf server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:37.647 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.773+0000 7f6d1e7fc700 1 -- 172.21.15.32:0/2030232614 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f6d10004160 con 0x7f6d2007de00 2024-04-25T00:23:37.647 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.773+0000 7f6d1e7fc700 1 -- 172.21.15.32:0/2030232614 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6d10004300 con 0x7f6d2007de00 2024-04-25T00:23:37.647 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.773+0000 7f6d1e7fc700 1 -- 172.21.15.32:0/2030232614 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f6d10011b20 con 0x7f6d2007de00 2024-04-25T00:23:37.647 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.773+0000 7f6d25d7c700 1 -- 172.21.15.32:0/2030232614 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6d2007de00 msgr2=0x7f6d2007c300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:37.647 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.773+0000 7f6d25d7c700 1 --2- 172.21.15.32:0/2030232614 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6d2007de00 0x7f6d2007c300 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f6d100098a0 tx=0x7f6d10009bb0 comp rx=0 tx=0).stop 2024-04-25T00:23:37.647 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.773+0000 7f6d25d7c700 1 -- 172.21.15.32:0/2030232614 shutdown_connections 2024-04-25T00:23:37.647 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.773+0000 7f6d25d7c700 1 --2- 172.21.15.32:0/2030232614 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6d2007de00 0x7f6d2007c300 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:37.647 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.773+0000 7f6d25d7c700 1 -- 172.21.15.32:0/2030232614 >> 172.21.15.32:0/2030232614 conn(0x7f6d20103230 msgr2=0x7f6d20105650 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:37.647 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.774+0000 7f6d25d7c700 1 -- 172.21.15.32:0/2030232614 shutdown_connections 2024-04-25T00:23:37.647 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.774+0000 7f6d25d7c700 1 -- 172.21.15.32:0/2030232614 wait complete. 2024-04-25T00:23:37.647 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.774+0000 7f6d25d7c700 1 Processor -- start 2024-04-25T00:23:37.647 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.775+0000 7f6d25d7c700 1 -- start start 2024-04-25T00:23:37.647 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.775+0000 7f6d25d7c700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6d2007de00 0x7f6d2019dce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:37.647 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.775+0000 7f6d25d7c700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f6d2010e190 con 0x7f6d2007de00 2024-04-25T00:23:37.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.775+0000 7f6d1f7fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6d2007de00 0x7f6d2019dce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:37.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.775+0000 7f6d1f7fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6d2007de00 0x7f6d2019dce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40746/0 (socket says 172.21.15.32:40746) 2024-04-25T00:23:37.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.775+0000 7f6d1f7fe700 1 -- 172.21.15.32:0/1881799781 learned_addr learned my addr 172.21.15.32:0/1881799781 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:37.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.775+0000 7f6d1f7fe700 1 -- 172.21.15.32:0/1881799781 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6d2019e220 con 0x7f6d2007de00 2024-04-25T00:23:37.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.775+0000 7f6d1f7fe700 1 --2- 172.21.15.32:0/1881799781 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6d2007de00 0x7f6d2019dce0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f6d100117e0 tx=0x7f6d10011c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:37.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.776+0000 7f6d1cff9700 1 -- 172.21.15.32:0/1881799781 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f6d10003bf0 con 0x7f6d2007de00 2024-04-25T00:23:37.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.776+0000 7f6d25d7c700 1 -- 172.21.15.32:0/1881799781 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6d2019e4b0 con 0x7f6d2007de00 2024-04-25T00:23:37.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.776+0000 7f6d25d7c700 1 -- 172.21.15.32:0/1881799781 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f6d2019e910 con 0x7f6d2007de00 2024-04-25T00:23:37.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.776+0000 7f6d1cff9700 1 -- 172.21.15.32:0/1881799781 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6d1001a910 con 0x7f6d2007de00 2024-04-25T00:23:37.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.776+0000 7f6d1cff9700 1 -- 172.21.15.32:0/1881799781 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f6d10024970 con 0x7f6d2007de00 2024-04-25T00:23:37.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.777+0000 7f6d25d7c700 1 -- 172.21.15.32:0/1881799781 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6d20063890 con 0x7f6d2007de00 2024-04-25T00:23:37.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.777+0000 7f6d1cff9700 1 -- 172.21.15.32:0/1881799781 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 9) ==== 46300+0+0 (secure 0 0 0) 0x7f6d10022020 con 0x7f6d2007de00 2024-04-25T00:23:37.648 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.777+0000 7f6d1cff9700 1 --2- 172.21.15.32:0/1881799781 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f6d080390b0 0x7f6d0803b570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:37.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.777+0000 7f6d1cff9700 1 -- 172.21.15.32:0/1881799781 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7f6d1004d3d0 con 0x7f6d2007de00 2024-04-25T00:23:37.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.777+0000 7f6d1effd700 1 -- 172.21.15.32:0/1881799781 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f6d080390b0 msgr2=0x7f6d0803b570 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/4160192081 2024-04-25T00:23:37.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.777+0000 7f6d1effd700 1 --2- 172.21.15.32:0/1881799781 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f6d080390b0 0x7f6d0803b570 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-25T00:23:37.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.786+0000 7f6d1cff9700 1 -- 172.21.15.32:0/1881799781 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f6d10011e20 con 0x7f6d2007de00 2024-04-25T00:23:37.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.978+0000 7f6d1effd700 1 -- 172.21.15.32:0/1881799781 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f6d080390b0 msgr2=0x7f6d0803b570 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/4160192081 2024-04-25T00:23:37.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:36.978+0000 7f6d1effd700 1 --2- 172.21.15.32:0/1881799781 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f6d080390b0 0x7f6d0803b570 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-04-25T00:23:37.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:37.099+0000 7f6d25d7c700 1 -- 172.21.15.32:0/1881799781 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "mgr stat"} v 0) -- 0x7f6d201a1740 con 0x7f6d2007de00 2024-04-25T00:23:37.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:37.099+0000 7f6d1cff9700 1 -- 172.21.15.32:0/1881799781 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "mgr stat"}]=0 v9) ==== 56+0+88 (secure 0 0 0) 0x7f6d1002ad90 con 0x7f6d2007de00 2024-04-25T00:23:37.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:37.102+0000 7f6d25d7c700 1 -- 172.21.15.32:0/1881799781 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f6d080390b0 msgr2=0x7f6d0803b570 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:23:37.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:37.102+0000 7f6d25d7c700 1 --2- 172.21.15.32:0/1881799781 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f6d080390b0 0x7f6d0803b570 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:37.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:37.102+0000 7f6d25d7c700 1 -- 172.21.15.32:0/1881799781 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6d2007de00 msgr2=0x7f6d2019dce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:37.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:37.102+0000 7f6d25d7c700 1 --2- 172.21.15.32:0/1881799781 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6d2007de00 0x7f6d2019dce0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f6d100117e0 tx=0x7f6d10011c10 comp rx=0 tx=0).stop 2024-04-25T00:23:37.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:37.102+0000 7f6d25d7c700 1 -- 172.21.15.32:0/1881799781 shutdown_connections 2024-04-25T00:23:37.649 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:37.102+0000 7f6d25d7c700 1 --2- 172.21.15.32:0/1881799781 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f6d080390b0 0x7f6d0803b570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:37.650 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:37.102+0000 7f6d25d7c700 1 --2- 172.21.15.32:0/1881799781 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6d2007de00 0x7f6d2019dce0 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:37.650 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:37.102+0000 7f6d25d7c700 1 -- 172.21.15.32:0/1881799781 >> 172.21.15.32:0/1881799781 conn(0x7f6d20103230 msgr2=0x7f6d20103d00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:37.650 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:37.102+0000 7f6d25d7c700 1 -- 172.21.15.32:0/1881799781 shutdown_connections 2024-04-25T00:23:37.650 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:37.102+0000 7f6d25d7c700 1 -- 172.21.15.32:0/1881799781 wait complete. 2024-04-25T00:23:37.650 INFO:teuthology.orchestra.run.smithi032.stdout:Waiting for the mgr to restart... 2024-04-25T00:23:37.650 INFO:teuthology.orchestra.run.smithi032.stdout:Waiting for mgr epoch 9... 2024-04-25T00:23:37.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:37 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1881799781' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-25T00:23:38.022 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:37 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:37.824+0000 7f97a48ad200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-25T00:23:38.023 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:38 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:38.007+0000 7f97a48ad200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-25T00:23:38.336 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:38 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:38.187+0000 7f97a48ad200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-25T00:23:38.736 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:38 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:38.487+0000 7f97a48ad200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-25T00:23:38.736 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:38 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:38.583+0000 7f97a48ad200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-25T00:23:38.994 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:38 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:38.793+0000 7f97a48ad200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-25T00:23:39.248 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:38 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:38.992+0000 7f97a48ad200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-25T00:23:39.998 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:39 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:39.620+0000 7f97a48ad200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-25T00:23:40.368 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:40 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:40.059+0000 7f97a48ad200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-25T00:23:40.368 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:40 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:40.159+0000 7f97a48ad200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-25T00:23:40.368 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:40 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:40.367+0000 7f97a48ad200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-25T00:23:41.248 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:40 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:40.911+0000 7f97a48ad200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-25T00:23:41.248 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:41 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:41.005+0000 7f97a48ad200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-25T00:23:41.615 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:41 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:41.257+0000 7f97a48ad200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-25T00:23:41.615 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:41 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:41.396+0000 7f97a48ad200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-25T00:23:41.615 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:41 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:41.494+0000 7f97a48ad200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-25T00:23:41.615 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:41 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:41.614+0000 7f97a48ad200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-25T00:23:41.998 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:41 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:41.708+0000 7f97a48ad200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-25T00:23:42.827 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:42 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:42.551+0000 7f97a48ad200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-25T00:23:42.827 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:23:42 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:23:42.774+0000 7f97a48ad200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-25T00:23:43.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:42 smithi032 ceph-mon[29357]: Active manager daemon x restarted 2024-04-25T00:23:43.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:42 smithi032 ceph-mon[29357]: Activating manager daemon x 2024-04-25T00:23:43.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:42 smithi032 ceph-mon[29357]: osdmap e3: 0 total, 0 up, 0 in 2024-04-25T00:23:43.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:42 smithi032 ceph-mon[29357]: mgrmap e10: x(active, starting, since 0.00337027s) 2024-04-25T00:23:43.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-25T00:23:43.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-25T00:23:43.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-25T00:23:43.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-25T00:23:43.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-25T00:23:43.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:42 smithi032 ceph-mon[29357]: Manager daemon x is now available 2024-04-25T00:23:43.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:23:43.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-25T00:23:43.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-25T00:23:43.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:43 smithi032 ceph-mon[29357]: [25/Apr/2024:00:23:43] ENGINE Bus STARTING 2024-04-25T00:23:43.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:43 smithi032 ceph-mon[29357]: [25/Apr/2024:00:23:43] ENGINE Serving on http://172.21.15.32:8765 2024-04-25T00:23:43.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:43 smithi032 ceph-mon[29357]: [25/Apr/2024:00:23:43] ENGINE Serving on https://172.21.15.32:7150 2024-04-25T00:23:43.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:43 smithi032 ceph-mon[29357]: [25/Apr/2024:00:23:43] ENGINE Bus STARTED 2024-04-25T00:23:43.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:43 smithi032 ceph-mon[29357]: mgrmap e11: x(active, since 1.00594s) 2024-04-25T00:23:44.321 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout { 2024-04-25T00:23:44.321 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-04-25T00:23:44.321 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-25T00:23:44.321 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout } 2024-04-25T00:23:44.321 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.128+0000 7f289d120700 1 Processor -- start 2024-04-25T00:23:44.321 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.128+0000 7f289d120700 1 -- start start 2024-04-25T00:23:44.321 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.128+0000 7f289d120700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f28981086c0 0x7f289810aab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:44.321 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.128+0000 7f289d120700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f289807be70 con 0x7f28981086c0 2024-04-25T00:23:44.322 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.129+0000 7f2896d9d700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f28981086c0 0x7f289810aab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:44.322 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.129+0000 7f2896d9d700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f28981086c0 0x7f289810aab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40754/0 (socket says 172.21.15.32:40754) 2024-04-25T00:23:44.322 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.129+0000 7f2896d9d700 1 -- 172.21.15.32:0/2592493623 learned_addr learned my addr 172.21.15.32:0/2592493623 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:44.322 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.129+0000 7f2896d9d700 1 -- 172.21.15.32:0/2592493623 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f289810aff0 con 0x7f28981086c0 2024-04-25T00:23:44.322 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.129+0000 7f2896d9d700 1 --2- 172.21.15.32:0/2592493623 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f28981086c0 0x7f289810aab0 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f28880098a0 tx=0x7f2888009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f25c6c9b5a4ec230 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:44.322 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.129+0000 7f2895d9b700 1 -- 172.21.15.32:0/2592493623 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f2888004160 con 0x7f28981086c0 2024-04-25T00:23:44.322 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.129+0000 7f2895d9b700 1 -- 172.21.15.32:0/2592493623 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f2888004300 con 0x7f28981086c0 2024-04-25T00:23:44.322 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.130+0000 7f2895d9b700 1 -- 172.21.15.32:0/2592493623 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f2888011b20 con 0x7f28981086c0 2024-04-25T00:23:44.322 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.130+0000 7f289d120700 1 -- 172.21.15.32:0/2592493623 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f28981086c0 msgr2=0x7f289810aab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:44.322 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.130+0000 7f289d120700 1 --2- 172.21.15.32:0/2592493623 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f28981086c0 0x7f289810aab0 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f28880098a0 tx=0x7f2888009bb0 comp rx=0 tx=0).stop 2024-04-25T00:23:44.322 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.130+0000 7f289d120700 1 -- 172.21.15.32:0/2592493623 shutdown_connections 2024-04-25T00:23:44.322 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.130+0000 7f289d120700 1 --2- 172.21.15.32:0/2592493623 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f28981086c0 0x7f289810aab0 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:44.322 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.130+0000 7f289d120700 1 -- 172.21.15.32:0/2592493623 >> 172.21.15.32:0/2592493623 conn(0x7f28981020d0 msgr2=0x7f28981044f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:44.323 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.130+0000 7f289d120700 1 -- 172.21.15.32:0/2592493623 shutdown_connections 2024-04-25T00:23:44.323 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.130+0000 7f289d120700 1 -- 172.21.15.32:0/2592493623 wait complete. 2024-04-25T00:23:44.323 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.131+0000 7f289d120700 1 Processor -- start 2024-04-25T00:23:44.323 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.131+0000 7f289d120700 1 -- start start 2024-04-25T00:23:44.323 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.131+0000 7f289d120700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f28981086c0 0x7f2898107f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:44.323 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.131+0000 7f289d120700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f289810c000 con 0x7f28981086c0 2024-04-25T00:23:44.323 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.132+0000 7f2896d9d700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f28981086c0 0x7f2898107f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:44.323 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.132+0000 7f2896d9d700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f28981086c0 0x7f2898107f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40764/0 (socket says 172.21.15.32:40764) 2024-04-25T00:23:44.323 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.132+0000 7f2896d9d700 1 -- 172.21.15.32:0/2112749089 learned_addr learned my addr 172.21.15.32:0/2112749089 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:44.323 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.132+0000 7f2896d9d700 1 -- 172.21.15.32:0/2112749089 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2898108480 con 0x7f28981086c0 2024-04-25T00:23:44.323 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.132+0000 7f2896d9d700 1 --2- 172.21.15.32:0/2112749089 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f28981086c0 0x7f2898107f40 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f28880117e0 tx=0x7f2888011c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:44.323 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.132+0000 7f287ffff700 1 -- 172.21.15.32:0/2112749089 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f2888003bf0 con 0x7f28981086c0 2024-04-25T00:23:44.323 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.132+0000 7f287ffff700 1 -- 172.21.15.32:0/2112749089 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f288801a910 con 0x7f28981086c0 2024-04-25T00:23:44.323 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.132+0000 7f289d120700 1 -- 172.21.15.32:0/2112749089 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2898106670 con 0x7f28981086c0 2024-04-25T00:23:44.323 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.132+0000 7f287ffff700 1 -- 172.21.15.32:0/2112749089 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f2888024970 con 0x7f28981086c0 2024-04-25T00:23:44.324 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.132+0000 7f289d120700 1 -- 172.21.15.32:0/2112749089 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2898106b30 con 0x7f28981086c0 2024-04-25T00:23:44.324 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.133+0000 7f287ffff700 1 -- 172.21.15.32:0/2112749089 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 9) ==== 46300+0+0 (secure 0 0 0) 0x7f2888022020 con 0x7f28981086c0 2024-04-25T00:23:44.324 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.133+0000 7f287ffff700 1 --2- 172.21.15.32:0/2112749089 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f2880039050 0x7f288003b510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:44.324 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.133+0000 7f287ffff700 1 -- 172.21.15.32:0/2112749089 --> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x7f288003bc20 con 0x7f2880039050 2024-04-25T00:23:44.324 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.133+0000 7f287ffff700 1 -- 172.21.15.32:0/2112749089 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7f288804d430 con 0x7f28981086c0 2024-04-25T00:23:44.324 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.133+0000 7f289659c700 1 -- 172.21.15.32:0/2112749089 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f2880039050 msgr2=0x7f288003b510 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/4160192081 2024-04-25T00:23:44.324 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.133+0000 7f289659c700 1 --2- 172.21.15.32:0/2112749089 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f2880039050 0x7f288003b510 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-25T00:23:44.324 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.334+0000 7f289659c700 1 -- 172.21.15.32:0/2112749089 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f2880039050 msgr2=0x7f288003b510 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/4160192081 2024-04-25T00:23:44.324 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.334+0000 7f289659c700 1 --2- 172.21.15.32:0/2112749089 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f2880039050 0x7f288003b510 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-04-25T00:23:44.324 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.735+0000 7f289659c700 1 -- 172.21.15.32:0/2112749089 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f2880039050 msgr2=0x7f288003b510 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/4160192081 2024-04-25T00:23:44.324 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:38.735+0000 7f289659c700 1 --2- 172.21.15.32:0/2112749089 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f2880039050 0x7f288003b510 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2024-04-25T00:23:44.324 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:39.537+0000 7f289659c700 1 -- 172.21.15.32:0/2112749089 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f2880039050 msgr2=0x7f288003b510 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/4160192081 2024-04-25T00:23:44.324 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:39.537+0000 7f289659c700 1 --2- 172.21.15.32:0/2112749089 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f2880039050 0x7f288003b510 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 1.600000 2024-04-25T00:23:44.324 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:41.138+0000 7f289659c700 1 -- 172.21.15.32:0/2112749089 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f2880039050 msgr2=0x7f288003b510 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/4160192081 2024-04-25T00:23:44.325 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:41.138+0000 7f289659c700 1 --2- 172.21.15.32:0/2112749089 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f2880039050 0x7f288003b510 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 3.200000 2024-04-25T00:23:44.325 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:42.777+0000 7f287ffff700 1 -- 172.21.15.32:0/2112749089 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mgrmap(e 10) ==== 46067+0+0 (secure 0 0 0) 0x7f2888024c20 con 0x7f28981086c0 2024-04-25T00:23:44.325 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:42.777+0000 7f287ffff700 1 -- 172.21.15.32:0/2112749089 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f2880039050 msgr2=0x7f288003b510 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:23:44.325 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:42.777+0000 7f287ffff700 1 --2- 172.21.15.32:0/2112749089 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f2880039050 0x7f288003b510 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:44.325 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:43.779+0000 7f287ffff700 1 -- 172.21.15.32:0/2112749089 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mgrmap(e 11) ==== 46194+0+0 (secure 0 0 0) 0x7f2888023d60 con 0x7f28981086c0 2024-04-25T00:23:44.325 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:43.779+0000 7f287ffff700 1 --2- 172.21.15.32:0/2112749089 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f288003cb10 0x7f288003ef00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:44.325 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:43.779+0000 7f287ffff700 1 -- 172.21.15.32:0/2112749089 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x7f2888019e50 con 0x7f288003cb10 2024-04-25T00:23:44.325 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:43.779+0000 7f289659c700 1 --2- 172.21.15.32:0/2112749089 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f288003cb10 0x7f288003ef00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:44.325 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:43.780+0000 7f289659c700 1 --2- 172.21.15.32:0/2112749089 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f288003cb10 0x7f288003ef00 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7f288c003a10 tx=0x7f288c0092b0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:44.325 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:43.781+0000 7f287ffff700 1 -- 172.21.15.32:0/2112749089 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== command_reply(tid 0: 0 ) ==== 8+0+7774 (secure 0 0 0) 0x7f2888019e50 con 0x7f288003cb10 2024-04-25T00:23:44.325 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:43.788+0000 7f289d120700 1 -- 172.21.15.32:0/2112749089 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- command(tid 1: {"prefix": "mgr_status"}) -- 0x7f2898063890 con 0x7f288003cb10 2024-04-25T00:23:44.325 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:43.788+0000 7f287ffff700 1 -- 172.21.15.32:0/2112749089 <== mgr.14148 v2:172.21.15.32:6800/585810931 2 ==== command_reply(tid 1: 0 ) ==== 8+0+52 (secure 0 0 0) 0x7f2898063890 con 0x7f288003cb10 2024-04-25T00:23:44.325 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:43.788+0000 7f289d120700 1 -- 172.21.15.32:0/2112749089 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f288003cb10 msgr2=0x7f288003ef00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:44.325 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:43.788+0000 7f289d120700 1 --2- 172.21.15.32:0/2112749089 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f288003cb10 0x7f288003ef00 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7f288c003a10 tx=0x7f288c0092b0 comp rx=0 tx=0).stop 2024-04-25T00:23:44.325 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:43.788+0000 7f289d120700 1 -- 172.21.15.32:0/2112749089 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f28981086c0 msgr2=0x7f2898107f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:44.325 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:43.788+0000 7f289d120700 1 --2- 172.21.15.32:0/2112749089 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f28981086c0 0x7f2898107f40 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f28880117e0 tx=0x7f2888011c10 comp rx=0 tx=0).stop 2024-04-25T00:23:44.326 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:43.788+0000 7f289d120700 1 -- 172.21.15.32:0/2112749089 shutdown_connections 2024-04-25T00:23:44.326 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:43.788+0000 7f289d120700 1 --2- 172.21.15.32:0/2112749089 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f288003cb10 0x7f288003ef00 unknown :-1 s=CLOSED pgs=2 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:44.326 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:43.788+0000 7f289d120700 1 --2- 172.21.15.32:0/2112749089 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f28981086c0 0x7f2898107f40 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:44.326 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:43.788+0000 7f289d120700 1 --2- 172.21.15.32:0/2112749089 >> [v2:172.21.15.32:6800/4160192081,v1:172.21.15.32:6801/4160192081] conn(0x7f2880039050 0x7f288003b510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:44.326 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:43.788+0000 7f289d120700 1 -- 172.21.15.32:0/2112749089 >> 172.21.15.32:0/2112749089 conn(0x7f28981020d0 msgr2=0x7f2898102b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:44.326 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:43.788+0000 7f289d120700 1 -- 172.21.15.32:0/2112749089 shutdown_connections 2024-04-25T00:23:44.326 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:43.788+0000 7f289d120700 1 -- 172.21.15.32:0/2112749089 wait complete. 2024-04-25T00:23:44.326 INFO:teuthology.orchestra.run.smithi032.stdout:mgr epoch 9 is available 2024-04-25T00:23:44.326 INFO:teuthology.orchestra.run.smithi032.stdout:Generating a dashboard self-signed certificate... 2024-04-25T00:23:45.096 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:44 smithi032 ceph-mon[29357]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-25T00:23:45.096 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:44 smithi032 ceph-mon[29357]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-25T00:23:45.684 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-04-25T00:23:45.685 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.786+0000 7fca2ad12700 1 Processor -- start 2024-04-25T00:23:45.685 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.786+0000 7fca2ad12700 1 -- start start 2024-04-25T00:23:45.685 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.786+0000 7fca2ad12700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fca24107bf0 0x7fca24108010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:45.685 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.786+0000 7fca2ad12700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fca241085e0 con 0x7fca24107bf0 2024-04-25T00:23:45.685 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.787+0000 7fca28aae700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fca24107bf0 0x7fca24108010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:45.685 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.787+0000 7fca28aae700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fca24107bf0 0x7fca24108010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:37188/0 (socket says 172.21.15.32:37188) 2024-04-25T00:23:45.685 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.787+0000 7fca28aae700 1 -- 172.21.15.32:0/2588297938 learned_addr learned my addr 172.21.15.32:0/2588297938 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:45.685 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.787+0000 7fca28aae700 1 -- 172.21.15.32:0/2588297938 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fca24108da0 con 0x7fca24107bf0 2024-04-25T00:23:45.685 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.787+0000 7fca28aae700 1 --2- 172.21.15.32:0/2588297938 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fca24107bf0 0x7fca24108010 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7fca180098a0 tx=0x7fca18009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3015181eb17148ef server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:45.685 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.787+0000 7fca237fe700 1 -- 172.21.15.32:0/2588297938 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fca18004160 con 0x7fca24107bf0 2024-04-25T00:23:45.685 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.787+0000 7fca237fe700 1 -- 172.21.15.32:0/2588297938 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fca18004300 con 0x7fca24107bf0 2024-04-25T00:23:45.686 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.787+0000 7fca237fe700 1 -- 172.21.15.32:0/2588297938 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fca18011b20 con 0x7fca24107bf0 2024-04-25T00:23:45.686 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.788+0000 7fca2ad12700 1 -- 172.21.15.32:0/2588297938 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fca24107bf0 msgr2=0x7fca24108010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:45.686 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.788+0000 7fca2ad12700 1 --2- 172.21.15.32:0/2588297938 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fca24107bf0 0x7fca24108010 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7fca180098a0 tx=0x7fca18009bb0 comp rx=0 tx=0).stop 2024-04-25T00:23:45.686 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.788+0000 7fca2ad12700 1 -- 172.21.15.32:0/2588297938 shutdown_connections 2024-04-25T00:23:45.686 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.788+0000 7fca2ad12700 1 --2- 172.21.15.32:0/2588297938 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fca24107bf0 0x7fca24108010 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:45.686 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.788+0000 7fca2ad12700 1 -- 172.21.15.32:0/2588297938 >> 172.21.15.32:0/2588297938 conn(0x7fca24103170 msgr2=0x7fca241055d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:45.686 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.788+0000 7fca2ad12700 1 -- 172.21.15.32:0/2588297938 shutdown_connections 2024-04-25T00:23:45.686 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.788+0000 7fca2ad12700 1 -- 172.21.15.32:0/2588297938 wait complete. 2024-04-25T00:23:45.686 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.789+0000 7fca2ad12700 1 Processor -- start 2024-04-25T00:23:45.686 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.789+0000 7fca2ad12700 1 -- start start 2024-04-25T00:23:45.686 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.789+0000 7fca2ad12700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fca24107bf0 0x7fca2419bb10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:45.686 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.789+0000 7fca2ad12700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fca2410c150 con 0x7fca24107bf0 2024-04-25T00:23:45.686 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.789+0000 7fca28aae700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fca24107bf0 0x7fca2419bb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:45.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.789+0000 7fca28aae700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fca24107bf0 0x7fca2419bb10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:37194/0 (socket says 172.21.15.32:37194) 2024-04-25T00:23:45.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.789+0000 7fca28aae700 1 -- 172.21.15.32:0/2865575464 learned_addr learned my addr 172.21.15.32:0/2865575464 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:45.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.789+0000 7fca28aae700 1 -- 172.21.15.32:0/2865575464 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fca2419c050 con 0x7fca24107bf0 2024-04-25T00:23:45.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.789+0000 7fca28aae700 1 --2- 172.21.15.32:0/2865575464 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fca24107bf0 0x7fca2419bb10 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7fca180117e0 tx=0x7fca18011c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:45.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.790+0000 7fca21ffb700 1 -- 172.21.15.32:0/2865575464 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fca18003bf0 con 0x7fca24107bf0 2024-04-25T00:23:45.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.790+0000 7fca21ffb700 1 -- 172.21.15.32:0/2865575464 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fca1801a910 con 0x7fca24107bf0 2024-04-25T00:23:45.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.790+0000 7fca2ad12700 1 -- 172.21.15.32:0/2865575464 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fca2419c2e0 con 0x7fca24107bf0 2024-04-25T00:23:45.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.790+0000 7fca21ffb700 1 -- 172.21.15.32:0/2865575464 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fca180249a0 con 0x7fca24107bf0 2024-04-25T00:23:45.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.790+0000 7fca2ad12700 1 -- 172.21.15.32:0/2865575464 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fca2419c740 con 0x7fca24107bf0 2024-04-25T00:23:45.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.791+0000 7fca21ffb700 1 -- 172.21.15.32:0/2865575464 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 11) ==== 46194+0+0 (secure 0 0 0) 0x7fca18022020 con 0x7fca24107bf0 2024-04-25T00:23:45.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.791+0000 7fca21ffb700 1 --2- 172.21.15.32:0/2865575464 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fca14038f90 0x7fca1403b450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:45.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.791+0000 7fca21ffb700 1 -- 172.21.15.32:0/2865575464 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7fca1804d2d0 con 0x7fca24107bf0 2024-04-25T00:23:45.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.791+0000 7fca23fff700 1 --2- 172.21.15.32:0/2865575464 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fca14038f90 0x7fca1403b450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:45.687 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.791+0000 7fca2ad12700 1 -- 172.21.15.32:0/2865575464 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fca24063890 con 0x7fca24107bf0 2024-04-25T00:23:45.688 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.792+0000 7fca23fff700 1 --2- 172.21.15.32:0/2865575464 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fca14038f90 0x7fca1403b450 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7fca10006fd0 tx=0x7fca10006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:45.688 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.797+0000 7fca21ffb700 1 -- 172.21.15.32:0/2865575464 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fca18011e20 con 0x7fca24107bf0 2024-04-25T00:23:45.688 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:44.833+0000 7fca21ffb700 1 -- 172.21.15.32:0/2865575464 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mgrmap(e 12) ==== 46300+0+0 (secure 0 0 0) 0x7fca1802d910 con 0x7fca24107bf0 2024-04-25T00:23:45.688 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:45.012+0000 7fca2ad12700 1 -- 172.21.15.32:0/2865575464 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}) -- 0x7fca24104390 con 0x7fca14038f90 2024-04-25T00:23:45.688 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:45.095+0000 7fca21ffb700 1 -- 172.21.15.32:0/2865575464 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+32 (secure 0 0 0) 0x7fca24104390 con 0x7fca14038f90 2024-04-25T00:23:45.688 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:45.097+0000 7fca2ad12700 1 -- 172.21.15.32:0/2865575464 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fca14038f90 msgr2=0x7fca1403b450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:45.688 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:45.097+0000 7fca2ad12700 1 --2- 172.21.15.32:0/2865575464 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fca14038f90 0x7fca1403b450 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7fca10006fd0 tx=0x7fca10006e40 comp rx=0 tx=0).stop 2024-04-25T00:23:45.688 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:45.097+0000 7fca2ad12700 1 -- 172.21.15.32:0/2865575464 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fca24107bf0 msgr2=0x7fca2419bb10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:45.688 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:45.097+0000 7fca2ad12700 1 --2- 172.21.15.32:0/2865575464 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fca24107bf0 0x7fca2419bb10 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7fca180117e0 tx=0x7fca18011c10 comp rx=0 tx=0).stop 2024-04-25T00:23:45.689 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:45.097+0000 7fca2ad12700 1 -- 172.21.15.32:0/2865575464 shutdown_connections 2024-04-25T00:23:45.689 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:45.097+0000 7fca2ad12700 1 --2- 172.21.15.32:0/2865575464 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fca14038f90 0x7fca1403b450 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:45.689 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:45.097+0000 7fca2ad12700 1 --2- 172.21.15.32:0/2865575464 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fca24107bf0 0x7fca2419bb10 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:45.689 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:45.097+0000 7fca2ad12700 1 -- 172.21.15.32:0/2865575464 >> 172.21.15.32:0/2865575464 conn(0x7fca24103170 msgr2=0x7fca24103c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:45.689 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:45.098+0000 7fca2ad12700 1 -- 172.21.15.32:0/2865575464 shutdown_connections 2024-04-25T00:23:45.689 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:45.098+0000 7fca2ad12700 1 -- 172.21.15.32:0/2865575464 wait complete. 2024-04-25T00:23:45.689 INFO:teuthology.orchestra.run.smithi032.stdout:Creating initial admin user... 2024-04-25T00:23:45.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:45 smithi032 ceph-mon[29357]: mgrmap e12: x(active, since 2s) 2024-04-25T00:23:45.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:45 smithi032 ceph-mon[29357]: from='client.14160 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:23:45.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:23:45.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:23:45.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:23:47.140 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$nUisqbSDa7TpdV68lGig5.4g9PZ5x4CwMSyqUf0irHw4e1daWTMD6", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1714004626, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-25T00:23:47.141 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.120+0000 7f7d16083700 1 Processor -- start 2024-04-25T00:23:47.141 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.120+0000 7f7d16083700 1 -- start start 2024-04-25T00:23:47.141 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.120+0000 7f7d16083700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7d101086c0 0x7f7d1010aab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:47.141 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.120+0000 7f7d16083700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f7d1007be30 con 0x7f7d101086c0 2024-04-25T00:23:47.141 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.120+0000 7f7d0f7fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7d101086c0 0x7f7d1010aab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:47.141 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.120+0000 7f7d0f7fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7d101086c0 0x7f7d1010aab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:37202/0 (socket says 172.21.15.32:37202) 2024-04-25T00:23:47.141 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.120+0000 7f7d0f7fe700 1 -- 172.21.15.32:0/213395170 learned_addr learned my addr 172.21.15.32:0/213395170 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:47.141 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.121+0000 7f7d0f7fe700 1 -- 172.21.15.32:0/213395170 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7d1010aff0 con 0x7f7d101086c0 2024-04-25T00:23:47.141 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.121+0000 7f7d0f7fe700 1 --2- 172.21.15.32:0/213395170 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7d101086c0 0x7f7d1010aab0 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f7d000098a0 tx=0x7f7d00009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f6101e23e018f0f1 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:47.141 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.121+0000 7f7d0e7fc700 1 -- 172.21.15.32:0/213395170 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7d00004160 con 0x7f7d101086c0 2024-04-25T00:23:47.141 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.121+0000 7f7d0e7fc700 1 -- 172.21.15.32:0/213395170 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7d00004300 con 0x7f7d101086c0 2024-04-25T00:23:47.141 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.121+0000 7f7d0e7fc700 1 -- 172.21.15.32:0/213395170 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7d00011b20 con 0x7f7d101086c0 2024-04-25T00:23:47.142 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.122+0000 7f7d16083700 1 -- 172.21.15.32:0/213395170 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7d101086c0 msgr2=0x7f7d1010aab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:47.142 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.122+0000 7f7d16083700 1 --2- 172.21.15.32:0/213395170 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7d101086c0 0x7f7d1010aab0 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f7d000098a0 tx=0x7f7d00009bb0 comp rx=0 tx=0).stop 2024-04-25T00:23:47.142 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.122+0000 7f7d16083700 1 -- 172.21.15.32:0/213395170 shutdown_connections 2024-04-25T00:23:47.142 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.122+0000 7f7d16083700 1 --2- 172.21.15.32:0/213395170 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7d101086c0 0x7f7d1010aab0 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:47.142 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.122+0000 7f7d16083700 1 -- 172.21.15.32:0/213395170 >> 172.21.15.32:0/213395170 conn(0x7f7d10102070 msgr2=0x7f7d10104490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:47.142 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.122+0000 7f7d16083700 1 -- 172.21.15.32:0/213395170 shutdown_connections 2024-04-25T00:23:47.142 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.122+0000 7f7d16083700 1 -- 172.21.15.32:0/213395170 wait complete. 2024-04-25T00:23:47.142 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.122+0000 7f7d16083700 1 Processor -- start 2024-04-25T00:23:47.142 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.123+0000 7f7d16083700 1 -- start start 2024-04-25T00:23:47.142 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.123+0000 7f7d16083700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7d101086c0 0x7f7d1019bae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:47.142 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.123+0000 7f7d16083700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f7d1010bff0 con 0x7f7d101086c0 2024-04-25T00:23:47.142 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.123+0000 7f7d0f7fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7d101086c0 0x7f7d1019bae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:47.142 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.123+0000 7f7d0f7fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7d101086c0 0x7f7d1019bae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:37206/0 (socket says 172.21.15.32:37206) 2024-04-25T00:23:47.142 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.123+0000 7f7d0f7fe700 1 -- 172.21.15.32:0/869059325 learned_addr learned my addr 172.21.15.32:0/869059325 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:47.143 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.123+0000 7f7d0f7fe700 1 -- 172.21.15.32:0/869059325 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7d1019c020 con 0x7f7d101086c0 2024-04-25T00:23:47.143 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.123+0000 7f7d0f7fe700 1 --2- 172.21.15.32:0/869059325 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7d101086c0 0x7f7d1019bae0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f7d000117e0 tx=0x7f7d00011c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:47.143 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.124+0000 7f7d0cff9700 1 -- 172.21.15.32:0/869059325 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7d00003bf0 con 0x7f7d101086c0 2024-04-25T00:23:47.143 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.124+0000 7f7d0cff9700 1 -- 172.21.15.32:0/869059325 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7d0001a910 con 0x7f7d101086c0 2024-04-25T00:23:47.143 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.124+0000 7f7d16083700 1 -- 172.21.15.32:0/869059325 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7d1019c2b0 con 0x7f7d101086c0 2024-04-25T00:23:47.143 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.124+0000 7f7d0cff9700 1 -- 172.21.15.32:0/869059325 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7d00024970 con 0x7f7d101086c0 2024-04-25T00:23:47.143 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.124+0000 7f7d16083700 1 -- 172.21.15.32:0/869059325 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7d1019c710 con 0x7f7d101086c0 2024-04-25T00:23:47.143 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.125+0000 7f7d0cff9700 1 -- 172.21.15.32:0/869059325 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 12) ==== 46300+0+0 (secure 0 0 0) 0x7f7d00022020 con 0x7f7d101086c0 2024-04-25T00:23:47.143 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.125+0000 7f7d16083700 1 -- 172.21.15.32:0/869059325 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7d10063890 con 0x7f7d101086c0 2024-04-25T00:23:47.143 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.125+0000 7f7d0cff9700 1 --2- 172.21.15.32:0/869059325 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f7cf80390b0 0x7f7cf803b570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:47.143 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.125+0000 7f7d0cff9700 1 -- 172.21.15.32:0/869059325 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7f7d0004d4a0 con 0x7f7d101086c0 2024-04-25T00:23:47.143 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.125+0000 7f7d0effd700 1 --2- 172.21.15.32:0/869059325 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f7cf80390b0 0x7f7cf803b570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:47.143 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.126+0000 7f7d0effd700 1 --2- 172.21.15.32:0/869059325 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f7cf80390b0 0x7f7cf803b570 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f7d04006fd0 tx=0x7f7d04006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:47.143 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.130+0000 7f7d0cff9700 1 -- 172.21.15.32:0/869059325 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f7d00011e20 con 0x7f7d101086c0 2024-04-25T00:23:47.144 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.367+0000 7f7d16083700 1 -- 172.21.15.32:0/869059325 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}) -- 0x7f7d101064b0 con 0x7f7cf80390b0 2024-04-25T00:23:47.144 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.614+0000 7f7d0cff9700 1 -- 172.21.15.32:0/869059325 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+252 (secure 0 0 0) 0x7f7d101064b0 con 0x7f7cf80390b0 2024-04-25T00:23:47.144 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.616+0000 7f7d16083700 1 -- 172.21.15.32:0/869059325 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f7cf80390b0 msgr2=0x7f7cf803b570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:47.144 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.616+0000 7f7d16083700 1 --2- 172.21.15.32:0/869059325 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f7cf80390b0 0x7f7cf803b570 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f7d04006fd0 tx=0x7f7d04006e40 comp rx=0 tx=0).stop 2024-04-25T00:23:47.144 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.616+0000 7f7d16083700 1 -- 172.21.15.32:0/869059325 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7d101086c0 msgr2=0x7f7d1019bae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:47.144 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.616+0000 7f7d16083700 1 --2- 172.21.15.32:0/869059325 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7d101086c0 0x7f7d1019bae0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f7d000117e0 tx=0x7f7d00011c10 comp rx=0 tx=0).stop 2024-04-25T00:23:47.144 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.616+0000 7f7d16083700 1 -- 172.21.15.32:0/869059325 shutdown_connections 2024-04-25T00:23:47.144 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.616+0000 7f7d16083700 1 --2- 172.21.15.32:0/869059325 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f7cf80390b0 0x7f7cf803b570 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:47.144 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.616+0000 7f7d16083700 1 --2- 172.21.15.32:0/869059325 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7d101086c0 0x7f7d1019bae0 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:47.144 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.616+0000 7f7d16083700 1 -- 172.21.15.32:0/869059325 >> 172.21.15.32:0/869059325 conn(0x7f7d10102070 msgr2=0x7f7d10102aa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:47.144 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.616+0000 7f7d16083700 1 -- 172.21.15.32:0/869059325 shutdown_connections 2024-04-25T00:23:47.144 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:46.616+0000 7f7d16083700 1 -- 172.21.15.32:0/869059325 wait complete. 2024-04-25T00:23:47.144 INFO:teuthology.orchestra.run.smithi032.stdout:Fetching dashboard port number... 2024-04-25T00:23:47.512 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:47 smithi032 ceph-mon[29357]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:23:47.512 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:47 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:23:47.512 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:47 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:23:48.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:48 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/4252526755' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-25T00:23:49.220 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 8443 2024-04-25T00:23:49.221 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.613+0000 7f0f6b11f700 1 Processor -- start 2024-04-25T00:23:49.221 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.613+0000 7f0f6b11f700 1 -- start start 2024-04-25T00:23:49.221 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.613+0000 7f0f6b11f700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0f6407de90 0x7f0f6407c340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:49.221 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.613+0000 7f0f6b11f700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f0f6407c910 con 0x7f0f6407de90 2024-04-25T00:23:49.221 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.614+0000 7f0f68ebb700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0f6407de90 0x7f0f6407c340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:49.221 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.614+0000 7f0f68ebb700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0f6407de90 0x7f0f6407c340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:37222/0 (socket says 172.21.15.32:37222) 2024-04-25T00:23:49.221 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.614+0000 7f0f68ebb700 1 -- 172.21.15.32:0/3484459209 learned_addr learned my addr 172.21.15.32:0/3484459209 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:49.222 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.614+0000 7f0f68ebb700 1 -- 172.21.15.32:0/3484459209 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0f6407ca90 con 0x7f0f6407de90 2024-04-25T00:23:49.222 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.614+0000 7f0f68ebb700 1 --2- 172.21.15.32:0/3484459209 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0f6407de90 0x7f0f6407c340 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f0f540098a0 tx=0x7f0f54009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5389e29a34d1ec73 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:49.222 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.614+0000 7f0f637fe700 1 -- 172.21.15.32:0/3484459209 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0f54004160 con 0x7f0f6407de90 2024-04-25T00:23:49.222 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.614+0000 7f0f637fe700 1 -- 172.21.15.32:0/3484459209 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0f54004300 con 0x7f0f6407de90 2024-04-25T00:23:49.222 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.614+0000 7f0f637fe700 1 -- 172.21.15.32:0/3484459209 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0f54011b20 con 0x7f0f6407de90 2024-04-25T00:23:49.222 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.615+0000 7f0f6b11f700 1 -- 172.21.15.32:0/3484459209 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0f6407de90 msgr2=0x7f0f6407c340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:49.222 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.615+0000 7f0f6b11f700 1 --2- 172.21.15.32:0/3484459209 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0f6407de90 0x7f0f6407c340 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f0f540098a0 tx=0x7f0f54009bb0 comp rx=0 tx=0).stop 2024-04-25T00:23:49.222 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.615+0000 7f0f6b11f700 1 -- 172.21.15.32:0/3484459209 shutdown_connections 2024-04-25T00:23:49.222 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.615+0000 7f0f6b11f700 1 --2- 172.21.15.32:0/3484459209 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0f6407de90 0x7f0f6407c340 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:49.222 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.615+0000 7f0f6b11f700 1 -- 172.21.15.32:0/3484459209 >> 172.21.15.32:0/3484459209 conn(0x7f0f64103150 msgr2=0x7f0f641055b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:49.222 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.615+0000 7f0f6b11f700 1 -- 172.21.15.32:0/3484459209 shutdown_connections 2024-04-25T00:23:49.222 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.615+0000 7f0f6b11f700 1 -- 172.21.15.32:0/3484459209 wait complete. 2024-04-25T00:23:49.222 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.615+0000 7f0f6b11f700 1 Processor -- start 2024-04-25T00:23:49.222 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.615+0000 7f0f6b11f700 1 -- start start 2024-04-25T00:23:49.223 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.616+0000 7f0f6b11f700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0f6407de90 0x7f0f6419dca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:49.223 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.616+0000 7f0f6b11f700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f0f6410e1b0 con 0x7f0f6407de90 2024-04-25T00:23:49.223 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.616+0000 7f0f68ebb700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0f6407de90 0x7f0f6419dca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:49.223 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.616+0000 7f0f68ebb700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0f6407de90 0x7f0f6419dca0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:37236/0 (socket says 172.21.15.32:37236) 2024-04-25T00:23:49.223 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.616+0000 7f0f68ebb700 1 -- 172.21.15.32:0/4252526755 learned_addr learned my addr 172.21.15.32:0/4252526755 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:49.223 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.616+0000 7f0f68ebb700 1 -- 172.21.15.32:0/4252526755 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0f6419e1e0 con 0x7f0f6407de90 2024-04-25T00:23:49.223 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.616+0000 7f0f68ebb700 1 --2- 172.21.15.32:0/4252526755 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0f6407de90 0x7f0f6419dca0 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f0f540117e0 tx=0x7f0f54011c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:49.223 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.616+0000 7f0f61ffb700 1 -- 172.21.15.32:0/4252526755 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0f54003bf0 con 0x7f0f6407de90 2024-04-25T00:23:49.223 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.616+0000 7f0f6b11f700 1 -- 172.21.15.32:0/4252526755 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0f6419e470 con 0x7f0f6407de90 2024-04-25T00:23:49.223 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.616+0000 7f0f61ffb700 1 -- 172.21.15.32:0/4252526755 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0f5401a910 con 0x7f0f6407de90 2024-04-25T00:23:49.223 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.616+0000 7f0f61ffb700 1 -- 172.21.15.32:0/4252526755 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0f54024970 con 0x7f0f6407de90 2024-04-25T00:23:49.223 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.617+0000 7f0f6b11f700 1 -- 172.21.15.32:0/4252526755 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0f6419e8d0 con 0x7f0f6407de90 2024-04-25T00:23:49.223 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.617+0000 7f0f61ffb700 1 -- 172.21.15.32:0/4252526755 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 12) ==== 46300+0+0 (secure 0 0 0) 0x7f0f54022020 con 0x7f0f6407de90 2024-04-25T00:23:49.223 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.617+0000 7f0f61ffb700 1 --2- 172.21.15.32:0/4252526755 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f0f4c039050 0x7f0f4c03b510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:49.223 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.618+0000 7f0f61ffb700 1 -- 172.21.15.32:0/4252526755 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7f0f5404d670 con 0x7f0f6407de90 2024-04-25T00:23:49.224 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.618+0000 7f0f6b11f700 1 -- 172.21.15.32:0/4252526755 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0f64063890 con 0x7f0f6407de90 2024-04-25T00:23:49.224 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.618+0000 7f0f63fff700 1 --2- 172.21.15.32:0/4252526755 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f0f4c039050 0x7f0f4c03b510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:49.224 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.618+0000 7f0f63fff700 1 --2- 172.21.15.32:0/4252526755 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f0f4c039050 0x7f0f4c03b510 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f0f58006fd0 tx=0x7f0f58006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:49.224 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.623+0000 7f0f61ffb700 1 -- 172.21.15.32:0/4252526755 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f0f54011e20 con 0x7f0f6407de90 2024-04-25T00:23:49.224 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.844+0000 7f0f6b11f700 1 -- 172.21.15.32:0/4252526755 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"} v 0) -- 0x7f0f641a1650 con 0x7f0f6407de90 2024-04-25T00:23:49.224 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.844+0000 7f0f61ffb700 1 -- 172.21.15.32:0/4252526755 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]=0 v8) ==== 112+0+5 (secure 0 0 0) 0x7f0f54032020 con 0x7f0f6407de90 2024-04-25T00:23:49.224 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.846+0000 7f0f6b11f700 1 -- 172.21.15.32:0/4252526755 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f0f4c039050 msgr2=0x7f0f4c03b510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:49.224 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.846+0000 7f0f6b11f700 1 --2- 172.21.15.32:0/4252526755 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f0f4c039050 0x7f0f4c03b510 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f0f58006fd0 tx=0x7f0f58006e40 comp rx=0 tx=0).stop 2024-04-25T00:23:49.224 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.846+0000 7f0f6b11f700 1 -- 172.21.15.32:0/4252526755 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0f6407de90 msgr2=0x7f0f6419dca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:49.224 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.846+0000 7f0f6b11f700 1 --2- 172.21.15.32:0/4252526755 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0f6407de90 0x7f0f6419dca0 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f0f540117e0 tx=0x7f0f54011c10 comp rx=0 tx=0).stop 2024-04-25T00:23:49.224 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.846+0000 7f0f6b11f700 1 -- 172.21.15.32:0/4252526755 shutdown_connections 2024-04-25T00:23:49.224 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.846+0000 7f0f6b11f700 1 --2- 172.21.15.32:0/4252526755 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f0f4c039050 0x7f0f4c03b510 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:49.224 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.846+0000 7f0f6b11f700 1 --2- 172.21.15.32:0/4252526755 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0f6407de90 0x7f0f6419dca0 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:49.224 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.846+0000 7f0f6b11f700 1 -- 172.21.15.32:0/4252526755 >> 172.21.15.32:0/4252526755 conn(0x7f0f64103150 msgr2=0x7f0f64103c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:49.225 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.846+0000 7f0f6b11f700 1 -- 172.21.15.32:0/4252526755 shutdown_connections 2024-04-25T00:23:49.225 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:47.846+0000 7f0f6b11f700 1 -- 172.21.15.32:0/4252526755 wait complete. 2024-04-25T00:23:49.229 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-25T00:23:49.229 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout disabled 2024-04-25T00:23:49.238 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-25T00:23:49.238 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout inactive 2024-04-25T00:23:49.238 INFO:teuthology.orchestra.run.smithi032.stdout:firewalld.service is not enabled 2024-04-25T00:23:49.238 INFO:teuthology.orchestra.run.smithi032.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-25T00:23:49.240 INFO:teuthology.orchestra.run.smithi032.stdout:Ceph Dashboard is now available at: 2024-04-25T00:23:49.240 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:23:49.240 INFO:teuthology.orchestra.run.smithi032.stdout: URL: https://smithi032:8443/ 2024-04-25T00:23:49.240 INFO:teuthology.orchestra.run.smithi032.stdout: User: admin 2024-04-25T00:23:49.240 INFO:teuthology.orchestra.run.smithi032.stdout: Password: bf3qior72v 2024-04-25T00:23:49.240 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:23:49.241 INFO:teuthology.orchestra.run.smithi032.stdout:Saving cluster configuration to /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config directory 2024-04-25T00:23:49.241 INFO:teuthology.orchestra.run.smithi032.stdout:Enabling autotune for osd_memory_target 2024-04-25T00:23:50.492 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:50 smithi032 ceph-mon[29357]: mgrmap e13: x(active, since 6s) 2024-04-25T00:23:50.501 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.694+0000 7ff8a8e11700 1 Processor -- start 2024-04-25T00:23:50.501 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.695+0000 7ff8a8e11700 1 -- start start 2024-04-25T00:23:50.501 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.695+0000 7ff8a8e11700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff8a407de60 0x7ff8a407c360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:50.501 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.695+0000 7ff8a8e11700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7ff8a407c8a0 con 0x7ff8a407de60 2024-04-25T00:23:50.501 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.695+0000 7ff8a259c700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff8a407de60 0x7ff8a407c360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:50.501 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.695+0000 7ff8a259c700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff8a407de60 0x7ff8a407c360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:37242/0 (socket says 172.21.15.32:37242) 2024-04-25T00:23:50.501 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.695+0000 7ff8a259c700 1 -- 172.21.15.32:0/526328732 learned_addr learned my addr 172.21.15.32:0/526328732 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:50.501 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.695+0000 7ff8a259c700 1 -- 172.21.15.32:0/526328732 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff8a407ca20 con 0x7ff8a407de60 2024-04-25T00:23:50.501 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.696+0000 7ff8a259c700 1 --2- 172.21.15.32:0/526328732 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff8a407de60 0x7ff8a407c360 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7ff8940098a0 tx=0x7ff894009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c7d646a6ea1b3ebf server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:50.501 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.696+0000 7ff8a159a700 1 -- 172.21.15.32:0/526328732 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7ff894004160 con 0x7ff8a407de60 2024-04-25T00:23:50.501 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.696+0000 7ff8a159a700 1 -- 172.21.15.32:0/526328732 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff894004300 con 0x7ff8a407de60 2024-04-25T00:23:50.501 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.696+0000 7ff8a159a700 1 -- 172.21.15.32:0/526328732 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7ff894011b20 con 0x7ff8a407de60 2024-04-25T00:23:50.502 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.696+0000 7ff8a8e11700 1 -- 172.21.15.32:0/526328732 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff8a407de60 msgr2=0x7ff8a407c360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:50.502 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.696+0000 7ff8a8e11700 1 --2- 172.21.15.32:0/526328732 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff8a407de60 0x7ff8a407c360 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7ff8940098a0 tx=0x7ff894009bb0 comp rx=0 tx=0).stop 2024-04-25T00:23:50.502 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.697+0000 7ff8a8e11700 1 -- 172.21.15.32:0/526328732 shutdown_connections 2024-04-25T00:23:50.502 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.697+0000 7ff8a8e11700 1 --2- 172.21.15.32:0/526328732 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff8a407de60 0x7ff8a407c360 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:50.502 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.697+0000 7ff8a8e11700 1 -- 172.21.15.32:0/526328732 >> 172.21.15.32:0/526328732 conn(0x7ff8a4103230 msgr2=0x7ff8a4105650 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:50.502 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.697+0000 7ff8a8e11700 1 -- 172.21.15.32:0/526328732 shutdown_connections 2024-04-25T00:23:50.502 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.697+0000 7ff8a8e11700 1 -- 172.21.15.32:0/526328732 wait complete. 2024-04-25T00:23:50.502 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.697+0000 7ff8a8e11700 1 Processor -- start 2024-04-25T00:23:50.502 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.697+0000 7ff8a8e11700 1 -- start start 2024-04-25T00:23:50.502 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.698+0000 7ff8a8e11700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff8a407de60 0x7ff8a419dd00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:50.502 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.698+0000 7ff8a8e11700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7ff8a410e1b0 con 0x7ff8a407de60 2024-04-25T00:23:50.502 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.698+0000 7ff8a259c700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff8a407de60 0x7ff8a419dd00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:50.502 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.698+0000 7ff8a259c700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff8a407de60 0x7ff8a419dd00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:37256/0 (socket says 172.21.15.32:37256) 2024-04-25T00:23:50.503 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.698+0000 7ff8a259c700 1 -- 172.21.15.32:0/1266438458 learned_addr learned my addr 172.21.15.32:0/1266438458 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:50.503 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.698+0000 7ff8a259c700 1 -- 172.21.15.32:0/1266438458 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff8a419e240 con 0x7ff8a407de60 2024-04-25T00:23:50.503 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.698+0000 7ff8a259c700 1 --2- 172.21.15.32:0/1266438458 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff8a407de60 0x7ff8a419dd00 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7ff894011b60 tx=0x7ff894011c40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:50.503 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.698+0000 7ff89b7fe700 1 -- 172.21.15.32:0/1266438458 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7ff894003b60 con 0x7ff8a407de60 2024-04-25T00:23:50.503 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.698+0000 7ff89b7fe700 1 -- 172.21.15.32:0/1266438458 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff89401a910 con 0x7ff8a407de60 2024-04-25T00:23:50.503 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.698+0000 7ff8a8e11700 1 -- 172.21.15.32:0/1266438458 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff8a419e4d0 con 0x7ff8a407de60 2024-04-25T00:23:50.503 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.698+0000 7ff89b7fe700 1 -- 172.21.15.32:0/1266438458 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7ff894024930 con 0x7ff8a407de60 2024-04-25T00:23:50.503 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.699+0000 7ff8a8e11700 1 -- 172.21.15.32:0/1266438458 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff8a419e930 con 0x7ff8a407de60 2024-04-25T00:23:50.503 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.699+0000 7ff89b7fe700 1 -- 172.21.15.32:0/1266438458 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 13) ==== 46343+0+0 (secure 0 0 0) 0x7ff894022020 con 0x7ff8a407de60 2024-04-25T00:23:50.503 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.699+0000 7ff8a8e11700 1 -- 172.21.15.32:0/1266438458 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff8a4063890 con 0x7ff8a407de60 2024-04-25T00:23:50.503 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.700+0000 7ff89b7fe700 1 --2- 172.21.15.32:0/1266438458 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff890039140 0x7ff89003b600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:50.503 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.700+0000 7ff89b7fe700 1 -- 172.21.15.32:0/1266438458 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7ff89404d420 con 0x7ff8a407de60 2024-04-25T00:23:50.503 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.700+0000 7ff8a1d9b700 1 --2- 172.21.15.32:0/1266438458 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff890039140 0x7ff89003b600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:50.503 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.700+0000 7ff8a1d9b700 1 --2- 172.21.15.32:0/1266438458 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff890039140 0x7ff89003b600 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7ff88c006fd0 tx=0x7ff88c006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:50.504 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.705+0000 7ff89b7fe700 1 -- 172.21.15.32:0/1266438458 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7ff894019c70 con 0x7ff8a407de60 2024-04-25T00:23:50.504 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.914+0000 7ff8a8e11700 1 -- 172.21.15.32:0/1266438458 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command([{prefix=config set, name=osd_memory_target_autotune}] v 0) -- 0x7ff8a41a1390 con 0x7ff8a407de60 2024-04-25T00:23:50.504 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.914+0000 7ff89b7fe700 1 -- 172.21.15.32:0/1266438458 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{prefix=config set, name=osd_memory_target_autotune}]=0 v8)=0 v8) ==== 127+0+0 (secure 0 0 0) 0x7ff894016df0 con 0x7ff8a407de60 2024-04-25T00:23:50.504 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.916+0000 7ff8a8e11700 1 -- 172.21.15.32:0/1266438458 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff890039140 msgr2=0x7ff89003b600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:50.504 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.916+0000 7ff8a8e11700 1 --2- 172.21.15.32:0/1266438458 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff890039140 0x7ff89003b600 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7ff88c006fd0 tx=0x7ff88c006e40 comp rx=0 tx=0).stop 2024-04-25T00:23:50.504 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.916+0000 7ff8a8e11700 1 -- 172.21.15.32:0/1266438458 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff8a407de60 msgr2=0x7ff8a419dd00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:50.504 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.916+0000 7ff8a8e11700 1 --2- 172.21.15.32:0/1266438458 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff8a407de60 0x7ff8a419dd00 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7ff894011b60 tx=0x7ff894011c40 comp rx=0 tx=0).stop 2024-04-25T00:23:50.504 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.916+0000 7ff8a8e11700 1 -- 172.21.15.32:0/1266438458 shutdown_connections 2024-04-25T00:23:50.504 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.916+0000 7ff8a8e11700 1 --2- 172.21.15.32:0/1266438458 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff890039140 0x7ff89003b600 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:50.504 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.916+0000 7ff8a8e11700 1 --2- 172.21.15.32:0/1266438458 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff8a407de60 0x7ff8a419dd00 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:50.504 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.916+0000 7ff8a8e11700 1 -- 172.21.15.32:0/1266438458 >> 172.21.15.32:0/1266438458 conn(0x7ff8a4103230 msgr2=0x7ff8a4103cd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:50.504 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.916+0000 7ff8a8e11700 1 -- 172.21.15.32:0/1266438458 shutdown_connections 2024-04-25T00:23:50.504 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:49.916+0000 7ff8a8e11700 1 -- 172.21.15.32:0/1266438458 wait complete. 2024-04-25T00:23:51.868 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.959+0000 7f601f4c8700 1 Processor -- start 2024-04-25T00:23:51.868 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.959+0000 7f601f4c8700 1 -- start start 2024-04-25T00:23:51.868 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.959+0000 7f601f4c8700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6018107bf0 0x7f6018108010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:51.868 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.959+0000 7f601f4c8700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f60181085e0 con 0x7f6018107bf0 2024-04-25T00:23:51.869 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.959+0000 7f601d264700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6018107bf0 0x7f6018108010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:51.869 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.960+0000 7f601d264700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6018107bf0 0x7f6018108010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:37262/0 (socket says 172.21.15.32:37262) 2024-04-25T00:23:51.869 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.960+0000 7f601d264700 1 -- 172.21.15.32:0/516807753 learned_addr learned my addr 172.21.15.32:0/516807753 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:51.869 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.960+0000 7f601d264700 1 -- 172.21.15.32:0/516807753 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6018108da0 con 0x7f6018107bf0 2024-04-25T00:23:51.869 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.960+0000 7f601d264700 1 --2- 172.21.15.32:0/516807753 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6018107bf0 0x7f6018108010 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f60140098a0 tx=0x7f6014009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c82b6d62958a63c9 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:51.869 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.960+0000 7f600ffff700 1 -- 172.21.15.32:0/516807753 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f6014004160 con 0x7f6018107bf0 2024-04-25T00:23:51.869 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.960+0000 7f600ffff700 1 -- 172.21.15.32:0/516807753 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6014004300 con 0x7f6018107bf0 2024-04-25T00:23:51.869 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.960+0000 7f600ffff700 1 -- 172.21.15.32:0/516807753 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f6014011b20 con 0x7f6018107bf0 2024-04-25T00:23:51.869 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.961+0000 7f601f4c8700 1 -- 172.21.15.32:0/516807753 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6018107bf0 msgr2=0x7f6018108010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:51.869 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.961+0000 7f601f4c8700 1 --2- 172.21.15.32:0/516807753 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6018107bf0 0x7f6018108010 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f60140098a0 tx=0x7f6014009bb0 comp rx=0 tx=0).stop 2024-04-25T00:23:51.869 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.961+0000 7f601f4c8700 1 -- 172.21.15.32:0/516807753 shutdown_connections 2024-04-25T00:23:51.869 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.961+0000 7f601f4c8700 1 --2- 172.21.15.32:0/516807753 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6018107bf0 0x7f6018108010 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:51.869 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.961+0000 7f601f4c8700 1 -- 172.21.15.32:0/516807753 >> 172.21.15.32:0/516807753 conn(0x7f6018103170 msgr2=0x7f60181055d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:51.869 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.961+0000 7f601f4c8700 1 -- 172.21.15.32:0/516807753 shutdown_connections 2024-04-25T00:23:51.870 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.961+0000 7f601f4c8700 1 -- 172.21.15.32:0/516807753 wait complete. 2024-04-25T00:23:51.870 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.961+0000 7f601f4c8700 1 Processor -- start 2024-04-25T00:23:51.870 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.962+0000 7f601f4c8700 1 -- start start 2024-04-25T00:23:51.870 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.962+0000 7f601f4c8700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6018107bf0 0x7f601819bb00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:51.870 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.962+0000 7f601f4c8700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f601810c150 con 0x7f6018107bf0 2024-04-25T00:23:51.870 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.962+0000 7f601d264700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6018107bf0 0x7f601819bb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:51.870 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.962+0000 7f601d264700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6018107bf0 0x7f601819bb00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:37268/0 (socket says 172.21.15.32:37268) 2024-04-25T00:23:51.870 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.962+0000 7f601d264700 1 -- 172.21.15.32:0/445056348 learned_addr learned my addr 172.21.15.32:0/445056348 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:51.870 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.962+0000 7f601d264700 1 -- 172.21.15.32:0/445056348 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f601819c040 con 0x7f6018107bf0 2024-04-25T00:23:51.870 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.962+0000 7f601d264700 1 --2- 172.21.15.32:0/445056348 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6018107bf0 0x7f601819bb00 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f60140117e0 tx=0x7f6014011ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:51.870 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.962+0000 7f600e7fc700 1 -- 172.21.15.32:0/445056348 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f6014003bf0 con 0x7f6018107bf0 2024-04-25T00:23:51.870 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.962+0000 7f600e7fc700 1 -- 172.21.15.32:0/445056348 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f601401a910 con 0x7f6018107bf0 2024-04-25T00:23:51.870 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.962+0000 7f601f4c8700 1 -- 172.21.15.32:0/445056348 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f601819c2d0 con 0x7f6018107bf0 2024-04-25T00:23:51.871 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.962+0000 7f600e7fc700 1 -- 172.21.15.32:0/445056348 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f6014024970 con 0x7f6018107bf0 2024-04-25T00:23:51.871 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.963+0000 7f601f4c8700 1 -- 172.21.15.32:0/445056348 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f601819c730 con 0x7f6018107bf0 2024-04-25T00:23:51.871 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.963+0000 7f600e7fc700 1 -- 172.21.15.32:0/445056348 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 13) ==== 46343+0+0 (secure 0 0 0) 0x7f6014022020 con 0x7f6018107bf0 2024-04-25T00:23:51.871 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.963+0000 7f601f4c8700 1 -- 172.21.15.32:0/445056348 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6018063890 con 0x7f6018107bf0 2024-04-25T00:23:51.871 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.964+0000 7f600e7fc700 1 --2- 172.21.15.32:0/445056348 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f60040391a0 0x7f600403b660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:51.871 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.964+0000 7f600e7fc700 1 -- 172.21.15.32:0/445056348 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7f601404d5f0 con 0x7f6018107bf0 2024-04-25T00:23:51.871 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.964+0000 7f601ca63700 1 --2- 172.21.15.32:0/445056348 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f60040391a0 0x7f600403b660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:51.871 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.964+0000 7f601ca63700 1 --2- 172.21.15.32:0/445056348 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f60040391a0 0x7f600403b660 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f6008006fd0 tx=0x7f6008006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:51.871 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:50.969+0000 7f600e7fc700 1 -- 172.21.15.32:0/445056348 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f601402d910 con 0x7f6018107bf0 2024-04-25T00:23:51.871 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:51.285+0000 7f601f4c8700 1 -- 172.21.15.32:0/445056348 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command([{prefix=config-key set, key=mgr/dashboard/cluster/status}] v 0) -- 0x7f601819f1c0 con 0x7f6018107bf0 2024-04-25T00:23:51.871 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:51.288+0000 7f600e7fc700 1 -- 172.21.15.32:0/445056348 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{prefix=config-key set, key=mgr/dashboard/cluster/status}]=0 set mgr/dashboard/cluster/status v28)=0 set mgr/dashboard/cluster/status v28) ==== 153+0+0 (secure 0 0 0) 0x7f601402d3f0 con 0x7f6018107bf0 2024-04-25T00:23:51.871 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:51.290+0000 7f601f4c8700 1 -- 172.21.15.32:0/445056348 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f60040391a0 msgr2=0x7f600403b660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:51.871 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:51.290+0000 7f601f4c8700 1 --2- 172.21.15.32:0/445056348 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f60040391a0 0x7f600403b660 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f6008006fd0 tx=0x7f6008006e40 comp rx=0 tx=0).stop 2024-04-25T00:23:51.871 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:51.290+0000 7f601f4c8700 1 -- 172.21.15.32:0/445056348 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6018107bf0 msgr2=0x7f601819bb00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:51.872 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:51.290+0000 7f601f4c8700 1 --2- 172.21.15.32:0/445056348 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6018107bf0 0x7f601819bb00 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f60140117e0 tx=0x7f6014011ba0 comp rx=0 tx=0).stop 2024-04-25T00:23:51.872 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:51.290+0000 7f601f4c8700 1 -- 172.21.15.32:0/445056348 shutdown_connections 2024-04-25T00:23:51.872 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:51.290+0000 7f601f4c8700 1 --2- 172.21.15.32:0/445056348 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f60040391a0 0x7f600403b660 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:51.872 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:51.290+0000 7f601f4c8700 1 --2- 172.21.15.32:0/445056348 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6018107bf0 0x7f601819bb00 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:51.872 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:51.290+0000 7f601f4c8700 1 -- 172.21.15.32:0/445056348 >> 172.21.15.32:0/445056348 conn(0x7f6018103170 msgr2=0x7f6018103c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:51.872 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:51.290+0000 7f601f4c8700 1 -- 172.21.15.32:0/445056348 shutdown_connections 2024-04-25T00:23:51.872 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr 2024-04-25T00:23:51.290+0000 7f601f4c8700 1 -- 172.21.15.32:0/445056348 wait complete. 2024-04-25T00:23:51.872 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-04-25T00:23:51.872 INFO:teuthology.orchestra.run.smithi032.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-04-25T00:23:51.872 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:23:51.872 INFO:teuthology.orchestra.run.smithi032.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-25T00:23:51.872 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:23:51.872 INFO:teuthology.orchestra.run.smithi032.stdout:Or, if you are only running a single cluster on this host: 2024-04-25T00:23:51.872 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:23:51.872 INFO:teuthology.orchestra.run.smithi032.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-04-25T00:23:51.873 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:23:51.873 INFO:teuthology.orchestra.run.smithi032.stdout:Please consider enabling telemetry to help improve Ceph: 2024-04-25T00:23:51.873 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:23:51.873 INFO:teuthology.orchestra.run.smithi032.stdout: ceph telemetry on 2024-04-25T00:23:51.873 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:23:51.873 INFO:teuthology.orchestra.run.smithi032.stdout:For more information see: 2024-04-25T00:23:51.873 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:23:51.873 INFO:teuthology.orchestra.run.smithi032.stdout: https://docs.ceph.com/en/latest/mgr/telemetry/ 2024-04-25T00:23:51.873 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:23:51.873 INFO:teuthology.orchestra.run.smithi032.stdout:Bootstrap complete. 2024-04-25T00:23:51.915 INFO:tasks.cephadm:Fetching config... 2024-04-25T00:23:51.915 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-04-25T00:23:51.916 DEBUG:teuthology.orchestra.run.smithi032:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-25T00:23:51.933 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-25T00:23:51.933 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-04-25T00:23:51.933 DEBUG:teuthology.orchestra.run.smithi032:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-25T00:23:51.990 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-25T00:23:51.990 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-04-25T00:23:51.990 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd if=/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/keyring of=/dev/stdout 2024-04-25T00:23:52.056 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-25T00:23:52.056 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-04-25T00:23:52.056 DEBUG:teuthology.orchestra.run.smithi032:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-25T00:23:52.112 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-25T00:23:52.113 DEBUG:teuthology.orchestra.run.smithi032:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDbpksuQENO7qVQyOf4/NPXGtN/S59yLoR/VLuJhl6gtPW2+mwrirhW3lOvbVCzMhpcBOeyLdEBmEri3c8xP4rb1vp4KXDAOjBcMdHSIxvf677Q1276d/ouFeaCwDvMx1GQ4KjCs6XPjMMWf1FyKp5But6ES6iZ3dmNYpEQCdX7UPhTPFuLZ7HihF88x5vTGM9Dz7W1js3lU+dlHb6gY4LrQPiElUF885/JYfG8MmG/8WULjzmPTsvi5o249ihqbGgjOZyhl5K4uUdA3U+M/LxrnkpCsT2pEF7Iq+3gpfSzkxrU6ZT6yK/gPXY2yMfLcEVintsCrxhT5W3ueFBXUCcaLgAoORDod/GleFgV+K1VZ89z0vCEKw8zucIqhTtWxgsgKaNdu7uesusu5rigVVAE8y9RlLj0WQquE4G/x/Wg+MfRZptvlTbq7xlsqhOZVTpW0ZWewipALhTaq9vT17jdWnhIOzGSDscg0Enz4MiLRcWbr7ICEeUxBzQapchBj0c= ceph-cb997156-0299-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-25T00:23:52.188 INFO:teuthology.orchestra.run.smithi032.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDbpksuQENO7qVQyOf4/NPXGtN/S59yLoR/VLuJhl6gtPW2+mwrirhW3lOvbVCzMhpcBOeyLdEBmEri3c8xP4rb1vp4KXDAOjBcMdHSIxvf677Q1276d/ouFeaCwDvMx1GQ4KjCs6XPjMMWf1FyKp5But6ES6iZ3dmNYpEQCdX7UPhTPFuLZ7HihF88x5vTGM9Dz7W1js3lU+dlHb6gY4LrQPiElUF885/JYfG8MmG/8WULjzmPTsvi5o249ihqbGgjOZyhl5K4uUdA3U+M/LxrnkpCsT2pEF7Iq+3gpfSzkxrU6ZT6yK/gPXY2yMfLcEVintsCrxhT5W3ueFBXUCcaLgAoORDod/GleFgV+K1VZ89z0vCEKw8zucIqhTtWxgsgKaNdu7uesusu5rigVVAE8y9RlLj0WQquE4G/x/Wg+MfRZptvlTbq7xlsqhOZVTpW0ZWewipALhTaq9vT17jdWnhIOzGSDscg0Enz4MiLRcWbr7ICEeUxBzQapchBj0c= ceph-cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:23:52.199 DEBUG:teuthology.orchestra.run.smithi121:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDbpksuQENO7qVQyOf4/NPXGtN/S59yLoR/VLuJhl6gtPW2+mwrirhW3lOvbVCzMhpcBOeyLdEBmEri3c8xP4rb1vp4KXDAOjBcMdHSIxvf677Q1276d/ouFeaCwDvMx1GQ4KjCs6XPjMMWf1FyKp5But6ES6iZ3dmNYpEQCdX7UPhTPFuLZ7HihF88x5vTGM9Dz7W1js3lU+dlHb6gY4LrQPiElUF885/JYfG8MmG/8WULjzmPTsvi5o249ihqbGgjOZyhl5K4uUdA3U+M/LxrnkpCsT2pEF7Iq+3gpfSzkxrU6ZT6yK/gPXY2yMfLcEVintsCrxhT5W3ueFBXUCcaLgAoORDod/GleFgV+K1VZ89z0vCEKw8zucIqhTtWxgsgKaNdu7uesusu5rigVVAE8y9RlLj0WQquE4G/x/Wg+MfRZptvlTbq7xlsqhOZVTpW0ZWewipALhTaq9vT17jdWnhIOzGSDscg0Enz4MiLRcWbr7ICEeUxBzQapchBj0c= ceph-cb997156-0299-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-25T00:23:52.238 INFO:teuthology.orchestra.run.smithi121.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDbpksuQENO7qVQyOf4/NPXGtN/S59yLoR/VLuJhl6gtPW2+mwrirhW3lOvbVCzMhpcBOeyLdEBmEri3c8xP4rb1vp4KXDAOjBcMdHSIxvf677Q1276d/ouFeaCwDvMx1GQ4KjCs6XPjMMWf1FyKp5But6ES6iZ3dmNYpEQCdX7UPhTPFuLZ7HihF88x5vTGM9Dz7W1js3lU+dlHb6gY4LrQPiElUF885/JYfG8MmG/8WULjzmPTsvi5o249ihqbGgjOZyhl5K4uUdA3U+M/LxrnkpCsT2pEF7Iq+3gpfSzkxrU6ZT6yK/gPXY2yMfLcEVintsCrxhT5W3ueFBXUCcaLgAoORDod/GleFgV+K1VZ89z0vCEKw8zucIqhTtWxgsgKaNdu7uesusu5rigVVAE8y9RlLj0WQquE4G/x/Wg+MfRZptvlTbq7xlsqhOZVTpW0ZWewipALhTaq9vT17jdWnhIOzGSDscg0Enz4MiLRcWbr7ICEeUxBzQapchBj0c= ceph-cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:23:52.249 DEBUG:teuthology.orchestra.run.smithi130:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDbpksuQENO7qVQyOf4/NPXGtN/S59yLoR/VLuJhl6gtPW2+mwrirhW3lOvbVCzMhpcBOeyLdEBmEri3c8xP4rb1vp4KXDAOjBcMdHSIxvf677Q1276d/ouFeaCwDvMx1GQ4KjCs6XPjMMWf1FyKp5But6ES6iZ3dmNYpEQCdX7UPhTPFuLZ7HihF88x5vTGM9Dz7W1js3lU+dlHb6gY4LrQPiElUF885/JYfG8MmG/8WULjzmPTsvi5o249ihqbGgjOZyhl5K4uUdA3U+M/LxrnkpCsT2pEF7Iq+3gpfSzkxrU6ZT6yK/gPXY2yMfLcEVintsCrxhT5W3ueFBXUCcaLgAoORDod/GleFgV+K1VZ89z0vCEKw8zucIqhTtWxgsgKaNdu7uesusu5rigVVAE8y9RlLj0WQquE4G/x/Wg+MfRZptvlTbq7xlsqhOZVTpW0ZWewipALhTaq9vT17jdWnhIOzGSDscg0Enz4MiLRcWbr7ICEeUxBzQapchBj0c= ceph-cb997156-0299-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-25T00:23:52.289 INFO:teuthology.orchestra.run.smithi130.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDbpksuQENO7qVQyOf4/NPXGtN/S59yLoR/VLuJhl6gtPW2+mwrirhW3lOvbVCzMhpcBOeyLdEBmEri3c8xP4rb1vp4KXDAOjBcMdHSIxvf677Q1276d/ouFeaCwDvMx1GQ4KjCs6XPjMMWf1FyKp5But6ES6iZ3dmNYpEQCdX7UPhTPFuLZ7HihF88x5vTGM9Dz7W1js3lU+dlHb6gY4LrQPiElUF885/JYfG8MmG/8WULjzmPTsvi5o249ihqbGgjOZyhl5K4uUdA3U+M/LxrnkpCsT2pEF7Iq+3gpfSzkxrU6ZT6yK/gPXY2yMfLcEVintsCrxhT5W3ueFBXUCcaLgAoORDod/GleFgV+K1VZ89z0vCEKw8zucIqhTtWxgsgKaNdu7uesusu5rigVVAE8y9RlLj0WQquE4G/x/Wg+MfRZptvlTbq7xlsqhOZVTpW0ZWewipALhTaq9vT17jdWnhIOzGSDscg0Enz4MiLRcWbr7ICEeUxBzQapchBj0c= ceph-cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:23:52.301 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-25T00:23:52.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:52 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/445056348' entity='client.admin' 2024-04-25T00:23:52.555 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:23:55.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.058+0000 7f3624380700 1 -- 172.21.15.32:0/2224529272 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f361c100ba0 msgr2=0x7f361c100fc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:55.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.058+0000 7f3624380700 1 --2- 172.21.15.32:0/2224529272 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f361c100ba0 0x7f361c100fc0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f3610009910 tx=0x7f3610009c20 comp rx=0 tx=0).stop 2024-04-25T00:23:55.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.058+0000 7f3624380700 1 -- 172.21.15.32:0/2224529272 shutdown_connections 2024-04-25T00:23:55.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.058+0000 7f3624380700 1 --2- 172.21.15.32:0/2224529272 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f361c100ba0 0x7f361c100fc0 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:55.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.058+0000 7f3624380700 1 -- 172.21.15.32:0/2224529272 >> 172.21.15.32:0/2224529272 conn(0x7f361c0fc180 msgr2=0x7f361c0fe5a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:55.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.058+0000 7f3624380700 1 -- 172.21.15.32:0/2224529272 shutdown_connections 2024-04-25T00:23:55.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.058+0000 7f3624380700 1 -- 172.21.15.32:0/2224529272 wait complete. 2024-04-25T00:23:55.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.059+0000 7f3624380700 1 Processor -- start 2024-04-25T00:23:55.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.059+0000 7f3624380700 1 -- start start 2024-04-25T00:23:55.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.059+0000 7f3624380700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f361c100ba0 0x7f361c06ab60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:55.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.059+0000 7f3624380700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f361c106c30 con 0x7f361c100ba0 2024-04-25T00:23:55.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.059+0000 7f362211c700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f361c100ba0 0x7f361c06ab60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:55.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.059+0000 7f362211c700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f361c100ba0 0x7f361c06ab60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:37784/0 (socket says 172.21.15.32:37784) 2024-04-25T00:23:55.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.059+0000 7f362211c700 1 -- 172.21.15.32:0/3700905027 learned_addr learned my addr 172.21.15.32:0/3700905027 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:55.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.060+0000 7f362211c700 1 -- 172.21.15.32:0/3700905027 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f361c19a0f0 con 0x7f361c100ba0 2024-04-25T00:23:55.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.060+0000 7f362211c700 1 --2- 172.21.15.32:0/3700905027 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f361c100ba0 0x7f361c06ab60 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f3610006010 tx=0x7f3610005050 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:55.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.060+0000 7f360f7fe700 1 -- 172.21.15.32:0/3700905027 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f3610027070 con 0x7f361c100ba0 2024-04-25T00:23:55.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.060+0000 7f3624380700 1 -- 172.21.15.32:0/3700905027 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f361c19a2c0 con 0x7f361c100ba0 2024-04-25T00:23:55.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.060+0000 7f3624380700 1 -- 172.21.15.32:0/3700905027 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f361c19a660 con 0x7f361c100ba0 2024-04-25T00:23:55.062 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.060+0000 7f360f7fe700 1 -- 172.21.15.32:0/3700905027 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f361000be40 con 0x7f361c100ba0 2024-04-25T00:23:55.062 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.060+0000 7f360f7fe700 1 -- 172.21.15.32:0/3700905027 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f361000f460 con 0x7f361c100ba0 2024-04-25T00:23:55.062 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.061+0000 7f360f7fe700 1 -- 172.21.15.32:0/3700905027 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 13) ==== 46343+0+0 (secure 0 0 0) 0x7f361002a070 con 0x7f361c100ba0 2024-04-25T00:23:55.063 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.061+0000 7f3624380700 1 -- 172.21.15.32:0/3700905027 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f361c1073b0 con 0x7f361c100ba0 2024-04-25T00:23:55.063 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.061+0000 7f360f7fe700 1 --2- 172.21.15.32:0/3700905027 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3608039150 0x7f360803b610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:55.063 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.061+0000 7f360f7fe700 1 -- 172.21.15.32:0/3700905027 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7f361001e030 con 0x7f361c100ba0 2024-04-25T00:23:55.063 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.062+0000 7f362191b700 1 --2- 172.21.15.32:0/3700905027 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3608039150 0x7f360803b610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:55.063 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.062+0000 7f362191b700 1 --2- 172.21.15.32:0/3700905027 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3608039150 0x7f360803b610 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f3618006fd0 tx=0x7f3618006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:55.067 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.067+0000 7f360f7fe700 1 -- 172.21.15.32:0/3700905027 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f3610013070 con 0x7f361c100ba0 2024-04-25T00:23:55.274 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.273+0000 7f3624380700 1 -- 172.21.15.32:0/3700905027 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command([{prefix=config set, name=mgr/cephadm/allow_ptrace}] v 0) -- 0x7f361c02dc90 con 0x7f361c100ba0 2024-04-25T00:23:55.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.276+0000 7f360f7fe700 1 -- 172.21.15.32:0/3700905027 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/cephadm/allow_ptrace}]=0 v9)=0 v9) ==== 125+0+0 (secure 0 0 0) 0x7f3610031020 con 0x7f361c100ba0 2024-04-25T00:23:55.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.278+0000 7f3624380700 1 -- 172.21.15.32:0/3700905027 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3608039150 msgr2=0x7f360803b610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:55.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.278+0000 7f3624380700 1 --2- 172.21.15.32:0/3700905027 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3608039150 0x7f360803b610 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f3618006fd0 tx=0x7f3618006e40 comp rx=0 tx=0).stop 2024-04-25T00:23:55.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.278+0000 7f3624380700 1 -- 172.21.15.32:0/3700905027 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f361c100ba0 msgr2=0x7f361c06ab60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:55.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.278+0000 7f3624380700 1 --2- 172.21.15.32:0/3700905027 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f361c100ba0 0x7f361c06ab60 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f3610006010 tx=0x7f3610005050 comp rx=0 tx=0).stop 2024-04-25T00:23:55.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.279+0000 7f3624380700 1 -- 172.21.15.32:0/3700905027 shutdown_connections 2024-04-25T00:23:55.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.279+0000 7f3624380700 1 --2- 172.21.15.32:0/3700905027 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3608039150 0x7f360803b610 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:55.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.279+0000 7f3624380700 1 --2- 172.21.15.32:0/3700905027 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f361c100ba0 0x7f361c06ab60 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:55.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.279+0000 7f3624380700 1 -- 172.21.15.32:0/3700905027 >> 172.21.15.32:0/3700905027 conn(0x7f361c0fc180 msgr2=0x7f361c0fcc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:55.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.279+0000 7f3624380700 1 -- 172.21.15.32:0/3700905027 shutdown_connections 2024-04-25T00:23:55.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:55.279+0000 7f3624380700 1 -- 172.21.15.32:0/3700905027 wait complete. 2024-04-25T00:23:56.535 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:56 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3700905027' entity='client.admin' 2024-04-25T00:23:56.535 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:56 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:23:56.535 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:56 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:23:56.535 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:56 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi032", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:23:56.535 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:56 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:23:56.536 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:56 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:23:56.536 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:56 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:23:56.564 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-25T00:23:56.564 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-25T00:23:56.809 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:23:58.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.288+0000 7f3d1a377700 1 -- 172.21.15.32:0/2969466877 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3d141033a0 msgr2=0x7f3d141037c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:58.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.288+0000 7f3d1a377700 1 --2- 172.21.15.32:0/2969466877 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3d141033a0 0x7f3d141037c0 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f3d08009910 tx=0x7f3d08009c20 comp rx=0 tx=0).stop 2024-04-25T00:23:58.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.288+0000 7f3d1a377700 1 -- 172.21.15.32:0/2969466877 shutdown_connections 2024-04-25T00:23:58.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.288+0000 7f3d1a377700 1 --2- 172.21.15.32:0/2969466877 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3d141033a0 0x7f3d141037c0 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:58.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.288+0000 7f3d1a377700 1 -- 172.21.15.32:0/2969466877 >> 172.21.15.32:0/2969466877 conn(0x7f3d140fd280 msgr2=0x7f3d140ff6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:58.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.288+0000 7f3d1a377700 1 -- 172.21.15.32:0/2969466877 shutdown_connections 2024-04-25T00:23:58.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.288+0000 7f3d1a377700 1 -- 172.21.15.32:0/2969466877 wait complete. 2024-04-25T00:23:58.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.288+0000 7f3d1a377700 1 Processor -- start 2024-04-25T00:23:58.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.288+0000 7f3d1a377700 1 -- start start 2024-04-25T00:23:58.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.289+0000 7f3d1a377700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3d141033a0 0x7f3d14075ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:58.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.289+0000 7f3d1a377700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f3d14107e60 con 0x7f3d141033a0 2024-04-25T00:23:58.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.289+0000 7f3d13fff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3d141033a0 0x7f3d14075ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:58.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.289+0000 7f3d13fff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3d141033a0 0x7f3d14075ef0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:37810/0 (socket says 172.21.15.32:37810) 2024-04-25T00:23:58.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.289+0000 7f3d13fff700 1 -- 172.21.15.32:0/2739760158 learned_addr learned my addr 172.21.15.32:0/2739760158 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:23:58.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.289+0000 7f3d13fff700 1 -- 172.21.15.32:0/2739760158 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3d14076430 con 0x7f3d141033a0 2024-04-25T00:23:58.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.289+0000 7f3d13fff700 1 --2- 172.21.15.32:0/2739760158 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3d141033a0 0x7f3d14075ef0 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f3d08005850 tx=0x7f3d08005030 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:58.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.289+0000 7f3d117fa700 1 -- 172.21.15.32:0/2739760158 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f3d08026070 con 0x7f3d141033a0 2024-04-25T00:23:58.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.289+0000 7f3d117fa700 1 -- 172.21.15.32:0/2739760158 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3d080044a0 con 0x7f3d141033a0 2024-04-25T00:23:58.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.289+0000 7f3d117fa700 1 -- 172.21.15.32:0/2739760158 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f3d08003ab0 con 0x7f3d141033a0 2024-04-25T00:23:58.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.289+0000 7f3d1a377700 1 -- 172.21.15.32:0/2739760158 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3d14074620 con 0x7f3d141033a0 2024-04-25T00:23:58.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.290+0000 7f3d1a377700 1 -- 172.21.15.32:0/2739760158 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f3d14074b60 con 0x7f3d141033a0 2024-04-25T00:23:58.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.291+0000 7f3d117fa700 1 -- 172.21.15.32:0/2739760158 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 13) ==== 46343+0+0 (secure 0 0 0) 0x7f3d08029070 con 0x7f3d141033a0 2024-04-25T00:23:58.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.291+0000 7f3d1a377700 1 -- 172.21.15.32:0/2739760158 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f3d141085e0 con 0x7f3d141033a0 2024-04-25T00:23:58.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.291+0000 7f3d117fa700 1 --2- 172.21.15.32:0/2739760158 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3cfc039150 0x7f3cfc03b610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:23:58.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.291+0000 7f3d117fa700 1 -- 172.21.15.32:0/2739760158 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7f3d0801e070 con 0x7f3d141033a0 2024-04-25T00:23:58.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.291+0000 7f3d137fe700 1 --2- 172.21.15.32:0/2739760158 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3cfc039150 0x7f3cfc03b610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:23:58.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.291+0000 7f3d137fe700 1 --2- 172.21.15.32:0/2739760158 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3cfc039150 0x7f3cfc03b610 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f3d04006fd0 tx=0x7f3d04006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:23:58.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.296+0000 7f3d117fa700 1 -- 172.21.15.32:0/2739760158 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f3d08013070 con 0x7f3d141033a0 2024-04-25T00:23:58.506 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.505+0000 7f3d1a377700 1 -- 172.21.15.32:0/2739760158 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}) -- 0x7f3d141907d0 con 0x7f3cfc039150 2024-04-25T00:23:58.509 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.509+0000 7f3d117fa700 1 -- 172.21.15.32:0/2739760158 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+0 (secure 0 0 0) 0x7f3d141907d0 con 0x7f3cfc039150 2024-04-25T00:23:58.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.511+0000 7f3d1a377700 1 -- 172.21.15.32:0/2739760158 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3cfc039150 msgr2=0x7f3cfc03b610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:58.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.511+0000 7f3d1a377700 1 --2- 172.21.15.32:0/2739760158 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3cfc039150 0x7f3cfc03b610 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f3d04006fd0 tx=0x7f3d04006e40 comp rx=0 tx=0).stop 2024-04-25T00:23:58.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.511+0000 7f3d1a377700 1 -- 172.21.15.32:0/2739760158 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3d141033a0 msgr2=0x7f3d14075ef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:23:58.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.511+0000 7f3d1a377700 1 --2- 172.21.15.32:0/2739760158 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3d141033a0 0x7f3d14075ef0 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f3d08005850 tx=0x7f3d08005030 comp rx=0 tx=0).stop 2024-04-25T00:23:58.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.511+0000 7f3d1a377700 1 -- 172.21.15.32:0/2739760158 shutdown_connections 2024-04-25T00:23:58.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.511+0000 7f3d1a377700 1 --2- 172.21.15.32:0/2739760158 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3cfc039150 0x7f3cfc03b610 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:58.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.511+0000 7f3d1a377700 1 --2- 172.21.15.32:0/2739760158 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3d141033a0 0x7f3d14075ef0 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:23:58.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.511+0000 7f3d1a377700 1 -- 172.21.15.32:0/2739760158 >> 172.21.15.32:0/2739760158 conn(0x7f3d140fd280 msgr2=0x7f3d140fdce0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:23:58.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.511+0000 7f3d1a377700 1 -- 172.21.15.32:0/2739760158 shutdown_connections 2024-04-25T00:23:58.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:23:58.511+0000 7f3d1a377700 1 -- 172.21.15.32:0/2739760158 wait complete. 2024-04-25T00:23:59.247 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi121 2024-04-25T00:23:59.247 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-04-25T00:23:59.247 DEBUG:teuthology.orchestra.run.smithi121:> dd of=/etc/ceph/ceph.conf 2024-04-25T00:23:59.265 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-04-25T00:23:59.265 DEBUG:teuthology.orchestra.run.smithi121:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-25T00:23:59.323 INFO:tasks.cephadm:Adding host smithi121 to orchestrator... 2024-04-25T00:23:59.323 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph orch host add smithi121 2024-04-25T00:23:59.577 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:23:59.800 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:59 smithi032 ceph-mon[29357]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:23:59.801 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:59 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:23:59.801 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:59 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:23:59.801 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:59 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:23:59.801 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:59 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:23:59.801 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:59 smithi032 ceph-mon[29357]: Updating smithi032:/etc/ceph/ceph.conf 2024-04-25T00:23:59.801 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:59 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:23:59.801 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:59 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:23:59.801 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:23:59 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:00.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:00 smithi032 ceph-mon[29357]: Updating smithi032:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:24:00.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:00 smithi032 ceph-mon[29357]: Updating smithi032:/etc/ceph/ceph.client.admin.keyring 2024-04-25T00:24:00.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:00 smithi032 ceph-mon[29357]: Updating smithi032:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T00:24:01.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.044+0000 7fc0d9f63700 1 -- 172.21.15.32:0/3147415427 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc0d4100da0 msgr2=0x7fc0d4101180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:01.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.044+0000 7fc0d9f63700 1 --2- 172.21.15.32:0/3147415427 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc0d4100da0 0x7fc0d4101180 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7fc0c4009910 tx=0x7fc0c4009c20 comp rx=0 tx=0).stop 2024-04-25T00:24:01.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.044+0000 7fc0d9f63700 1 -- 172.21.15.32:0/3147415427 shutdown_connections 2024-04-25T00:24:01.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.044+0000 7fc0d9f63700 1 --2- 172.21.15.32:0/3147415427 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc0d4100da0 0x7fc0d4101180 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:01.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.044+0000 7fc0d9f63700 1 -- 172.21.15.32:0/3147415427 >> 172.21.15.32:0/3147415427 conn(0x7fc0d40fc650 msgr2=0x7fc0d40fea70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:24:01.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.044+0000 7fc0d9f63700 1 -- 172.21.15.32:0/3147415427 shutdown_connections 2024-04-25T00:24:01.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.045+0000 7fc0d9f63700 1 -- 172.21.15.32:0/3147415427 wait complete. 2024-04-25T00:24:01.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.045+0000 7fc0d9f63700 1 Processor -- start 2024-04-25T00:24:01.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.045+0000 7fc0d9f63700 1 -- start start 2024-04-25T00:24:01.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.045+0000 7fc0d9f63700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc0d4100da0 0x7fc0d4197710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:01.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.045+0000 7fc0d9f63700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fc0d410b170 con 0x7fc0d4100da0 2024-04-25T00:24:01.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.046+0000 7fc0d37fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc0d4100da0 0x7fc0d4197710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:01.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.046+0000 7fc0d37fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc0d4100da0 0x7fc0d4197710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:37826/0 (socket says 172.21.15.32:37826) 2024-04-25T00:24:01.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.046+0000 7fc0d37fe700 1 -- 172.21.15.32:0/523770975 learned_addr learned my addr 172.21.15.32:0/523770975 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:24:01.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.046+0000 7fc0d37fe700 1 -- 172.21.15.32:0/523770975 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc0d4197c50 con 0x7fc0d4100da0 2024-04-25T00:24:01.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.046+0000 7fc0d37fe700 1 --2- 172.21.15.32:0/523770975 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc0d4100da0 0x7fc0d4197710 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7fc0c4005850 tx=0x7fc0c4005030 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:24:01.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.046+0000 7fc0d0ff9700 1 -- 172.21.15.32:0/523770975 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fc0c4026070 con 0x7fc0d4100da0 2024-04-25T00:24:01.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.046+0000 7fc0d0ff9700 1 -- 172.21.15.32:0/523770975 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc0c400bbd0 con 0x7fc0d4100da0 2024-04-25T00:24:01.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.046+0000 7fc0d9f63700 1 -- 172.21.15.32:0/523770975 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc0d4197f40 con 0x7fc0d4100da0 2024-04-25T00:24:01.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.046+0000 7fc0d0ff9700 1 -- 172.21.15.32:0/523770975 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fc0c4003970 con 0x7fc0d4100da0 2024-04-25T00:24:01.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.046+0000 7fc0d9f63700 1 -- 172.21.15.32:0/523770975 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc0d419bfe0 con 0x7fc0d4100da0 2024-04-25T00:24:01.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.047+0000 7fc0d0ff9700 1 -- 172.21.15.32:0/523770975 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 13) ==== 46343+0+0 (secure 0 0 0) 0x7fc0c4029070 con 0x7fc0d4100da0 2024-04-25T00:24:01.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.047+0000 7fc0d9f63700 1 -- 172.21.15.32:0/523770975 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc0d4102280 con 0x7fc0d4100da0 2024-04-25T00:24:01.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.047+0000 7fc0d0ff9700 1 --2- 172.21.15.32:0/523770975 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc0bc039150 0x7fc0bc03b610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:01.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.047+0000 7fc0d0ff9700 1 -- 172.21.15.32:0/523770975 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7fc0c401e070 con 0x7fc0d4100da0 2024-04-25T00:24:01.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.048+0000 7fc0d2ffd700 1 --2- 172.21.15.32:0/523770975 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc0bc039150 0x7fc0bc03b610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:01.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.048+0000 7fc0d2ffd700 1 --2- 172.21.15.32:0/523770975 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc0bc039150 0x7fc0bc03b610 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7fc0c8006fd0 tx=0x7fc0c8006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:24:01.053 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.052+0000 7fc0d0ff9700 1 -- 172.21.15.32:0/523770975 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fc0c4050030 con 0x7fc0d4100da0 2024-04-25T00:24:01.264 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:01.263+0000 7fc0d9f63700 1 -- 172.21.15.32:0/523770975 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch host add", "hostname": "smithi121", "target": ["mon-mgr", ""]}) -- 0x7fc0d4198a70 con 0x7fc0bc039150 2024-04-25T00:24:01.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:01 smithi032 ceph-mon[29357]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi121", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:24:02.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:02 smithi032 ceph-mon[29357]: Deploying cephadm binary to smithi121 2024-04-25T00:24:03.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:03 smithi032 ceph-mon[29357]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:05.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:05.490+0000 7fc0d0ff9700 1 -- 172.21.15.32:0/523770975 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+49 (secure 0 0 0) 0x7fc0d4198a70 con 0x7fc0bc039150 2024-04-25T00:24:05.492 INFO:teuthology.orchestra.run.smithi032.stdout:Added host 'smithi121' with addr '172.21.15.121' 2024-04-25T00:24:05.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:05.493+0000 7fc0d9f63700 1 -- 172.21.15.32:0/523770975 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc0bc039150 msgr2=0x7fc0bc03b610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:05.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:05.493+0000 7fc0d9f63700 1 --2- 172.21.15.32:0/523770975 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc0bc039150 0x7fc0bc03b610 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7fc0c8006fd0 tx=0x7fc0c8006e40 comp rx=0 tx=0).stop 2024-04-25T00:24:05.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:05.493+0000 7fc0d9f63700 1 -- 172.21.15.32:0/523770975 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc0d4100da0 msgr2=0x7fc0d4197710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:05.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:05.493+0000 7fc0d9f63700 1 --2- 172.21.15.32:0/523770975 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc0d4100da0 0x7fc0d4197710 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7fc0c4005850 tx=0x7fc0c4005030 comp rx=0 tx=0).stop 2024-04-25T00:24:05.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:05.493+0000 7fc0d9f63700 1 -- 172.21.15.32:0/523770975 shutdown_connections 2024-04-25T00:24:05.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:05.493+0000 7fc0d9f63700 1 --2- 172.21.15.32:0/523770975 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc0bc039150 0x7fc0bc03b610 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:05.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:05.493+0000 7fc0d9f63700 1 --2- 172.21.15.32:0/523770975 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc0d4100da0 0x7fc0d4197710 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:05.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:05.493+0000 7fc0d9f63700 1 -- 172.21.15.32:0/523770975 >> 172.21.15.32:0/523770975 conn(0x7fc0d40fc650 msgr2=0x7fc0d406a0a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:24:05.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:05.493+0000 7fc0d9f63700 1 -- 172.21.15.32:0/523770975 shutdown_connections 2024-04-25T00:24:05.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:05.493+0000 7fc0d9f63700 1 -- 172.21.15.32:0/523770975 wait complete. 2024-04-25T00:24:06.073 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph orch host ls --format=json 2024-04-25T00:24:06.321 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:24:06.733 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:06 smithi032 ceph-mon[29357]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:06.733 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:06 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:06.733 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:06 smithi032 ceph-mon[29357]: Added host smithi121 2024-04-25T00:24:06.733 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:06 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:24:06.733 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:06 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:07.610 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:07.865 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.864+0000 7fde1316a700 1 -- 172.21.15.32:0/721441391 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fde0c101ea0 msgr2=0x7fde0c102280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:07.865 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.864+0000 7fde1316a700 1 --2- 172.21.15.32:0/721441391 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fde0c101ea0 0x7fde0c102280 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7fde00009910 tx=0x7fde00009c20 comp rx=0 tx=0).stop 2024-04-25T00:24:07.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.864+0000 7fde1316a700 1 -- 172.21.15.32:0/721441391 shutdown_connections 2024-04-25T00:24:07.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.864+0000 7fde1316a700 1 --2- 172.21.15.32:0/721441391 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fde0c101ea0 0x7fde0c102280 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:07.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.864+0000 7fde1316a700 1 -- 172.21.15.32:0/721441391 >> 172.21.15.32:0/721441391 conn(0x7fde0c0fd770 msgr2=0x7fde0c0ffb90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:24:07.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.864+0000 7fde1316a700 1 -- 172.21.15.32:0/721441391 shutdown_connections 2024-04-25T00:24:07.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.864+0000 7fde1316a700 1 -- 172.21.15.32:0/721441391 wait complete. 2024-04-25T00:24:07.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.865+0000 7fde1316a700 1 Processor -- start 2024-04-25T00:24:07.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.865+0000 7fde1316a700 1 -- start start 2024-04-25T00:24:07.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.865+0000 7fde1316a700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fde0c101ea0 0x7fde0c074530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:07.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.865+0000 7fde1316a700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fde0c10c270 con 0x7fde0c101ea0 2024-04-25T00:24:07.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.865+0000 7fde10f06700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fde0c101ea0 0x7fde0c074530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:07.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.865+0000 7fde10f06700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fde0c101ea0 0x7fde0c074530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:47706/0 (socket says 172.21.15.32:47706) 2024-04-25T00:24:07.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.865+0000 7fde10f06700 1 -- 172.21.15.32:0/4157061530 learned_addr learned my addr 172.21.15.32:0/4157061530 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:24:07.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.865+0000 7fde10f06700 1 -- 172.21.15.32:0/4157061530 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fde0c076420 con 0x7fde0c101ea0 2024-04-25T00:24:07.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.865+0000 7fde10f06700 1 --2- 172.21.15.32:0/4157061530 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fde0c101ea0 0x7fde0c074530 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7fde00006010 tx=0x7fde00005030 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:24:07.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.866+0000 7fde09ffb700 1 -- 172.21.15.32:0/4157061530 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fde00026070 con 0x7fde0c101ea0 2024-04-25T00:24:07.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.866+0000 7fde1316a700 1 -- 172.21.15.32:0/4157061530 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fde0c074ad0 con 0x7fde0c101ea0 2024-04-25T00:24:07.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.866+0000 7fde09ffb700 1 -- 172.21.15.32:0/4157061530 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fde0000bc30 con 0x7fde0c101ea0 2024-04-25T00:24:07.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.866+0000 7fde09ffb700 1 -- 172.21.15.32:0/4157061530 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fde000039d0 con 0x7fde0c101ea0 2024-04-25T00:24:07.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.866+0000 7fde1316a700 1 -- 172.21.15.32:0/4157061530 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fde0c074f90 con 0x7fde0c101ea0 2024-04-25T00:24:07.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.867+0000 7fde09ffb700 1 -- 172.21.15.32:0/4157061530 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 13) ==== 46343+0+0 (secure 0 0 0) 0x7fde00029070 con 0x7fde0c101ea0 2024-04-25T00:24:07.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.867+0000 7fde1316a700 1 -- 172.21.15.32:0/4157061530 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fde0c103380 con 0x7fde0c101ea0 2024-04-25T00:24:07.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.867+0000 7fde09ffb700 1 --2- 172.21.15.32:0/4157061530 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fddfc039150 0x7fddfc03b610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:07.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.867+0000 7fde09ffb700 1 -- 172.21.15.32:0/4157061530 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7fde0001e070 con 0x7fde0c101ea0 2024-04-25T00:24:07.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.867+0000 7fde0bfff700 1 --2- 172.21.15.32:0/4157061530 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fddfc039150 0x7fddfc03b610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:07.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.868+0000 7fde0bfff700 1 --2- 172.21.15.32:0/4157061530 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fddfc039150 0x7fddfc03b610 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7fddf8006fd0 tx=0x7fddf8006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:24:07.873 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:07.872+0000 7fde09ffb700 1 -- 172.21.15.32:0/4157061530 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fde00013070 con 0x7fde0c101ea0 2024-04-25T00:24:08.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:08.089+0000 7fde1316a700 1 -- 172.21.15.32:0/4157061530 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7fde0c075220 con 0x7fddfc039150 2024-04-25T00:24:08.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:08.091+0000 7fde09ffb700 1 -- 172.21.15.32:0/4157061530 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+160 (secure 0 0 0) 0x7fde0c075220 con 0x7fddfc039150 2024-04-25T00:24:08.092 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:24:08.092 INFO:teuthology.orchestra.run.smithi032.stdout:[{"addr": "172.21.15.32", "hostname": "smithi032", "labels": [], "status": ""}, {"addr": "172.21.15.121", "hostname": "smithi121", "labels": [], "status": ""}] 2024-04-25T00:24:08.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:08.093+0000 7fde1316a700 1 -- 172.21.15.32:0/4157061530 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fddfc039150 msgr2=0x7fddfc03b610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:08.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:08.093+0000 7fde1316a700 1 --2- 172.21.15.32:0/4157061530 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fddfc039150 0x7fddfc03b610 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7fddf8006fd0 tx=0x7fddf8006e40 comp rx=0 tx=0).stop 2024-04-25T00:24:08.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:08.093+0000 7fde1316a700 1 -- 172.21.15.32:0/4157061530 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fde0c101ea0 msgr2=0x7fde0c074530 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:08.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:08.093+0000 7fde1316a700 1 --2- 172.21.15.32:0/4157061530 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fde0c101ea0 0x7fde0c074530 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7fde00006010 tx=0x7fde00005030 comp rx=0 tx=0).stop 2024-04-25T00:24:08.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:08.093+0000 7fde1316a700 1 -- 172.21.15.32:0/4157061530 shutdown_connections 2024-04-25T00:24:08.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:08.093+0000 7fde1316a700 1 --2- 172.21.15.32:0/4157061530 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fddfc039150 0x7fddfc03b610 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:08.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:08.093+0000 7fde1316a700 1 --2- 172.21.15.32:0/4157061530 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fde0c101ea0 0x7fde0c074530 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:08.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:08.093+0000 7fde1316a700 1 -- 172.21.15.32:0/4157061530 >> 172.21.15.32:0/4157061530 conn(0x7fde0c0fd770 msgr2=0x7fde0c103870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:24:08.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:08.093+0000 7fde1316a700 1 -- 172.21.15.32:0/4157061530 shutdown_connections 2024-04-25T00:24:08.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:08.093+0000 7fde1316a700 1 -- 172.21.15.32:0/4157061530 wait complete. 2024-04-25T00:24:08.662 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:08 smithi032 ceph-mon[29357]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:08.662 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:08 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:08.663 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi130 2024-04-25T00:24:08.663 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-04-25T00:24:08.663 DEBUG:teuthology.orchestra.run.smithi130:> dd of=/etc/ceph/ceph.conf 2024-04-25T00:24:08.683 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-04-25T00:24:08.684 DEBUG:teuthology.orchestra.run.smithi130:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-25T00:24:08.741 INFO:tasks.cephadm:Adding host smithi130 to orchestrator... 2024-04-25T00:24:08.741 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph orch host add smithi130 2024-04-25T00:24:08.985 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:24:09.931 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:09 smithi032 ceph-mon[29357]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:24:10.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.417+0000 7fa6c63aa700 1 -- 172.21.15.32:0/4119342287 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa6c0102e40 msgr2=0x7fa6c0103220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:10.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.417+0000 7fa6c63aa700 1 --2- 172.21.15.32:0/4119342287 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa6c0102e40 0x7fa6c0103220 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7fa6b4009910 tx=0x7fa6b4009c20 comp rx=0 tx=0).stop 2024-04-25T00:24:10.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.417+0000 7fa6c63aa700 1 -- 172.21.15.32:0/4119342287 shutdown_connections 2024-04-25T00:24:10.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.417+0000 7fa6c63aa700 1 --2- 172.21.15.32:0/4119342287 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa6c0102e40 0x7fa6c0103220 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:10.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.417+0000 7fa6c63aa700 1 -- 172.21.15.32:0/4119342287 >> 172.21.15.32:0/4119342287 conn(0x7fa6c00fc690 msgr2=0x7fa6c00feab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:24:10.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.417+0000 7fa6c63aa700 1 -- 172.21.15.32:0/4119342287 shutdown_connections 2024-04-25T00:24:10.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.417+0000 7fa6c63aa700 1 -- 172.21.15.32:0/4119342287 wait complete. 2024-04-25T00:24:10.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.417+0000 7fa6c63aa700 1 Processor -- start 2024-04-25T00:24:10.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.418+0000 7fa6c63aa700 1 -- start start 2024-04-25T00:24:10.420 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.418+0000 7fa6c63aa700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa6c0102e40 0x7fa6c0100aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:10.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.418+0000 7fa6c63aa700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fa6c010c310 con 0x7fa6c0102e40 2024-04-25T00:24:10.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.418+0000 7fa6bffff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa6c0102e40 0x7fa6c0100aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:10.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.418+0000 7fa6bffff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa6c0102e40 0x7fa6c0100aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:47716/0 (socket says 172.21.15.32:47716) 2024-04-25T00:24:10.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.418+0000 7fa6bffff700 1 -- 172.21.15.32:0/520441198 learned_addr learned my addr 172.21.15.32:0/520441198 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:24:10.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.418+0000 7fa6bffff700 1 -- 172.21.15.32:0/520441198 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa6c0102990 con 0x7fa6c0102e40 2024-04-25T00:24:10.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.418+0000 7fa6bffff700 1 --2- 172.21.15.32:0/520441198 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa6c0102e40 0x7fa6c0100aa0 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7fa6b4006010 tx=0x7fa6b4005050 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:24:10.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.419+0000 7fa6bd7fa700 1 -- 172.21.15.32:0/520441198 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fa6b4026070 con 0x7fa6c0102e40 2024-04-25T00:24:10.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.419+0000 7fa6bd7fa700 1 -- 172.21.15.32:0/520441198 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa6b400bc30 con 0x7fa6c0102e40 2024-04-25T00:24:10.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.419+0000 7fa6c63aa700 1 -- 172.21.15.32:0/520441198 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa6c0100fe0 con 0x7fa6c0102e40 2024-04-25T00:24:10.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.419+0000 7fa6bd7fa700 1 -- 172.21.15.32:0/520441198 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fa6b4003ab0 con 0x7fa6c0102e40 2024-04-25T00:24:10.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.419+0000 7fa6c63aa700 1 -- 172.21.15.32:0/520441198 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa6c01014a0 con 0x7fa6c0102e40 2024-04-25T00:24:10.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.419+0000 7fa6bd7fa700 1 -- 172.21.15.32:0/520441198 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 13) ==== 46343+0+0 (secure 0 0 0) 0x7fa6b4029070 con 0x7fa6c0102e40 2024-04-25T00:24:10.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.420+0000 7fa6c63aa700 1 -- 172.21.15.32:0/520441198 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa6c006a980 con 0x7fa6c0102e40 2024-04-25T00:24:10.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.420+0000 7fa6bd7fa700 1 --2- 172.21.15.32:0/520441198 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa6a8039150 0x7fa6a803b610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:10.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.420+0000 7fa6bd7fa700 1 -- 172.21.15.32:0/520441198 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7fa6b401e070 con 0x7fa6c0102e40 2024-04-25T00:24:10.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.420+0000 7fa6bf7fe700 1 --2- 172.21.15.32:0/520441198 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa6a8039150 0x7fa6a803b610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:10.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.420+0000 7fa6bf7fe700 1 --2- 172.21.15.32:0/520441198 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa6a8039150 0x7fa6a803b610 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7fa6b0006fd0 tx=0x7fa6b0006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:24:10.425 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.425+0000 7fa6bd7fa700 1 -- 172.21.15.32:0/520441198 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fa6b4013070 con 0x7fa6c0102e40 2024-04-25T00:24:10.640 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:10.639+0000 7fa6c63aa700 1 -- 172.21.15.32:0/520441198 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch host add", "hostname": "smithi130", "target": ["mon-mgr", ""]}) -- 0x7fa6c01016d0 con 0x7fa6a8039150 2024-04-25T00:24:10.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:10 smithi032 ceph-mon[29357]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:11.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:11 smithi032 ceph-mon[29357]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi130", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:24:11.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:11 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:11.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:11 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:11.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:11 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:11.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:11 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:11.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:11 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:24:11.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:11 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:11.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:11 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:24:12.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:12 smithi032 ceph-mon[29357]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:12.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:12 smithi032 ceph-mon[29357]: Updating smithi121:/etc/ceph/ceph.conf 2024-04-25T00:24:12.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:12 smithi032 ceph-mon[29357]: Deploying cephadm binary to smithi130 2024-04-25T00:24:12.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:12 smithi032 ceph-mon[29357]: Updating smithi121:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:24:13.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:13 smithi032 ceph-mon[29357]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-04-25T00:24:14.890 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:14 smithi032 ceph-mon[29357]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:14.890 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:14.889+0000 7fa6bd7fa700 1 -- 172.21.15.32:0/520441198 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+49 (secure 0 0 0) 0x7fa6c01016d0 con 0x7fa6a8039150 2024-04-25T00:24:14.891 INFO:teuthology.orchestra.run.smithi032.stdout:Added host 'smithi130' with addr '172.21.15.130' 2024-04-25T00:24:14.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:14.891+0000 7fa6c63aa700 1 -- 172.21.15.32:0/520441198 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa6a8039150 msgr2=0x7fa6a803b610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:14.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:14.891+0000 7fa6c63aa700 1 --2- 172.21.15.32:0/520441198 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa6a8039150 0x7fa6a803b610 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7fa6b0006fd0 tx=0x7fa6b0006e40 comp rx=0 tx=0).stop 2024-04-25T00:24:14.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:14.891+0000 7fa6c63aa700 1 -- 172.21.15.32:0/520441198 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa6c0102e40 msgr2=0x7fa6c0100aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:14.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:14.891+0000 7fa6c63aa700 1 --2- 172.21.15.32:0/520441198 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa6c0102e40 0x7fa6c0100aa0 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7fa6b4006010 tx=0x7fa6b4005050 comp rx=0 tx=0).stop 2024-04-25T00:24:14.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:14.891+0000 7fa6c63aa700 1 -- 172.21.15.32:0/520441198 shutdown_connections 2024-04-25T00:24:14.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:14.892+0000 7fa6c63aa700 1 --2- 172.21.15.32:0/520441198 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa6a8039150 0x7fa6a803b610 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:14.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:14.892+0000 7fa6c63aa700 1 --2- 172.21.15.32:0/520441198 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa6c0102e40 0x7fa6c0100aa0 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:14.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:14.892+0000 7fa6c63aa700 1 -- 172.21.15.32:0/520441198 >> 172.21.15.32:0/520441198 conn(0x7fa6c00fc690 msgr2=0x7fa6c0103bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:24:14.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:14.892+0000 7fa6c63aa700 1 -- 172.21.15.32:0/520441198 shutdown_connections 2024-04-25T00:24:14.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:14.892+0000 7fa6c63aa700 1 -- 172.21.15.32:0/520441198 wait complete. 2024-04-25T00:24:15.484 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph orch host ls --format=json 2024-04-25T00:24:15.728 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:24:15.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:15 smithi032 ceph-mon[29357]: Updating smithi121:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T00:24:15.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:15 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:15.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:15 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:15.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:15 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:15.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:15 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:15.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:15 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:24:15.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:15 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:16.735 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:16 smithi032 ceph-mon[29357]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:16.735 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:16 smithi032 ceph-mon[29357]: Added host smithi130 2024-04-25T00:24:16.735 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:16 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:17.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.226+0000 7f835a5e9700 1 -- 172.21.15.32:0/987675649 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8354102e40 msgr2=0x7f8354103220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:17.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.226+0000 7f835a5e9700 1 --2- 172.21.15.32:0/987675649 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8354102e40 0x7f8354103220 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f8344009910 tx=0x7f8344009c20 comp rx=0 tx=0).stop 2024-04-25T00:24:17.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.226+0000 7f835a5e9700 1 -- 172.21.15.32:0/987675649 shutdown_connections 2024-04-25T00:24:17.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.226+0000 7f835a5e9700 1 --2- 172.21.15.32:0/987675649 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8354102e40 0x7f8354103220 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:17.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.226+0000 7f835a5e9700 1 -- 172.21.15.32:0/987675649 >> 172.21.15.32:0/987675649 conn(0x7f83540fc690 msgr2=0x7f83540feab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:24:17.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.226+0000 7f835a5e9700 1 -- 172.21.15.32:0/987675649 shutdown_connections 2024-04-25T00:24:17.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.226+0000 7f835a5e9700 1 -- 172.21.15.32:0/987675649 wait complete. 2024-04-25T00:24:17.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.227+0000 7f835a5e9700 1 Processor -- start 2024-04-25T00:24:17.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.227+0000 7f835a5e9700 1 -- start start 2024-04-25T00:24:17.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.227+0000 7f835a5e9700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8354102e40 0x7f835419d430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:17.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.227+0000 7f835a5e9700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f835410c310 con 0x7f8354102e40 2024-04-25T00:24:17.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.227+0000 7f8353fff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8354102e40 0x7f835419d430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:17.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.227+0000 7f8353fff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8354102e40 0x7f835419d430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:38644/0 (socket says 172.21.15.32:38644) 2024-04-25T00:24:17.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.227+0000 7f8353fff700 1 -- 172.21.15.32:0/1021392420 learned_addr learned my addr 172.21.15.32:0/1021392420 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:24:17.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.228+0000 7f8353fff700 1 -- 172.21.15.32:0/1021392420 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f835419d970 con 0x7f8354102e40 2024-04-25T00:24:17.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.228+0000 7f8353fff700 1 --2- 172.21.15.32:0/1021392420 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8354102e40 0x7f835419d430 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f8344006010 tx=0x7f8344005050 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:24:17.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.228+0000 7f83517fa700 1 -- 172.21.15.32:0/1021392420 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f8344026070 con 0x7f8354102e40 2024-04-25T00:24:17.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.228+0000 7f83517fa700 1 -- 172.21.15.32:0/1021392420 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f834400bc30 con 0x7f8354102e40 2024-04-25T00:24:17.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.228+0000 7f835a5e9700 1 -- 172.21.15.32:0/1021392420 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f835419dc00 con 0x7f8354102e40 2024-04-25T00:24:17.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.228+0000 7f83517fa700 1 -- 172.21.15.32:0/1021392420 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f8344003ab0 con 0x7f8354102e40 2024-04-25T00:24:17.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.228+0000 7f835a5e9700 1 -- 172.21.15.32:0/1021392420 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f83541975f0 con 0x7f8354102e40 2024-04-25T00:24:17.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.229+0000 7f83517fa700 1 -- 172.21.15.32:0/1021392420 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 13) ==== 46343+0+0 (secure 0 0 0) 0x7f8344029070 con 0x7f8354102e40 2024-04-25T00:24:17.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.229+0000 7f835a5e9700 1 -- 172.21.15.32:0/1021392420 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f835406a980 con 0x7f8354102e40 2024-04-25T00:24:17.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.229+0000 7f83517fa700 1 --2- 172.21.15.32:0/1021392420 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8340039150 0x7f834003b610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:17.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.229+0000 7f83517fa700 1 -- 172.21.15.32:0/1021392420 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7f834401e070 con 0x7f8354102e40 2024-04-25T00:24:17.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.229+0000 7f83537fe700 1 --2- 172.21.15.32:0/1021392420 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8340039150 0x7f834003b610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:17.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.230+0000 7f83537fe700 1 --2- 172.21.15.32:0/1021392420 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8340039150 0x7f834003b610 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f833c006fd0 tx=0x7f833c006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:24:17.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.234+0000 7f83517fa700 1 -- 172.21.15.32:0/1021392420 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f8344013070 con 0x7f8354102e40 2024-04-25T00:24:17.445 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.444+0000 7f835a5e9700 1 -- 172.21.15.32:0/1021392420 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f8354063890 con 0x7f8340039150 2024-04-25T00:24:17.446 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.446+0000 7f83517fa700 1 -- 172.21.15.32:0/1021392420 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+240 (secure 0 0 0) 0x7f8354063890 con 0x7f8340039150 2024-04-25T00:24:17.446 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:24:17.447 INFO:teuthology.orchestra.run.smithi032.stdout:[{"addr": "172.21.15.32", "hostname": "smithi032", "labels": [], "status": ""}, {"addr": "172.21.15.121", "hostname": "smithi121", "labels": [], "status": ""}, {"addr": "172.21.15.130", "hostname": "smithi130", "labels": [], "status": ""}] 2024-04-25T00:24:17.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.448+0000 7f835a5e9700 1 -- 172.21.15.32:0/1021392420 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8340039150 msgr2=0x7f834003b610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:17.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.448+0000 7f835a5e9700 1 --2- 172.21.15.32:0/1021392420 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8340039150 0x7f834003b610 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f833c006fd0 tx=0x7f833c006e40 comp rx=0 tx=0).stop 2024-04-25T00:24:17.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.448+0000 7f835a5e9700 1 -- 172.21.15.32:0/1021392420 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8354102e40 msgr2=0x7f835419d430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:17.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.448+0000 7f835a5e9700 1 --2- 172.21.15.32:0/1021392420 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8354102e40 0x7f835419d430 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f8344006010 tx=0x7f8344005050 comp rx=0 tx=0).stop 2024-04-25T00:24:17.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.448+0000 7f835a5e9700 1 -- 172.21.15.32:0/1021392420 shutdown_connections 2024-04-25T00:24:17.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.448+0000 7f835a5e9700 1 --2- 172.21.15.32:0/1021392420 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8340039150 0x7f834003b610 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:17.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.448+0000 7f835a5e9700 1 --2- 172.21.15.32:0/1021392420 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8354102e40 0x7f835419d430 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:17.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.448+0000 7f835a5e9700 1 -- 172.21.15.32:0/1021392420 >> 172.21.15.32:0/1021392420 conn(0x7f83540fc690 msgr2=0x7f8354103bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:24:17.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.448+0000 7f835a5e9700 1 -- 172.21.15.32:0/1021392420 shutdown_connections 2024-04-25T00:24:17.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:17.448+0000 7f835a5e9700 1 -- 172.21.15.32:0/1021392420 wait complete. 2024-04-25T00:24:18.053 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-25T00:24:18.054 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph osd crush tunables default 2024-04-25T00:24:18.294 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:24:18.661 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:18 smithi032 ceph-mon[29357]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:18.661 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:18 smithi032 ceph-mon[29357]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:24:18.661 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:18 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:19.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.898+0000 7f83e0a8c700 1 -- 172.21.15.32:0/2292660620 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f83dc101ea0 msgr2=0x7f83dc102280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:19.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.898+0000 7f83e0a8c700 1 --2- 172.21.15.32:0/2292660620 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f83dc101ea0 0x7f83dc102280 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f83cc009910 tx=0x7f83cc009c20 comp rx=0 tx=0).stop 2024-04-25T00:24:19.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.899+0000 7f83e0a8c700 1 -- 172.21.15.32:0/2292660620 shutdown_connections 2024-04-25T00:24:19.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.899+0000 7f83e0a8c700 1 --2- 172.21.15.32:0/2292660620 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f83dc101ea0 0x7f83dc102280 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:19.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.899+0000 7f83e0a8c700 1 -- 172.21.15.32:0/2292660620 >> 172.21.15.32:0/2292660620 conn(0x7f83dc0fd750 msgr2=0x7f83dc0ffb70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:24:19.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.899+0000 7f83e0a8c700 1 -- 172.21.15.32:0/2292660620 shutdown_connections 2024-04-25T00:24:19.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.899+0000 7f83e0a8c700 1 -- 172.21.15.32:0/2292660620 wait complete. 2024-04-25T00:24:19.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.899+0000 7f83e0a8c700 1 Processor -- start 2024-04-25T00:24:19.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.899+0000 7f83e0a8c700 1 -- start start 2024-04-25T00:24:19.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.900+0000 7f83e0a8c700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f83dc101ea0 0x7f83dc19d3e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:19.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.900+0000 7f83e0a8c700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f83dc10c270 con 0x7f83dc101ea0 2024-04-25T00:24:19.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.900+0000 7f83da59c700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f83dc101ea0 0x7f83dc19d3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:19.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.900+0000 7f83da59c700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f83dc101ea0 0x7f83dc19d3e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:38662/0 (socket says 172.21.15.32:38662) 2024-04-25T00:24:19.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.900+0000 7f83da59c700 1 -- 172.21.15.32:0/4182769857 learned_addr learned my addr 172.21.15.32:0/4182769857 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:24:19.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.900+0000 7f83da59c700 1 -- 172.21.15.32:0/4182769857 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f83dc19d920 con 0x7f83dc101ea0 2024-04-25T00:24:19.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.900+0000 7f83da59c700 1 --2- 172.21.15.32:0/4182769857 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f83dc101ea0 0x7f83dc19d3e0 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f83cc006010 tx=0x7f83cc005030 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:24:19.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.900+0000 7f83cb7fe700 1 -- 172.21.15.32:0/4182769857 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f83cc026070 con 0x7f83dc101ea0 2024-04-25T00:24:19.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.900+0000 7f83cb7fe700 1 -- 172.21.15.32:0/4182769857 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f83cc00bbd0 con 0x7f83dc101ea0 2024-04-25T00:24:19.903 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.900+0000 7f83e0a8c700 1 -- 172.21.15.32:0/4182769857 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f83dc19dc10 con 0x7f83dc101ea0 2024-04-25T00:24:19.903 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.901+0000 7f83cb7fe700 1 -- 172.21.15.32:0/4182769857 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f83cc003910 con 0x7f83dc101ea0 2024-04-25T00:24:19.903 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.901+0000 7f83e0a8c700 1 -- 172.21.15.32:0/4182769857 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f83dc1975a0 con 0x7f83dc101ea0 2024-04-25T00:24:19.904 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.901+0000 7f83cb7fe700 1 -- 172.21.15.32:0/4182769857 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 13) ==== 46343+0+0 (secure 0 0 0) 0x7f83cc029070 con 0x7f83dc101ea0 2024-04-25T00:24:19.904 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.902+0000 7f83cb7fe700 1 --2- 172.21.15.32:0/4182769857 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f83c40390f0 0x7f83c403b5b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:19.904 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.902+0000 7f83cb7fe700 1 -- 172.21.15.32:0/4182769857 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7f83cc01e070 con 0x7f83dc101ea0 2024-04-25T00:24:19.904 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.902+0000 7f83d9d9b700 1 --2- 172.21.15.32:0/4182769857 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f83c40390f0 0x7f83c403b5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:19.904 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.902+0000 7f83e0a8c700 1 -- 172.21.15.32:0/4182769857 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f83dc103380 con 0x7f83dc101ea0 2024-04-25T00:24:19.904 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.902+0000 7f83d9d9b700 1 --2- 172.21.15.32:0/4182769857 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f83c40390f0 0x7f83c403b5b0 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f83d0006fd0 tx=0x7f83d0006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:24:19.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:19.907+0000 7f83cb7fe700 1 -- 172.21.15.32:0/4182769857 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f83cc030020 con 0x7f83dc101ea0 2024-04-25T00:24:20.119 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:20.119+0000 7f83e0a8c700 1 -- 172.21.15.32:0/4182769857 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd crush tunables", "profile": "default"} v 0) -- 0x7f83dc198ad0 con 0x7f83dc101ea0 2024-04-25T00:24:20.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:20.462+0000 7f83cb7fe700 1 -- 172.21.15.32:0/4182769857 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd crush tunables", "profile": "default"}]=0 adjusted tunables profile to default v4) ==== 124+0+0 (secure 0 0 0) 0x7f83cc013070 con 0x7f83dc101ea0 2024-04-25T00:24:20.465 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:20.464+0000 7f83e0a8c700 1 -- 172.21.15.32:0/4182769857 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f83c40390f0 msgr2=0x7f83c403b5b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:20.465 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:20.464+0000 7f83e0a8c700 1 --2- 172.21.15.32:0/4182769857 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f83c40390f0 0x7f83c403b5b0 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f83d0006fd0 tx=0x7f83d0006e40 comp rx=0 tx=0).stop 2024-04-25T00:24:20.465 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:20.464+0000 7f83e0a8c700 1 -- 172.21.15.32:0/4182769857 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f83dc101ea0 msgr2=0x7f83dc19d3e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:20.465 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:20.464+0000 7f83e0a8c700 1 --2- 172.21.15.32:0/4182769857 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f83dc101ea0 0x7f83dc19d3e0 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f83cc006010 tx=0x7f83cc005030 comp rx=0 tx=0).stop 2024-04-25T00:24:20.466 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:20.465+0000 7f83e0a8c700 1 -- 172.21.15.32:0/4182769857 shutdown_connections 2024-04-25T00:24:20.466 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:20.465+0000 7f83e0a8c700 1 --2- 172.21.15.32:0/4182769857 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f83c40390f0 0x7f83c403b5b0 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:20.466 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:20.465+0000 7f83e0a8c700 1 --2- 172.21.15.32:0/4182769857 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f83dc101ea0 0x7f83dc19d3e0 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:20.466 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:20.465+0000 7f83e0a8c700 1 -- 172.21.15.32:0/4182769857 >> 172.21.15.32:0/4182769857 conn(0x7f83dc0fd750 msgr2=0x7f83dc103870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:24:20.466 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:20.465+0000 7f83e0a8c700 1 -- 172.21.15.32:0/4182769857 shutdown_connections 2024-04-25T00:24:20.466 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:20.465+0000 7f83e0a8c700 1 -- 172.21.15.32:0/4182769857 wait complete. 2024-04-25T00:24:20.466 INFO:teuthology.orchestra.run.smithi032.stderr:adjusted tunables profile to default 2024-04-25T00:24:20.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:20 smithi032 ceph-mon[29357]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:20.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:20 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/4182769857' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-25T00:24:21.133 INFO:tasks.cephadm:Adding mon.a on smithi032 2024-04-25T00:24:21.133 INFO:tasks.cephadm:Adding mon.b on smithi121 2024-04-25T00:24:21.134 INFO:tasks.cephadm:Adding mon.c on smithi130 2024-04-25T00:24:21.134 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph orch apply mon '3;smithi032:172.21.15.32=a;smithi121:172.21.15.121=b;smithi130:172.21.15.130=c' 2024-04-25T00:24:21.365 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:24:21.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:21 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/4182769857' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-25T00:24:21.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:21 smithi032 ceph-mon[29357]: osdmap e4: 0 total, 0 up, 0 in 2024-04-25T00:24:21.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:21 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:21.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:21 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:21.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:21 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:21.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:21 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:21.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:21 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi130", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:24:21.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:21 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:21.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:21 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:24:21.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:21 smithi032 ceph-mon[29357]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-25T00:24:21.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:21 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:21.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:21 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:21.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:21 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:22.992 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.990+0000 7f47231f8700 1 -- 172.21.15.130:0/3151514574 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f471c1018c0 msgr2=0x7f471c101ca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:22.992 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.990+0000 7f47231f8700 1 --2- 172.21.15.130:0/3151514574 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f471c1018c0 0x7f471c101ca0 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f4710009910 tx=0x7f4710009c20 comp rx=0 tx=0).stop 2024-04-25T00:24:22.992 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.990+0000 7f47231f8700 1 -- 172.21.15.130:0/3151514574 shutdown_connections 2024-04-25T00:24:22.992 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.990+0000 7f47231f8700 1 --2- 172.21.15.130:0/3151514574 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f471c1018c0 0x7f471c101ca0 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:22.992 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.990+0000 7f47231f8700 1 -- 172.21.15.130:0/3151514574 >> 172.21.15.130:0/3151514574 conn(0x7f471c0fb9a0 msgr2=0x7f471c0fddc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:24:22.992 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.990+0000 7f47231f8700 1 -- 172.21.15.130:0/3151514574 shutdown_connections 2024-04-25T00:24:22.992 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.990+0000 7f47231f8700 1 -- 172.21.15.130:0/3151514574 wait complete. 2024-04-25T00:24:22.992 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.990+0000 7f47231f8700 1 Processor -- start 2024-04-25T00:24:22.992 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.990+0000 7f47231f8700 1 -- start start 2024-04-25T00:24:22.992 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.990+0000 7f47231f8700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f471c1018c0 0x7f471c1985b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:22.993 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.990+0000 7f47231f8700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f471c10b160 con 0x7f471c1018c0 2024-04-25T00:24:22.993 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.991+0000 7f4720f94700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f471c1018c0 0x7f471c1985b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:22.993 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.991+0000 7f4720f94700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f471c1018c0 0x7f471c1985b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.130:36094/0 (socket says 172.21.15.130:36094) 2024-04-25T00:24:22.993 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.991+0000 7f4720f94700 1 -- 172.21.15.130:0/1622494879 learned_addr learned my addr 172.21.15.130:0/1622494879 (peer_addr_for_me v2:172.21.15.130:0/0) 2024-04-25T00:24:22.993 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.991+0000 7f4720f94700 1 -- 172.21.15.130:0/1622494879 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f471c198af0 con 0x7f471c1018c0 2024-04-25T00:24:22.993 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.991+0000 7f4720f94700 1 --2- 172.21.15.130:0/1622494879 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f471c1018c0 0x7f471c1985b0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f4710005850 tx=0x7f4710005030 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:24:22.993 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.991+0000 7f4719ffb700 1 -- 172.21.15.130:0/1622494879 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4710026070 con 0x7f471c1018c0 2024-04-25T00:24:22.993 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.991+0000 7f47231f8700 1 -- 172.21.15.130:0/1622494879 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f471c198de0 con 0x7f471c1018c0 2024-04-25T00:24:22.993 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.991+0000 7f4719ffb700 1 -- 172.21.15.130:0/1622494879 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f471000bbd0 con 0x7f471c1018c0 2024-04-25T00:24:22.993 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.991+0000 7f4719ffb700 1 -- 172.21.15.130:0/1622494879 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4710003970 con 0x7f471c1018c0 2024-04-25T00:24:22.993 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.991+0000 7f47231f8700 1 -- 172.21.15.130:0/1622494879 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f471c1992a0 con 0x7f471c1018c0 2024-04-25T00:24:22.994 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.992+0000 7f47231f8700 1 -- 172.21.15.130:0/1622494879 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f471c100170 con 0x7f471c1018c0 2024-04-25T00:24:22.994 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.992+0000 7f4719ffb700 1 -- 172.21.15.130:0/1622494879 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 13) ==== 46343+0+0 (secure 0 0 0) 0x7f4710029070 con 0x7f471c1018c0 2024-04-25T00:24:22.994 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.993+0000 7f4719ffb700 1 --2- 172.21.15.130:0/1622494879 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f470c0390f0 0x7f470c03b5b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:22.995 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.993+0000 7f4719ffb700 1 -- 172.21.15.130:0/1622494879 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(4..4 src has 1..4) ==== 1155+0+0 (secure 0 0 0) 0x7f471001e070 con 0x7f471c1018c0 2024-04-25T00:24:22.995 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.993+0000 7f471bfff700 1 --2- 172.21.15.130:0/1622494879 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f470c0390f0 0x7f470c03b5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:22.995 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.993+0000 7f471bfff700 1 --2- 172.21.15.130:0/1622494879 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f470c0390f0 0x7f470c03b5b0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f4708006fd0 tx=0x7f4708006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:24:22.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:22 smithi032 ceph-mon[29357]: Updating smithi130:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:24:22.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:22 smithi032 ceph-mon[29357]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:22.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:22 smithi032 ceph-mon[29357]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-25T00:24:22.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:22 smithi032 ceph-mon[29357]: Updating smithi130:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T00:24:22.998 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:22.998+0000 7f4719ffb700 1 -- 172.21.15.130:0/1622494879 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f4710013070 con 0x7f471c1018c0 2024-04-25T00:24:23.204 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:23.204+0000 7f47231f8700 1 -- 172.21.15.130:0/1622494879 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi032:172.21.15.32=a;smithi121:172.21.15.121=b;smithi130:172.21.15.130=c", "target": ["mon-mgr", ""]}) -- 0x7f471c100b40 con 0x7f470c0390f0 2024-04-25T00:24:23.211 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:23.210+0000 7f4719ffb700 1 -- 172.21.15.130:0/1622494879 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+24 (secure 0 0 0) 0x7f471c100b40 con 0x7f470c0390f0 2024-04-25T00:24:23.211 INFO:teuthology.orchestra.run.smithi130.stdout:Scheduled mon update... 2024-04-25T00:24:23.213 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:23.212+0000 7f47231f8700 1 -- 172.21.15.130:0/1622494879 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f470c0390f0 msgr2=0x7f470c03b5b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:23.213 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:23.212+0000 7f47231f8700 1 --2- 172.21.15.130:0/1622494879 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f470c0390f0 0x7f470c03b5b0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f4708006fd0 tx=0x7f4708006e40 comp rx=0 tx=0).stop 2024-04-25T00:24:23.214 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:23.212+0000 7f47231f8700 1 -- 172.21.15.130:0/1622494879 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f471c1018c0 msgr2=0x7f471c1985b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:23.214 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:23.212+0000 7f47231f8700 1 --2- 172.21.15.130:0/1622494879 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f471c1018c0 0x7f471c1985b0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f4710005850 tx=0x7f4710005030 comp rx=0 tx=0).stop 2024-04-25T00:24:23.214 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:23.212+0000 7f47231f8700 1 -- 172.21.15.130:0/1622494879 shutdown_connections 2024-04-25T00:24:23.214 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:23.212+0000 7f47231f8700 1 --2- 172.21.15.130:0/1622494879 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f470c0390f0 0x7f470c03b5b0 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:23.214 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:23.212+0000 7f47231f8700 1 --2- 172.21.15.130:0/1622494879 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f471c1018c0 0x7f471c1985b0 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:23.214 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:23.212+0000 7f47231f8700 1 -- 172.21.15.130:0/1622494879 >> 172.21.15.130:0/1622494879 conn(0x7f471c0fb9a0 msgr2=0x7f471c108640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:24:23.214 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:23.213+0000 7f47231f8700 1 -- 172.21.15.130:0/1622494879 shutdown_connections 2024-04-25T00:24:23.214 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:23.213+0000 7f47231f8700 1 -- 172.21.15.130:0/1622494879 wait complete. 2024-04-25T00:24:23.804 DEBUG:teuthology.orchestra.run.smithi121:mon.b> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mon.b.service 2024-04-25T00:24:23.806 DEBUG:teuthology.orchestra.run.smithi130:mon.c> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mon.c.service 2024-04-25T00:24:23.809 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-25T00:24:23.809 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-25T00:24:24.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:24 smithi032 ceph-mon[29357]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:24.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:24 smithi032 ceph-mon[29357]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi032:172.21.15.32=a;smithi121:172.21.15.121=b;smithi130:172.21.15.130=c", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:24:24.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:24 smithi032 ceph-mon[29357]: Saving service mon spec with placement smithi032:172.21.15.32=a;smithi121:172.21.15.121=b;smithi130:172.21.15.130=c;count:3 2024-04-25T00:24:24.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:24 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:24.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:24 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:24:24.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:24 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:24.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:24 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:24:24.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:24 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:24.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:24 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T00:24:24.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:24 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:24.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:24 smithi032 ceph-mon[29357]: Deploying daemon mon.c on smithi130 2024-04-25T00:24:24.842 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:24:26.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:26 smithi032 ceph-mon[29357]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:27.559 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.526+0000 7fb921294700 1 -- 172.21.15.130:0/3234153987 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb91c107c50 msgr2=0x7fb91c108030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:27.559 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.526+0000 7fb921294700 1 --2- 172.21.15.130:0/3234153987 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb91c107c50 0x7fb91c108030 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7fb90c009910 tx=0x7fb90c009c20 comp rx=0 tx=0).stop 2024-04-25T00:24:27.559 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.526+0000 7fb921294700 1 -- 172.21.15.130:0/3234153987 shutdown_connections 2024-04-25T00:24:27.559 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.526+0000 7fb921294700 1 --2- 172.21.15.130:0/3234153987 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb91c107c50 0x7fb91c108030 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:27.559 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.526+0000 7fb921294700 1 -- 172.21.15.130:0/3234153987 >> 172.21.15.130:0/3234153987 conn(0x7fb91c0780c0 msgr2=0x7fb91c0784d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:24:27.559 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.526+0000 7fb921294700 1 -- 172.21.15.130:0/3234153987 shutdown_connections 2024-04-25T00:24:27.559 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.526+0000 7fb921294700 1 -- 172.21.15.130:0/3234153987 wait complete. 2024-04-25T00:24:27.560 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.526+0000 7fb921294700 1 Processor -- start 2024-04-25T00:24:27.560 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.527+0000 7fb921294700 1 -- start start 2024-04-25T00:24:27.560 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.527+0000 7fb921294700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb91c107c50 0x7fb91c19d410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:27.560 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.527+0000 7fb921294700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fb91c10c270 con 0x7fb91c107c50 2024-04-25T00:24:27.560 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.527+0000 7fb91affd700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb91c107c50 0x7fb91c19d410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:27.560 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.527+0000 7fb91affd700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb91c107c50 0x7fb91c19d410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.130:42514/0 (socket says 172.21.15.130:42514) 2024-04-25T00:24:27.560 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.527+0000 7fb91affd700 1 -- 172.21.15.130:0/2380851424 learned_addr learned my addr 172.21.15.130:0/2380851424 (peer_addr_for_me v2:172.21.15.130:0/0) 2024-04-25T00:24:27.560 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.528+0000 7fb91affd700 1 -- 172.21.15.130:0/2380851424 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb91c19d950 con 0x7fb91c107c50 2024-04-25T00:24:27.560 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.528+0000 7fb91affd700 1 --2- 172.21.15.130:0/2380851424 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb91c107c50 0x7fb91c19d410 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7fb90c005850 tx=0x7fb90c005030 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:24:27.560 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.528+0000 7fb903fff700 1 -- 172.21.15.130:0/2380851424 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fb90c026070 con 0x7fb91c107c50 2024-04-25T00:24:27.560 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.528+0000 7fb921294700 1 -- 172.21.15.130:0/2380851424 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb91c19dc40 con 0x7fb91c107c50 2024-04-25T00:24:27.560 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.528+0000 7fb903fff700 1 -- 172.21.15.130:0/2380851424 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb90c00bbd0 con 0x7fb91c107c50 2024-04-25T00:24:27.561 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.528+0000 7fb903fff700 1 -- 172.21.15.130:0/2380851424 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fb90c003970 con 0x7fb91c107c50 2024-04-25T00:24:27.561 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.528+0000 7fb921294700 1 -- 172.21.15.130:0/2380851424 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb91c1975d0 con 0x7fb91c107c50 2024-04-25T00:24:27.561 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.529+0000 7fb921294700 1 -- 172.21.15.130:0/2380851424 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb91c069e10 con 0x7fb91c107c50 2024-04-25T00:24:27.561 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.529+0000 7fb903fff700 1 -- 172.21.15.130:0/2380851424 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 13) ==== 46343+0+0 (secure 0 0 0) 0x7fb90c029070 con 0x7fb91c107c50 2024-04-25T00:24:27.561 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.529+0000 7fb903fff700 1 --2- 172.21.15.130:0/2380851424 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb9040390f0 0x7fb90403b5b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:27.561 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.529+0000 7fb903fff700 1 -- 172.21.15.130:0/2380851424 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(4..4 src has 1..4) ==== 1155+0+0 (secure 0 0 0) 0x7fb90c01e070 con 0x7fb91c107c50 2024-04-25T00:24:27.561 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.529+0000 7fb91a7fc700 1 --2- 172.21.15.130:0/2380851424 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb9040390f0 0x7fb90403b5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:27.561 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.530+0000 7fb91a7fc700 1 --2- 172.21.15.130:0/2380851424 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb9040390f0 0x7fb90403b5b0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fb910006fd0 tx=0x7fb910006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:24:27.561 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.534+0000 7fb903fff700 1 -- 172.21.15.130:0/2380851424 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fb90c013070 con 0x7fb91c107c50 2024-04-25T00:24:27.848 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.846+0000 7fb921294700 1 -- 172.21.15.130:0/2380851424 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) -- 0x7fb91c198b00 con 0x7fb91c107c50 2024-04-25T00:24:27.848 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.847+0000 7fb903fff700 1 -- 172.21.15.130:0/2380851424 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) ==== 95+0+745 (secure 0 0 0) 0x7fb90c030020 con 0x7fb91c107c50 2024-04-25T00:24:27.848 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-25T00:24:27.848 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":1,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","modified":"2024-04-25T00:22:42.756376Z","created":"2024-04-25T00:22:42.756376Z","min_mon_release":19,"min_mon_release_name":"squid","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef","squid"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:3300","nonce":0},{"type":"v1","addr":"172.21.15.32:6789","nonce":0}]},"addr":"172.21.15.32:6789/0","public_addr":"172.21.15.32:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-25T00:24:27.851 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.850+0000 7fb921294700 1 -- 172.21.15.130:0/2380851424 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb9040390f0 msgr2=0x7fb90403b5b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:27.851 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.850+0000 7fb921294700 1 --2- 172.21.15.130:0/2380851424 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb9040390f0 0x7fb90403b5b0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fb910006fd0 tx=0x7fb910006e40 comp rx=0 tx=0).stop 2024-04-25T00:24:27.851 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.850+0000 7fb921294700 1 -- 172.21.15.130:0/2380851424 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb91c107c50 msgr2=0x7fb91c19d410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:27.851 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.850+0000 7fb921294700 1 --2- 172.21.15.130:0/2380851424 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb91c107c50 0x7fb91c19d410 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7fb90c005850 tx=0x7fb90c005030 comp rx=0 tx=0).stop 2024-04-25T00:24:27.851 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.850+0000 7fb921294700 1 -- 172.21.15.130:0/2380851424 shutdown_connections 2024-04-25T00:24:27.851 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.850+0000 7fb921294700 1 --2- 172.21.15.130:0/2380851424 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb9040390f0 0x7fb90403b5b0 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:27.851 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.850+0000 7fb921294700 1 --2- 172.21.15.130:0/2380851424 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb91c107c50 0x7fb91c19d410 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:27.851 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.850+0000 7fb921294700 1 -- 172.21.15.130:0/2380851424 >> 172.21.15.130:0/2380851424 conn(0x7fb91c0780c0 msgr2=0x7fb91c0fd8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:24:27.852 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.850+0000 7fb921294700 1 -- 172.21.15.130:0/2380851424 shutdown_connections 2024-04-25T00:24:27.852 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:27.850+0000 7fb921294700 1 -- 172.21.15.130:0/2380851424 wait complete. 2024-04-25T00:24:27.852 INFO:teuthology.orchestra.run.smithi130.stderr:dumped monmap epoch 1 2024-04-25T00:24:28.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:27 smithi130 systemd[1]: Starting Ceph mon.c for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:24:28.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:28 smithi032 ceph-mon[29357]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:28.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:28 smithi032 ceph-mon[29357]: from='client.? 172.21.15.130:0/2380851424' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-25T00:24:29.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 podman[32505]: 2024-04-25 00:24:28.72889648 +0000 UTC m=+0.104477271 container create ea82c64cab542c9ec472a26f193defca49c74812c8dcdf60cfa63494e9ee1344 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-c, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, RELEASE=squid-d2c45d9, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2) 2024-04-25T00:24:29.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 podman[32505]: 2024-04-25 00:24:28.642817206 +0000 UTC m=+0.018398014 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901 2024-04-25T00:24:29.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 podman[32505]: 2024-04-25 00:24:28.906069615 +0000 UTC m=+0.281650412 container init ea82c64cab542c9ec472a26f193defca49c74812c8dcdf60cfa63494e9ee1344 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-c, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=squid-d2c45d9, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240424) 2024-04-25T00:24:29.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 podman[32505]: 2024-04-25 00:24:28.909045801 +0000 UTC m=+0.284626587 container start ea82c64cab542c9ec472a26f193defca49c74812c8dcdf60cfa63494e9ee1344 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-c, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-d2c45d9, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-25T00:24:29.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: set uid:gid to 167:167 (ceph:ceph) 2024-04-25T00:24:29.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: ceph version 19.0.0-2445-gd2c45d97 (d2c45d977142e024a59b68a8f1dce8f0b9840551) squid (dev), process ceph-mon, pid 2 2024-04-25T00:24:29.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: pidfile_write: ignore empty --pid-file 2024-04-25T00:24:29.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: load: jerasure load: lrc 2024-04-25T00:24:29.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: RocksDB version: 7.9.2 2024-04-25T00:24:29.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Git sha 0 2024-04-25T00:24:29.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Compile date 2024-04-24 16:41:23 2024-04-25T00:24:29.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: DB SUMMARY 2024-04-25T00:24:29.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: DB Session ID: WNSGEE0W9NPYWOAYYWKL 2024-04-25T00:24:29.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: CURRENT file: CURRENT 2024-04-25T00:24:29.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: IDENTITY file: IDENTITY 2024-04-25T00:24:29.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: MANIFEST file: MANIFEST-000005 size: 59 Bytes 2024-04-25T00:24:29.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-04-25T00:24:29.144 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 511 ; 2024-04-25T00:24:29.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.error_if_exists: 0 2024-04-25T00:24:29.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.create_if_missing: 0 2024-04-25T00:24:29.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.paranoid_checks: 1 2024-04-25T00:24:29.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-25T00:24:29.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-25T00:24:29.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-25T00:24:29.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.env: 0x557b9fd50c60 2024-04-25T00:24:29.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.fs: PosixFileSystem 2024-04-25T00:24:29.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.info_log: 0x557ba224daa0 2024-04-25T00:24:29.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_file_opening_threads: 16 2024-04-25T00:24:29.147 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.statistics: (nil) 2024-04-25T00:24:29.147 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.use_fsync: 0 2024-04-25T00:24:29.147 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_log_file_size: 0 2024-04-25T00:24:29.147 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-25T00:24:29.147 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-25T00:24:29.147 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.keep_log_file_num: 1000 2024-04-25T00:24:29.147 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.recycle_log_file_num: 0 2024-04-25T00:24:29.147 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.allow_fallocate: 1 2024-04-25T00:24:29.147 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.allow_mmap_reads: 0 2024-04-25T00:24:29.147 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.allow_mmap_writes: 0 2024-04-25T00:24:29.147 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.use_direct_reads: 0 2024-04-25T00:24:29.147 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-25T00:24:29.148 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.create_missing_column_families: 0 2024-04-25T00:24:29.148 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.db_log_dir: 2024-04-25T00:24:29.148 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.wal_dir: 2024-04-25T00:24:29.148 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-25T00:24:29.148 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-25T00:24:29.148 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-25T00:24:29.148 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-25T00:24:29.148 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-25T00:24:29.148 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-25T00:24:29.148 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.advise_random_on_open: 1 2024-04-25T00:24:29.148 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.db_write_buffer_size: 0 2024-04-25T00:24:29.148 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.write_buffer_manager: 0x557ba22201e0 2024-04-25T00:24:29.149 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-25T00:24:29.149 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-25T00:24:29.149 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-25T00:24:29.149 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.rate_limiter: (nil) 2024-04-25T00:24:29.149 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-25T00:24:29.149 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.wal_recovery_mode: 2 2024-04-25T00:24:29.149 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.enable_thread_tracking: 0 2024-04-25T00:24:29.149 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.enable_pipelined_write: 0 2024-04-25T00:24:29.149 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.unordered_write: 0 2024-04-25T00:24:29.149 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-25T00:24:29.149 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-25T00:24:29.149 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-25T00:24:29.150 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-25T00:24:29.150 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.row_cache: None 2024-04-25T00:24:29.150 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.wal_filter: None 2024-04-25T00:24:29.150 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-25T00:24:29.150 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.allow_ingest_behind: 0 2024-04-25T00:24:29.150 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.two_write_queues: 0 2024-04-25T00:24:29.150 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.manual_wal_flush: 0 2024-04-25T00:24:29.150 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.wal_compression: 0 2024-04-25T00:24:29.150 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.atomic_flush: 0 2024-04-25T00:24:29.150 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-25T00:24:29.150 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-25T00:24:29.150 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-25T00:24:29.151 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.log_readahead_size: 0 2024-04-25T00:24:29.151 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-25T00:24:29.151 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.best_efforts_recovery: 0 2024-04-25T00:24:29.151 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-25T00:24:29.151 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-25T00:24:29.151 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.allow_data_in_errors: 0 2024-04-25T00:24:29.151 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.db_host_id: __hostname__ 2024-04-25T00:24:29.151 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-25T00:24:29.151 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_background_jobs: 2 2024-04-25T00:24:29.151 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_background_compactions: -1 2024-04-25T00:24:29.151 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_subcompactions: 1 2024-04-25T00:24:29.151 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-25T00:24:29.152 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-25T00:24:29.152 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-25T00:24:29.152 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_total_wal_size: 0 2024-04-25T00:24:29.152 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-25T00:24:29.152 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-25T00:24:29.152 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-25T00:24:29.152 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-25T00:24:29.152 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_open_files: -1 2024-04-25T00:24:29.152 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.bytes_per_sync: 0 2024-04-25T00:24:29.152 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-25T00:24:29.152 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-25T00:24:29.152 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.compaction_readahead_size: 0 2024-04-25T00:24:29.153 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_background_flushes: -1 2024-04-25T00:24:29.153 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Compression algorithms supported: 2024-04-25T00:24:29.153 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-25T00:24:29.153 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: kZSTD supported: 0 2024-04-25T00:24:29.153 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: kXpressCompression supported: 0 2024-04-25T00:24:29.153 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: kLZ4HCCompression supported: 1 2024-04-25T00:24:29.153 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: kZlibCompression supported: 1 2024-04-25T00:24:29.153 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: kSnappyCompression supported: 1 2024-04-25T00:24:29.153 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: kLZ4Compression supported: 1 2024-04-25T00:24:29.153 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: kBZip2Compression supported: 0 2024-04-25T00:24:29.153 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-25T00:24:29.154 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-25T00:24:29.154 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2024-04-25T00:24:29.154 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-25T00:24:29.154 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-25T00:24:29.154 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.merge_operator: 2024-04-25T00:24:29.154 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.compaction_filter: None 2024-04-25T00:24:29.154 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.compaction_filter_factory: None 2024-04-25T00:24:29.154 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.sst_partitioner_factory: None 2024-04-25T00:24:29.154 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-25T00:24:29.154 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-25T00:24:29.154 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x557ba224dba0) 2024-04-25T00:24:29.154 INFO:journalctl@ceph.mon.c.smithi130.stdout: cache_index_and_filter_blocks: 1 2024-04-25T00:24:29.155 INFO:journalctl@ceph.mon.c.smithi130.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-25T00:24:29.155 INFO:journalctl@ceph.mon.c.smithi130.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-25T00:24:29.155 INFO:journalctl@ceph.mon.c.smithi130.stdout: pin_top_level_index_and_filter: 1 2024-04-25T00:24:29.155 INFO:journalctl@ceph.mon.c.smithi130.stdout: index_type: 0 2024-04-25T00:24:29.155 INFO:journalctl@ceph.mon.c.smithi130.stdout: data_block_index_type: 0 2024-04-25T00:24:29.155 INFO:journalctl@ceph.mon.c.smithi130.stdout: index_shortening: 1 2024-04-25T00:24:29.155 INFO:journalctl@ceph.mon.c.smithi130.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-25T00:24:29.155 INFO:journalctl@ceph.mon.c.smithi130.stdout: checksum: 4 2024-04-25T00:24:29.155 INFO:journalctl@ceph.mon.c.smithi130.stdout: no_block_cache: 0 2024-04-25T00:24:29.155 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_cache: 0x557ba2251090 2024-04-25T00:24:29.155 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_cache_name: BinnedLRUCache 2024-04-25T00:24:29.155 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_cache_options: 2024-04-25T00:24:29.156 INFO:journalctl@ceph.mon.c.smithi130.stdout: capacity : 536870912 2024-04-25T00:24:29.156 INFO:journalctl@ceph.mon.c.smithi130.stdout: num_shard_bits : 4 2024-04-25T00:24:29.156 INFO:journalctl@ceph.mon.c.smithi130.stdout: strict_capacity_limit : 0 2024-04-25T00:24:29.156 INFO:journalctl@ceph.mon.c.smithi130.stdout: high_pri_pool_ratio: 0.000 2024-04-25T00:24:29.156 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_cache_compressed: (nil) 2024-04-25T00:24:29.156 INFO:journalctl@ceph.mon.c.smithi130.stdout: persistent_cache: (nil) 2024-04-25T00:24:29.156 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_size: 4096 2024-04-25T00:24:29.156 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_size_deviation: 10 2024-04-25T00:24:29.156 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_restart_interval: 16 2024-04-25T00:24:29.156 INFO:journalctl@ceph.mon.c.smithi130.stdout: index_block_restart_interval: 1 2024-04-25T00:24:29.156 INFO:journalctl@ceph.mon.c.smithi130.stdout: metadata_block_size: 4096 2024-04-25T00:24:29.156 INFO:journalctl@ceph.mon.c.smithi130.stdout: partition_filters: 0 2024-04-25T00:24:29.156 INFO:journalctl@ceph.mon.c.smithi130.stdout: use_delta_encoding: 1 2024-04-25T00:24:29.157 INFO:journalctl@ceph.mon.c.smithi130.stdout: filter_policy: bloomfilter 2024-04-25T00:24:29.157 INFO:journalctl@ceph.mon.c.smithi130.stdout: whole_key_filtering: 1 2024-04-25T00:24:29.157 INFO:journalctl@ceph.mon.c.smithi130.stdout: verify_compression: 0 2024-04-25T00:24:29.157 INFO:journalctl@ceph.mon.c.smithi130.stdout: read_amp_bytes_per_bit: 0 2024-04-25T00:24:29.157 INFO:journalctl@ceph.mon.c.smithi130.stdout: format_version: 5 2024-04-25T00:24:29.157 INFO:journalctl@ceph.mon.c.smithi130.stdout: enable_index_compression: 1 2024-04-25T00:24:29.157 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_align: 0 2024-04-25T00:24:29.157 INFO:journalctl@ceph.mon.c.smithi130.stdout: max_auto_readahead_size: 262144 2024-04-25T00:24:29.157 INFO:journalctl@ceph.mon.c.smithi130.stdout: prepopulate_block_cache: 0 2024-04-25T00:24:29.157 INFO:journalctl@ceph.mon.c.smithi130.stdout: initial_auto_readahead_size: 8192 2024-04-25T00:24:29.157 INFO:journalctl@ceph.mon.c.smithi130.stdout: num_file_reads_for_auto_readahead: 2 2024-04-25T00:24:29.157 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.write_buffer_size: 33554432 2024-04-25T00:24:29.158 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_write_buffer_number: 2 2024-04-25T00:24:29.158 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.compression: NoCompression 2024-04-25T00:24:29.158 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.bottommost_compression: Disabled 2024-04-25T00:24:29.158 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.prefix_extractor: nullptr 2024-04-25T00:24:29.158 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-25T00:24:29.158 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.num_levels: 7 2024-04-25T00:24:29.158 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-25T00:24:29.158 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-25T00:24:29.158 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-25T00:24:29.158 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-25T00:24:29.158 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-25T00:24:29.158 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-25T00:24:29.159 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-25T00:24:29.159 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-25T00:24:29.159 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-25T00:24:29.159 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-25T00:24:29.159 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-25T00:24:29.159 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-25T00:24:29.159 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-25T00:24:29.159 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.compression_opts.level: 32767 2024-04-25T00:24:29.159 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.compression_opts.strategy: 0 2024-04-25T00:24:29.159 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-25T00:24:29.159 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-25T00:24:29.159 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-25T00:24:29.159 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-25T00:24:29.160 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.compression_opts.enabled: false 2024-04-25T00:24:29.160 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-25T00:24:29.160 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-25T00:24:29.160 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-25T00:24:29.160 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-25T00:24:29.160 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.target_file_size_base: 67108864 2024-04-25T00:24:29.160 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-25T00:24:29.160 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-25T00:24:29.160 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-25T00:24:29.160 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-25T00:24:29.160 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-25T00:24:29.160 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-25T00:24:29.160 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-25T00:24:29.161 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-25T00:24:29.161 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-25T00:24:29.161 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-25T00:24:29.161 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-25T00:24:29.161 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-25T00:24:29.161 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-25T00:24:29.161 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-25T00:24:29.161 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.arena_block_size: 1048576 2024-04-25T00:24:29.161 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-25T00:24:29.161 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-25T00:24:29.161 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.disable_auto_compactions: 0 2024-04-25T00:24:29.161 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-25T00:24:29.161 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-25T00:24:29.162 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-25T00:24:29.162 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-25T00:24:29.162 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-25T00:24:29.162 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-25T00:24:29.162 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-25T00:24:29.162 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-25T00:24:29.162 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-25T00:24:29.162 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-25T00:24:29.162 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.table_properties_collectors: 2024-04-25T00:24:29.162 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.inplace_update_support: 0 2024-04-25T00:24:29.162 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-25T00:24:29.162 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-25T00:24:29.163 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-25T00:24:29.163 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-25T00:24:29.163 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.bloom_locality: 0 2024-04-25T00:24:29.163 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.max_successive_merges: 0 2024-04-25T00:24:29.163 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-25T00:24:29.163 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.paranoid_file_checks: 0 2024-04-25T00:24:29.163 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.force_consistency_checks: 1 2024-04-25T00:24:29.163 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.report_bg_io_stats: 0 2024-04-25T00:24:29.163 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.ttl: 2592000 2024-04-25T00:24:29.163 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-25T00:24:29.163 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-25T00:24:29.164 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-25T00:24:29.164 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.enable_blob_files: false 2024-04-25T00:24:29.164 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.min_blob_size: 0 2024-04-25T00:24:29.164 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.blob_file_size: 268435456 2024-04-25T00:24:29.164 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-25T00:24:29.164 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-25T00:24:29.164 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-25T00:24:29.164 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-25T00:24:29.164 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-25T00:24:29.164 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.blob_file_starting_level: 0 2024-04-25T00:24:29.164 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-25T00:24:29.164 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-25T00:24:29.164 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2024-04-25T00:24:29.164 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: d7630c3f-c2d2-4e30-8012-6d47c675b6b2 2024-04-25T00:24:29.165 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714004668949034, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-25T00:24:29.165 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #4 mode 2 2024-04-25T00:24:29.165 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714004668949447, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1622, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714004668, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "d7630c3f-c2d2-4e30-8012-6d47c675b6b2", "db_session_id": "WNSGEE0W9NPYWOAYYWKL", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2024-04-25T00:24:29.165 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714004668949527, "job": 1, "event": "recovery_finished"} 2024-04-25T00:24:29.165 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: [db/version_set.cc:5047] Creating manifest 10 2024-04-25T00:24:29.165 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-25T00:24:29.165 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x557ba2342000 2024-04-25T00:24:29.165 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: DB pointer 0x557ba232c000 2024-04-25T00:24:29.165 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-25T00:24:29.165 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-25T00:24:29.165 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** DB Stats ** 2024-04-25T00:24:29.165 INFO:journalctl@ceph.mon.c.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-25T00:24:29.165 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-25T00:24:29.166 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-25T00:24:29.166 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-25T00:24:29.166 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-25T00:24:29.166 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-25T00:24:29.166 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-25T00:24:29.166 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-04-25T00:24:29.166 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** Compaction Stats [default] ** 2024-04-25T00:24:29.166 INFO:journalctl@ceph.mon.c.smithi130.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-25T00:24:29.166 INFO:journalctl@ceph.mon.c.smithi130.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-25T00:24:29.166 INFO:journalctl@ceph.mon.c.smithi130.stdout: L0 1/0 1.58 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-04-25T00:24:29.166 INFO:journalctl@ceph.mon.c.smithi130.stdout: Sum 1/0 1.58 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-04-25T00:24:29.166 INFO:journalctl@ceph.mon.c.smithi130.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-04-25T00:24:29.167 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-04-25T00:24:29.167 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** Compaction Stats [default] ** 2024-04-25T00:24:29.167 INFO:journalctl@ceph.mon.c.smithi130.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-25T00:24:29.167 INFO:journalctl@ceph.mon.c.smithi130.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-25T00:24:29.167 INFO:journalctl@ceph.mon.c.smithi130.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.0 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-04-25T00:24:29.167 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-04-25T00:24:29.167 INFO:journalctl@ceph.mon.c.smithi130.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-25T00:24:29.167 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-04-25T00:24:29.167 INFO:journalctl@ceph.mon.c.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-25T00:24:29.167 INFO:journalctl@ceph.mon.c.smithi130.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-25T00:24:29.167 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-25T00:24:29.167 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-25T00:24:29.167 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-25T00:24:29.167 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-25T00:24:29.168 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative compaction: 0.00 GB write, 0.12 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-25T00:24:29.168 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval compaction: 0.00 GB write, 0.12 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-25T00:24:29.168 INFO:journalctl@ceph.mon.c.smithi130.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-25T00:24:29.168 INFO:journalctl@ceph.mon.c.smithi130.stdout: Block cache BinnedLRUCache@0x557ba2251090#2 capacity: 512.00 MB usage: 0.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 8e-06 secs_since: 0 2024-04-25T00:24:29.168 INFO:journalctl@ceph.mon.c.smithi130.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2024-04-25T00:24:29.168 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-04-25T00:24:29.168 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-25T00:24:29.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-04-25T00:24:29.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: using public_addr v2:172.21.15.130:0/0 -> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] 2024-04-25T00:24:29.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: starting mon.c rank -1 at public addrs [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] at bind addrs [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon_data /var/lib/ceph/mon/ceph-c fsid cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:24:29.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: mon.c@-1(???) e0 preinit fsid cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:24:29.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: mon.c@-1(synchronizing).mds e1 new map 2024-04-25T00:24:29.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: mon.c@-1(synchronizing).mds e1 print_map 2024-04-25T00:24:29.168 INFO:journalctl@ceph.mon.c.smithi130.stdout: e1 2024-04-25T00:24:29.169 INFO:journalctl@ceph.mon.c.smithi130.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-25T00:24:29.169 INFO:journalctl@ceph.mon.c.smithi130.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2,11=minor log segments,12=quiesce subvolumes} 2024-04-25T00:24:29.169 INFO:journalctl@ceph.mon.c.smithi130.stdout: legacy client fscid: -1 2024-04-25T00:24:29.169 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-04-25T00:24:29.169 INFO:journalctl@ceph.mon.c.smithi130.stdout: No filesystems configured 2024-04-25T00:24:29.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-25T00:24:29.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-25T00:24:29.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-25T00:24:29.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-25T00:24:29.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-25T00:24:29.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-25T00:24:29.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-25T00:24:29.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-25T00:24:29.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-25T00:24:29.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-25T00:24:29.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:29.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi130", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:24:29.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:29.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:29.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:29.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:29.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:24:29.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:29.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:24:29.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:29.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: Updating smithi121:/etc/ceph/ceph.conf 2024-04-25T00:24:29.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: Deploying cephadm binary to smithi130 2024-04-25T00:24:29.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: Updating smithi121:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:24:29.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-04-25T00:24:29.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:29.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: Updating smithi121:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T00:24:29.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:29.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:29.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:29.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:29.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:24:29.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:29.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:29.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: Added host smithi130 2024-04-25T00:24:29.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:29.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:29.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:24:29.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:29.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:29.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/4182769857' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-25T00:24:29.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/4182769857' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-25T00:24:29.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: osdmap e4: 0 total, 0 up, 0 in 2024-04-25T00:24:29.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:29.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:29.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:29.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:29.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi130", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:24:29.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:29.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:24:29.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-25T00:24:29.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:29.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:29.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:29.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: Updating smithi130:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:24:29.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:29.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-25T00:24:29.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: Updating smithi130:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T00:24:29.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:29.174 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi032:172.21.15.32=a;smithi121:172.21.15.121=b;smithi130:172.21.15.130=c", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:24:29.174 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: Saving service mon spec with placement smithi032:172.21.15.32=a;smithi121:172.21.15.121=b;smithi130:172.21.15.130=c;count:3 2024-04-25T00:24:29.174 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:29.174 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:24:29.174 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:29.174 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:24:29.174 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:29.174 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T00:24:29.174 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:29.174 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: Deploying daemon mon.c on smithi130 2024-04-25T00:24:29.174 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:29.174 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:29.174 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: from='client.? 172.21.15.130:0/2380851424' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-25T00:24:29.175 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 ceph-mon[32525]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-25T00:24:29.175 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 bash[32505]: ea82c64cab542c9ec472a26f193defca49c74812c8dcdf60cfa63494e9ee1344 2024-04-25T00:24:29.175 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:28 smithi130 systemd[1]: Started Ceph mon.c for cb997156-0299-11ef-bc93-c7b262605968. 2024-04-25T00:24:29.505 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-25T00:24:29.506 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-25T00:24:29.739 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.c/config 2024-04-25T00:24:34.158 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.156+0000 7f725f959700 1 -- 172.21.15.130:0/504421195 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f72440035c0 msgr2=0x7f7244005a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:34.158 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.156+0000 7f725f959700 1 --2- 172.21.15.130:0/504421195 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f72440035c0 0x7f7244005a50 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f7258100b70 tx=0x7f725400feb0 comp rx=0 tx=0).stop 2024-04-25T00:24:34.158 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.156+0000 7f725f959700 1 -- 172.21.15.130:0/504421195 shutdown_connections 2024-04-25T00:24:34.158 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.156+0000 7f725f959700 1 --2- 172.21.15.130:0/504421195 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f72440035c0 0x7f7244005a50 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:34.158 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.156+0000 7f725f959700 1 -- 172.21.15.130:0/504421195 >> 172.21.15.130:0/504421195 conn(0x7f72580fc770 msgr2=0x7f72580feb90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:24:34.159 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.156+0000 7f725f959700 1 -- 172.21.15.130:0/504421195 shutdown_connections 2024-04-25T00:24:34.159 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.156+0000 7f725f959700 1 -- 172.21.15.130:0/504421195 wait complete. 2024-04-25T00:24:34.159 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.157+0000 7f725f959700 1 Processor -- start 2024-04-25T00:24:34.159 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.157+0000 7f725f959700 1 -- start start 2024-04-25T00:24:34.159 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.157+0000 7f725f959700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f72440035c0 0x7f725819d660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:34.159 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.157+0000 7f725f959700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f725819dba0 0x7f7258197730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:34.159 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.157+0000 7f725f959700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f7258108cd0 con 0x7f725819dba0 2024-04-25T00:24:34.159 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.157+0000 7f725f959700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f7258108b50 con 0x7f72440035c0 2024-04-25T00:24:34.159 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.157+0000 7f725d6f5700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f72440035c0 0x7f725819d660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:34.159 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.158+0000 7f725d6f5700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f72440035c0 0x7f725819d660 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.130:38732/0 (socket says 172.21.15.130:38732) 2024-04-25T00:24:34.159 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.158+0000 7f725d6f5700 1 -- 172.21.15.130:0/2129239003 learned_addr learned my addr 172.21.15.130:0/2129239003 (peer_addr_for_me v2:172.21.15.130:0/0) 2024-04-25T00:24:34.159 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.158+0000 7f725d6f5700 1 -- 172.21.15.130:0/2129239003 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f72440035c0 msgr2=0x7f725819d660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 12 2024-04-25T00:24:34.159 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.158+0000 7f725d6f5700 1 -- 172.21.15.130:0/2129239003 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f72440035c0 msgr2=0x7f725819d660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2024-04-25T00:24:34.159 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.158+0000 7f725d6f5700 1 --2- 172.21.15.130:0/2129239003 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f72440035c0 0x7f725819d660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-04-25T00:24:34.159 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.158+0000 7f725cef4700 1 --2- 172.21.15.130:0/2129239003 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f725819dba0 0x7f7258197730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:34.160 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.158+0000 7f725d6f5700 1 --2- 172.21.15.130:0/2129239003 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f72440035c0 0x7f725819d660 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-25T00:24:34.161 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.158+0000 7f725cef4700 1 -- 172.21.15.130:0/2129239003 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f72440035c0 msgr2=0x7f725819d660 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:24:34.161 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.158+0000 7f725cef4700 1 --2- 172.21.15.130:0/2129239003 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f72440035c0 0x7f725819d660 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:34.161 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.158+0000 7f725cef4700 1 -- 172.21.15.130:0/2129239003 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f725819e120 con 0x7f725819dba0 2024-04-25T00:24:34.161 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.158+0000 7f725cef4700 1 --2- 172.21.15.130:0/2129239003 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f725819dba0 0x7f7258197730 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f724800b4d0 tx=0x7f724800b890 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:24:34.161 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.158+0000 7f724e7fc700 1 -- 172.21.15.130:0/2129239003 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 309+0+0 (secure 0 0 0) 0x7f7248011630 con 0x7f725819dba0 2024-04-25T00:24:34.161 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.158+0000 7f725f959700 1 -- 172.21.15.130:0/2129239003 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7258197cd0 con 0x7f725819dba0 2024-04-25T00:24:34.162 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.159+0000 7f725f959700 1 -- 172.21.15.130:0/2129239003 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7258198260 con 0x7f725819dba0 2024-04-25T00:24:34.162 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.159+0000 7f724e7fc700 1 -- 172.21.15.130:0/2129239003 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7248011cb0 con 0x7f725819dba0 2024-04-25T00:24:34.162 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.159+0000 7f724e7fc700 1 -- 172.21.15.130:0/2129239003 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 309+0+0 (secure 0 0 0) 0x7f72480193b0 con 0x7f725819dba0 2024-04-25T00:24:34.162 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.160+0000 7f724e7fc700 1 -- 172.21.15.130:0/2129239003 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 13) ==== 46343+0+0 (secure 0 0 0) 0x7f7248018020 con 0x7f725819dba0 2024-04-25T00:24:34.162 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.160+0000 7f725f959700 1 -- 172.21.15.130:0/2129239003 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7258102380 con 0x7f725819dba0 2024-04-25T00:24:34.162 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.160+0000 7f724e7fc700 1 --2- 172.21.15.130:0/2129239003 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f723c038e90 0x7f723c03b350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:34.162 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.160+0000 7f724e7fc700 1 -- 172.21.15.130:0/2129239003 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(4..4 src has 1..4) ==== 1155+0+0 (secure 0 0 0) 0x7f724804be10 con 0x7f725819dba0 2024-04-25T00:24:34.162 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.160+0000 7f725d6f5700 1 --2- 172.21.15.130:0/2129239003 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f723c038e90 0x7f723c03b350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:34.162 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.161+0000 7f725d6f5700 1 --2- 172.21.15.130:0/2129239003 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f723c038e90 0x7f723c03b350 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f725400fa60 tx=0x7f7254005c20 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:24:34.166 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.166+0000 7f724e7fc700 1 -- 172.21.15.130:0/2129239003 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f724801f020 con 0x7f725819dba0 2024-04-25T00:24:34.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:34.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-25T00:24:34.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: mon.a calling monitor election 2024-04-25T00:24:34.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:24:34.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:24:34.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:34.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: mon.c calling monitor election 2024-04-25T00:24:34.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:24:34.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:24:34.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:34.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:24:34.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-25T00:24:34.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: monmap epoch 2 2024-04-25T00:24:34.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: fsid cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:24:34.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: last_changed 2024-04-25T00:24:28.972598+0000 2024-04-25T00:24:34.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: created 2024-04-25T00:22:42.756376+0000 2024-04-25T00:24:34.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: min_mon_release 19 (squid) 2024-04-25T00:24:34.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: election_strategy: 1 2024-04-25T00:24:34.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: 0: [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] mon.a 2024-04-25T00:24:34.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: 1: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.c 2024-04-25T00:24:34.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: fsmap 2024-04-25T00:24:34.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: osdmap e4: 0 total, 0 up, 0 in 2024-04-25T00:24:34.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: mgrmap e13: x(active, since 51s) 2024-04-25T00:24:34.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: overall HEALTH_OK 2024-04-25T00:24:34.394 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:34.394 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:24:34.394 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:34.394 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:34.394 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T00:24:34.394 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:34.447 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.446+0000 7f725f959700 1 -- 172.21.15.130:0/2129239003 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) -- 0x7f72581985d0 con 0x7f725819dba0 2024-04-25T00:24:34.447 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.447+0000 7f724e7fc700 1 -- 172.21.15.130:0/2129239003 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 2 v2) ==== 95+0+1015 (secure 0 0 0) 0x7f7248017ae0 con 0x7f725819dba0 2024-04-25T00:24:34.447 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-25T00:24:34.448 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":2,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","modified":"2024-04-25T00:24:28.972598Z","created":"2024-04-25T00:22:42.756376Z","min_mon_release":19,"min_mon_release_name":"squid","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef","squid"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:3300","nonce":0},{"type":"v1","addr":"172.21.15.32:6789","nonce":0}]},"addr":"172.21.15.32:6789/0","public_addr":"172.21.15.32:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3300","nonce":0},{"type":"v1","addr":"172.21.15.130:6789","nonce":0}]},"addr":"172.21.15.130:6789/0","public_addr":"172.21.15.130:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-25T00:24:34.450 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.449+0000 7f725f959700 1 -- 172.21.15.130:0/2129239003 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f723c038e90 msgr2=0x7f723c03b350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:34.450 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.449+0000 7f725f959700 1 --2- 172.21.15.130:0/2129239003 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f723c038e90 0x7f723c03b350 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f725400fa60 tx=0x7f7254005c20 comp rx=0 tx=0).stop 2024-04-25T00:24:34.450 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.449+0000 7f725f959700 1 -- 172.21.15.130:0/2129239003 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f725819dba0 msgr2=0x7f7258197730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:34.450 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.449+0000 7f725f959700 1 --2- 172.21.15.130:0/2129239003 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f725819dba0 0x7f7258197730 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f724800b4d0 tx=0x7f724800b890 comp rx=0 tx=0).stop 2024-04-25T00:24:34.450 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.449+0000 7f725f959700 1 -- 172.21.15.130:0/2129239003 shutdown_connections 2024-04-25T00:24:34.450 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.449+0000 7f725f959700 1 --2- 172.21.15.130:0/2129239003 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f723c038e90 0x7f723c03b350 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:34.450 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.449+0000 7f725f959700 1 --2- 172.21.15.130:0/2129239003 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f72440035c0 0x7f725819d660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:34.450 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.449+0000 7f725f959700 1 --2- 172.21.15.130:0/2129239003 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f725819dba0 0x7f7258197730 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:34.450 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.449+0000 7f725f959700 1 -- 172.21.15.130:0/2129239003 >> 172.21.15.130:0/2129239003 conn(0x7f72580fc770 msgr2=0x7f725806acd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:24:34.451 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.449+0000 7f725f959700 1 -- 172.21.15.130:0/2129239003 shutdown_connections 2024-04-25T00:24:34.451 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:34.449+0000 7f725f959700 1 -- 172.21.15.130:0/2129239003 wait complete. 2024-04-25T00:24:34.451 INFO:teuthology.orchestra.run.smithi130.stderr:dumped monmap epoch 2 2024-04-25T00:24:34.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:34.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-25T00:24:34.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: mon.a calling monitor election 2024-04-25T00:24:34.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:24:34.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:24:34.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:34.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: mon.c calling monitor election 2024-04-25T00:24:34.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:24:34.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:24:34.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:34.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:24:34.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-25T00:24:34.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: monmap epoch 2 2024-04-25T00:24:34.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: fsid cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:24:34.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: last_changed 2024-04-25T00:24:28.972598+0000 2024-04-25T00:24:34.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: created 2024-04-25T00:22:42.756376+0000 2024-04-25T00:24:34.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: min_mon_release 19 (squid) 2024-04-25T00:24:34.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: election_strategy: 1 2024-04-25T00:24:34.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: 0: [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] mon.a 2024-04-25T00:24:34.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: 1: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.c 2024-04-25T00:24:34.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: fsmap 2024-04-25T00:24:34.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: osdmap e4: 0 total, 0 up, 0 in 2024-04-25T00:24:34.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: mgrmap e13: x(active, since 51s) 2024-04-25T00:24:34.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: overall HEALTH_OK 2024-04-25T00:24:34.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:34.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:24:34.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:34.501 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:34.501 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T00:24:34.501 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:35.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:35 smithi032 ceph-mon[29357]: Deploying daemon mon.b on smithi121 2024-04-25T00:24:35.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:35 smithi032 ceph-mon[29357]: from='client.? 172.21.15.130:0/2129239003' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-25T00:24:35.249 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:24:34 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:24:34.982+0000 7f9761d53700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-04-25T00:24:35.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:35 smithi130 ceph-mon[32525]: Deploying daemon mon.b on smithi121 2024-04-25T00:24:35.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:35 smithi130 ceph-mon[32525]: from='client.? 172.21.15.130:0/2129239003' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-25T00:24:35.988 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-25T00:24:35.988 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-25T00:24:36.231 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.c/config 2024-04-25T00:24:36.345 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:36 smithi130 ceph-mon[32525]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:36.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:36 smithi032 ceph-mon[29357]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:37.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 podman[32211]: 2024-04-25 00:24:37.460696522 +0000 UTC m=+0.022723444 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901 2024-04-25T00:24:37.636 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.633+0000 7f8cda421700 1 -- 172.21.15.130:0/3829586056 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8cd4102ee0 msgr2=0x7f8cd41032c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:37.636 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.633+0000 7f8cda421700 1 --2- 172.21.15.130:0/3829586056 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8cd4102ee0 0x7f8cd41032c0 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f8cc4009910 tx=0x7f8cc4009c20 comp rx=0 tx=0).stop 2024-04-25T00:24:37.636 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.633+0000 7f8cda421700 1 -- 172.21.15.130:0/3829586056 shutdown_connections 2024-04-25T00:24:37.636 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.633+0000 7f8cda421700 1 --2- 172.21.15.130:0/3829586056 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8cd4102ee0 0x7f8cd41032c0 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:37.636 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.633+0000 7f8cda421700 1 -- 172.21.15.130:0/3829586056 >> 172.21.15.130:0/3829586056 conn(0x7f8cd40fc710 msgr2=0x7f8cd40feb30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:24:37.636 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.633+0000 7f8cda421700 1 -- 172.21.15.130:0/3829586056 shutdown_connections 2024-04-25T00:24:37.636 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.633+0000 7f8cda421700 1 -- 172.21.15.130:0/3829586056 wait complete. 2024-04-25T00:24:37.636 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.634+0000 7f8cda421700 1 Processor -- start 2024-04-25T00:24:37.636 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.634+0000 7f8cda421700 1 -- start start 2024-04-25T00:24:37.637 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.634+0000 7f8cda421700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8cd4102ee0 0x7f8cd419d6f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:37.637 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.634+0000 7f8cda421700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8cd419dc30 0x7f8cd41978b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:37.637 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.634+0000 7f8cda421700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f8cd410c3d0 con 0x7f8cd419dc30 2024-04-25T00:24:37.637 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.634+0000 7f8cda421700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f8cd410c250 con 0x7f8cd4102ee0 2024-04-25T00:24:37.637 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.634+0000 7f8cd3fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8cd4102ee0 0x7f8cd419d6f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:37.637 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.634+0000 7f8cd3fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8cd4102ee0 0x7f8cd419d6f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.130:38738/0 (socket says 172.21.15.130:38738) 2024-04-25T00:24:37.637 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.634+0000 7f8cd3fff700 1 -- 172.21.15.130:0/3495676906 learned_addr learned my addr 172.21.15.130:0/3495676906 (peer_addr_for_me v2:172.21.15.130:0/0) 2024-04-25T00:24:37.637 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.634+0000 7f8cd3fff700 1 -- 172.21.15.130:0/3495676906 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8cd4102ee0 msgr2=0x7f8cd419d6f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 12 2024-04-25T00:24:37.637 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.634+0000 7f8cd3fff700 1 -- 172.21.15.130:0/3495676906 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8cd4102ee0 msgr2=0x7f8cd419d6f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2024-04-25T00:24:37.637 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.634+0000 7f8cd3fff700 1 --2- 172.21.15.130:0/3495676906 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8cd4102ee0 0x7f8cd419d6f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-04-25T00:24:37.637 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.634+0000 7f8cd3fff700 1 --2- 172.21.15.130:0/3495676906 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8cd4102ee0 0x7f8cd419d6f0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-25T00:24:37.637 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.634+0000 7f8cd37fe700 1 --2- 172.21.15.130:0/3495676906 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8cd419dc30 0x7f8cd41978b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:37.637 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.635+0000 7f8cd37fe700 1 -- 172.21.15.130:0/3495676906 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8cd4102ee0 msgr2=0x7f8cd419d6f0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:24:37.638 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.635+0000 7f8cd37fe700 1 --2- 172.21.15.130:0/3495676906 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8cd4102ee0 0x7f8cd419d6f0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:37.638 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.635+0000 7f8cd37fe700 1 -- 172.21.15.130:0/3495676906 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8cd419e270 con 0x7f8cd419dc30 2024-04-25T00:24:37.638 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.635+0000 7f8cd37fe700 1 --2- 172.21.15.130:0/3495676906 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8cd419dc30 0x7f8cd41978b0 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f8cc800b4d0 tx=0x7f8cc800b890 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:24:37.638 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.635+0000 7f8cd17fa700 1 -- 172.21.15.130:0/3495676906 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 309+0+0 (secure 0 0 0) 0x7f8cc8011630 con 0x7f8cd419dc30 2024-04-25T00:24:37.638 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.635+0000 7f8cda421700 1 -- 172.21.15.130:0/3495676906 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8cd4197df0 con 0x7f8cd419dc30 2024-04-25T00:24:37.638 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.635+0000 7f8cd17fa700 1 -- 172.21.15.130:0/3495676906 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8cc8011cb0 con 0x7f8cd419dc30 2024-04-25T00:24:37.638 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.635+0000 7f8cd17fa700 1 -- 172.21.15.130:0/3495676906 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 309+0+0 (secure 0 0 0) 0x7f8cc8010e60 con 0x7f8cd419dc30 2024-04-25T00:24:37.638 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.635+0000 7f8cda421700 1 -- 172.21.15.130:0/3495676906 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8cd4198380 con 0x7f8cd419dc30 2024-04-25T00:24:37.639 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.636+0000 7f8cd17fa700 1 -- 172.21.15.130:0/3495676906 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 13) ==== 46343+0+0 (secure 0 0 0) 0x7f8cc8018020 con 0x7f8cd419dc30 2024-04-25T00:24:37.639 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.636+0000 7f8cda421700 1 -- 172.21.15.130:0/3495676906 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8cd406a490 con 0x7f8cd419dc30 2024-04-25T00:24:37.639 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.636+0000 7f8cd17fa700 1 --2- 172.21.15.130:0/3495676906 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8cbc038e40 0x7f8cbc03b300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:37.639 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.636+0000 7f8cd17fa700 1 -- 172.21.15.130:0/3495676906 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(4..4 src has 1..4) ==== 1155+0+0 (secure 0 0 0) 0x7f8cc804bd20 con 0x7f8cd419dc30 2024-04-25T00:24:37.639 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.636+0000 7f8cd3fff700 1 --2- 172.21.15.130:0/3495676906 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8cbc038e40 0x7f8cbc03b300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:37.639 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.637+0000 7f8cd3fff700 1 --2- 172.21.15.130:0/3495676906 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8cbc038e40 0x7f8cbc03b300 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f8cc4000c00 tx=0x7f8cc400b580 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:24:37.642 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.641+0000 7f8cd17fa700 1 -- 172.21.15.130:0/3495676906 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f8cc801f020 con 0x7f8cd419dc30 2024-04-25T00:24:37.854 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.854+0000 7f8cd17fa700 1 -- 172.21.15.130:0/3495676906 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8cc8021b90 con 0x7f8cd419dc30 2024-04-25T00:24:37.922 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:37.922+0000 7f8cda421700 1 -- 172.21.15.130:0/3495676906 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) -- 0x7f8cd4198e80 con 0x7f8cd419dc30 2024-04-25T00:24:38.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 podman[32211]: 2024-04-25 00:24:37.79914102 +0000 UTC m=+0.361167943 container init 61e6e5458627aac8f5215ee6a2564de11e7786b6511ad2d3b86a05969eddaf4e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-b, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-d2c45d9, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-25T00:24:38.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 podman[32211]: 2024-04-25 00:24:37.8025223 +0000 UTC m=+0.364549223 container start 61e6e5458627aac8f5215ee6a2564de11e7786b6511ad2d3b86a05969eddaf4e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-b, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-d2c45d9, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T00:24:38.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: set uid:gid to 167:167 (ceph:ceph) 2024-04-25T00:24:38.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: ceph version 19.0.0-2445-gd2c45d97 (d2c45d977142e024a59b68a8f1dce8f0b9840551) squid (dev), process ceph-mon, pid 2 2024-04-25T00:24:38.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: pidfile_write: ignore empty --pid-file 2024-04-25T00:24:38.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: load: jerasure load: lrc 2024-04-25T00:24:38.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: RocksDB version: 7.9.2 2024-04-25T00:24:38.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Git sha 0 2024-04-25T00:24:38.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Compile date 2024-04-24 16:41:23 2024-04-25T00:24:38.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: DB SUMMARY 2024-04-25T00:24:38.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: DB Session ID: KC9H2JRGPQ0C8HE5CC7U 2024-04-25T00:24:38.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: CURRENT file: CURRENT 2024-04-25T00:24:38.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: IDENTITY file: IDENTITY 2024-04-25T00:24:38.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: MANIFEST file: MANIFEST-000005 size: 59 Bytes 2024-04-25T00:24:38.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-04-25T00:24:38.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 636 ; 2024-04-25T00:24:38.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.error_if_exists: 0 2024-04-25T00:24:38.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.create_if_missing: 0 2024-04-25T00:24:38.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.paranoid_checks: 1 2024-04-25T00:24:38.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-25T00:24:38.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-25T00:24:38.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-25T00:24:38.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.env: 0x560a5d731c60 2024-04-25T00:24:38.090 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.fs: PosixFileSystem 2024-04-25T00:24:38.090 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.info_log: 0x560a5e375aa0 2024-04-25T00:24:38.090 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_file_opening_threads: 16 2024-04-25T00:24:38.090 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.statistics: (nil) 2024-04-25T00:24:38.090 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.use_fsync: 0 2024-04-25T00:24:38.090 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_log_file_size: 0 2024-04-25T00:24:38.090 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-25T00:24:38.090 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-25T00:24:38.090 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.keep_log_file_num: 1000 2024-04-25T00:24:38.090 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.recycle_log_file_num: 0 2024-04-25T00:24:38.090 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.allow_fallocate: 1 2024-04-25T00:24:38.090 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.allow_mmap_reads: 0 2024-04-25T00:24:38.090 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.allow_mmap_writes: 0 2024-04-25T00:24:38.091 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.use_direct_reads: 0 2024-04-25T00:24:38.091 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-25T00:24:38.091 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.create_missing_column_families: 0 2024-04-25T00:24:38.091 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.db_log_dir: 2024-04-25T00:24:38.091 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.wal_dir: 2024-04-25T00:24:38.091 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-25T00:24:38.091 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-25T00:24:38.091 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-25T00:24:38.091 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-25T00:24:38.091 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-25T00:24:38.091 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-25T00:24:38.091 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.advise_random_on_open: 1 2024-04-25T00:24:38.091 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.db_write_buffer_size: 0 2024-04-25T00:24:38.091 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.write_buffer_manager: 0x560a5e3481e0 2024-04-25T00:24:38.091 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-25T00:24:38.092 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-25T00:24:38.092 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-25T00:24:38.092 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.rate_limiter: (nil) 2024-04-25T00:24:38.092 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-25T00:24:38.092 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.wal_recovery_mode: 2 2024-04-25T00:24:38.092 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.enable_thread_tracking: 0 2024-04-25T00:24:38.092 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.enable_pipelined_write: 0 2024-04-25T00:24:38.092 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.unordered_write: 0 2024-04-25T00:24:38.092 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-25T00:24:38.092 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-25T00:24:38.092 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-25T00:24:38.092 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-25T00:24:38.092 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.row_cache: None 2024-04-25T00:24:38.092 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.wal_filter: None 2024-04-25T00:24:38.094 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-25T00:24:38.094 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.allow_ingest_behind: 0 2024-04-25T00:24:38.094 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.two_write_queues: 0 2024-04-25T00:24:38.094 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.manual_wal_flush: 0 2024-04-25T00:24:38.094 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.wal_compression: 0 2024-04-25T00:24:38.095 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.atomic_flush: 0 2024-04-25T00:24:38.095 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-25T00:24:38.095 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-25T00:24:38.095 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-25T00:24:38.095 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.log_readahead_size: 0 2024-04-25T00:24:38.095 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-25T00:24:38.095 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.best_efforts_recovery: 0 2024-04-25T00:24:38.095 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-25T00:24:38.095 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-25T00:24:38.095 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.allow_data_in_errors: 0 2024-04-25T00:24:38.095 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.db_host_id: __hostname__ 2024-04-25T00:24:38.095 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-25T00:24:38.095 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_background_jobs: 2 2024-04-25T00:24:38.095 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_background_compactions: -1 2024-04-25T00:24:38.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_subcompactions: 1 2024-04-25T00:24:38.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-25T00:24:38.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-25T00:24:38.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-25T00:24:38.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_total_wal_size: 0 2024-04-25T00:24:38.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-25T00:24:38.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-25T00:24:38.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-25T00:24:38.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-25T00:24:38.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_open_files: -1 2024-04-25T00:24:38.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.bytes_per_sync: 0 2024-04-25T00:24:38.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-25T00:24:38.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-25T00:24:38.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.compaction_readahead_size: 0 2024-04-25T00:24:38.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_background_flushes: -1 2024-04-25T00:24:38.097 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Compression algorithms supported: 2024-04-25T00:24:38.097 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-25T00:24:38.097 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: kZSTD supported: 0 2024-04-25T00:24:38.097 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: kXpressCompression supported: 0 2024-04-25T00:24:38.097 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: kLZ4HCCompression supported: 1 2024-04-25T00:24:38.097 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: kZlibCompression supported: 1 2024-04-25T00:24:38.097 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: kSnappyCompression supported: 1 2024-04-25T00:24:38.097 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: kLZ4Compression supported: 1 2024-04-25T00:24:38.097 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: kBZip2Compression supported: 0 2024-04-25T00:24:38.097 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-25T00:24:38.097 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-25T00:24:38.097 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2024-04-25T00:24:38.097 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-25T00:24:38.097 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-25T00:24:38.097 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.merge_operator: 2024-04-25T00:24:38.098 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.compaction_filter: None 2024-04-25T00:24:38.098 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.compaction_filter_factory: None 2024-04-25T00:24:38.098 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.sst_partitioner_factory: None 2024-04-25T00:24:38.098 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-25T00:24:38.098 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-25T00:24:38.098 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560a5e375ba0) 2024-04-25T00:24:38.098 INFO:journalctl@ceph.mon.b.smithi121.stdout: cache_index_and_filter_blocks: 1 2024-04-25T00:24:38.098 INFO:journalctl@ceph.mon.b.smithi121.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-25T00:24:38.098 INFO:journalctl@ceph.mon.b.smithi121.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-25T00:24:38.098 INFO:journalctl@ceph.mon.b.smithi121.stdout: pin_top_level_index_and_filter: 1 2024-04-25T00:24:38.098 INFO:journalctl@ceph.mon.b.smithi121.stdout: index_type: 0 2024-04-25T00:24:38.098 INFO:journalctl@ceph.mon.b.smithi121.stdout: data_block_index_type: 0 2024-04-25T00:24:38.098 INFO:journalctl@ceph.mon.b.smithi121.stdout: index_shortening: 1 2024-04-25T00:24:38.098 INFO:journalctl@ceph.mon.b.smithi121.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-25T00:24:38.099 INFO:journalctl@ceph.mon.b.smithi121.stdout: checksum: 4 2024-04-25T00:24:38.099 INFO:journalctl@ceph.mon.b.smithi121.stdout: no_block_cache: 0 2024-04-25T00:24:38.099 INFO:journalctl@ceph.mon.b.smithi121.stdout: block_cache: 0x560a5e379090 2024-04-25T00:24:38.099 INFO:journalctl@ceph.mon.b.smithi121.stdout: block_cache_name: BinnedLRUCache 2024-04-25T00:24:38.099 INFO:journalctl@ceph.mon.b.smithi121.stdout: block_cache_options: 2024-04-25T00:24:38.099 INFO:journalctl@ceph.mon.b.smithi121.stdout: capacity : 536870912 2024-04-25T00:24:38.099 INFO:journalctl@ceph.mon.b.smithi121.stdout: num_shard_bits : 4 2024-04-25T00:24:38.099 INFO:journalctl@ceph.mon.b.smithi121.stdout: strict_capacity_limit : 0 2024-04-25T00:24:38.099 INFO:journalctl@ceph.mon.b.smithi121.stdout: high_pri_pool_ratio: 0.000 2024-04-25T00:24:38.099 INFO:journalctl@ceph.mon.b.smithi121.stdout: block_cache_compressed: (nil) 2024-04-25T00:24:38.099 INFO:journalctl@ceph.mon.b.smithi121.stdout: persistent_cache: (nil) 2024-04-25T00:24:38.099 INFO:journalctl@ceph.mon.b.smithi121.stdout: block_size: 4096 2024-04-25T00:24:38.099 INFO:journalctl@ceph.mon.b.smithi121.stdout: block_size_deviation: 10 2024-04-25T00:24:38.099 INFO:journalctl@ceph.mon.b.smithi121.stdout: block_restart_interval: 16 2024-04-25T00:24:38.099 INFO:journalctl@ceph.mon.b.smithi121.stdout: index_block_restart_interval: 1 2024-04-25T00:24:38.100 INFO:journalctl@ceph.mon.b.smithi121.stdout: metadata_block_size: 4096 2024-04-25T00:24:38.100 INFO:journalctl@ceph.mon.b.smithi121.stdout: partition_filters: 0 2024-04-25T00:24:38.100 INFO:journalctl@ceph.mon.b.smithi121.stdout: use_delta_encoding: 1 2024-04-25T00:24:38.100 INFO:journalctl@ceph.mon.b.smithi121.stdout: filter_policy: bloomfilter 2024-04-25T00:24:38.100 INFO:journalctl@ceph.mon.b.smithi121.stdout: whole_key_filtering: 1 2024-04-25T00:24:38.100 INFO:journalctl@ceph.mon.b.smithi121.stdout: verify_compression: 0 2024-04-25T00:24:38.100 INFO:journalctl@ceph.mon.b.smithi121.stdout: read_amp_bytes_per_bit: 0 2024-04-25T00:24:38.100 INFO:journalctl@ceph.mon.b.smithi121.stdout: format_version: 5 2024-04-25T00:24:38.100 INFO:journalctl@ceph.mon.b.smithi121.stdout: enable_index_compression: 1 2024-04-25T00:24:38.100 INFO:journalctl@ceph.mon.b.smithi121.stdout: block_align: 0 2024-04-25T00:24:38.100 INFO:journalctl@ceph.mon.b.smithi121.stdout: max_auto_readahead_size: 262144 2024-04-25T00:24:38.100 INFO:journalctl@ceph.mon.b.smithi121.stdout: prepopulate_block_cache: 0 2024-04-25T00:24:38.100 INFO:journalctl@ceph.mon.b.smithi121.stdout: initial_auto_readahead_size: 8192 2024-04-25T00:24:38.100 INFO:journalctl@ceph.mon.b.smithi121.stdout: num_file_reads_for_auto_readahead: 2 2024-04-25T00:24:38.100 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.write_buffer_size: 33554432 2024-04-25T00:24:38.101 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_write_buffer_number: 2 2024-04-25T00:24:38.101 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.compression: NoCompression 2024-04-25T00:24:38.101 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.bottommost_compression: Disabled 2024-04-25T00:24:38.101 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.prefix_extractor: nullptr 2024-04-25T00:24:38.101 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-25T00:24:38.101 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.num_levels: 7 2024-04-25T00:24:38.101 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-25T00:24:38.101 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-25T00:24:38.101 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-25T00:24:38.101 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-25T00:24:38.101 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-25T00:24:38.101 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-25T00:24:38.101 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-25T00:24:38.101 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-25T00:24:38.101 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-25T00:24:38.102 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-25T00:24:38.102 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-25T00:24:38.102 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-25T00:24:38.102 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-25T00:24:38.102 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.compression_opts.level: 32767 2024-04-25T00:24:38.102 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.compression_opts.strategy: 0 2024-04-25T00:24:38.102 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-25T00:24:38.102 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-25T00:24:38.102 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-25T00:24:38.102 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-25T00:24:38.102 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.compression_opts.enabled: false 2024-04-25T00:24:38.102 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-25T00:24:38.102 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-25T00:24:38.102 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-25T00:24:38.103 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-25T00:24:38.103 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.target_file_size_base: 67108864 2024-04-25T00:24:38.103 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-25T00:24:38.103 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-25T00:24:38.103 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-25T00:24:38.103 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-25T00:24:38.103 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-25T00:24:38.103 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-25T00:24:38.103 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-25T00:24:38.103 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-25T00:24:38.103 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-25T00:24:38.103 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-25T00:24:38.103 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-25T00:24:38.103 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-25T00:24:38.103 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-25T00:24:38.103 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-25T00:24:38.104 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.arena_block_size: 1048576 2024-04-25T00:24:38.104 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-25T00:24:38.104 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-25T00:24:38.104 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.disable_auto_compactions: 0 2024-04-25T00:24:38.104 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-25T00:24:38.104 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-25T00:24:38.104 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-25T00:24:38.104 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-25T00:24:38.104 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-25T00:24:38.104 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-25T00:24:38.104 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-25T00:24:38.104 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-25T00:24:38.104 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-25T00:24:38.104 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-25T00:24:38.104 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.table_properties_collectors: 2024-04-25T00:24:38.105 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.inplace_update_support: 0 2024-04-25T00:24:38.105 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-25T00:24:38.105 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-25T00:24:38.105 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-25T00:24:38.105 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-25T00:24:38.105 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.bloom_locality: 0 2024-04-25T00:24:38.105 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.max_successive_merges: 0 2024-04-25T00:24:38.105 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-25T00:24:38.105 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.paranoid_file_checks: 0 2024-04-25T00:24:38.105 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.force_consistency_checks: 1 2024-04-25T00:24:38.105 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.report_bg_io_stats: 0 2024-04-25T00:24:38.105 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.ttl: 2592000 2024-04-25T00:24:38.105 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-25T00:24:38.105 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-25T00:24:38.105 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-25T00:24:38.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.enable_blob_files: false 2024-04-25T00:24:38.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.min_blob_size: 0 2024-04-25T00:24:38.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.blob_file_size: 268435456 2024-04-25T00:24:38.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-25T00:24:38.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-25T00:24:38.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-25T00:24:38.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-25T00:24:38.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-25T00:24:38.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.blob_file_starting_level: 0 2024-04-25T00:24:38.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-25T00:24:38.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-25T00:24:38.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2024-04-25T00:24:38.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: a3071325-389e-423e-91ca-c36d01518225 2024-04-25T00:24:38.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714004677832300, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-25T00:24:38.107 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #4 mode 2 2024-04-25T00:24:38.107 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714004677832704, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1747, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714004677, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "a3071325-389e-423e-91ca-c36d01518225", "db_session_id": "KC9H2JRGPQ0C8HE5CC7U", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2024-04-25T00:24:38.107 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714004677832782, "job": 1, "event": "recovery_finished"} 2024-04-25T00:24:38.107 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: [db/version_set.cc:5047] Creating manifest 10 2024-04-25T00:24:38.107 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-25T00:24:38.107 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x560a5e46a000 2024-04-25T00:24:38.107 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: DB pointer 0x560a5e454000 2024-04-25T00:24:38.107 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-25T00:24:38.107 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-25T00:24:38.107 INFO:journalctl@ceph.mon.b.smithi121.stdout: ** DB Stats ** 2024-04-25T00:24:38.107 INFO:journalctl@ceph.mon.b.smithi121.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-25T00:24:38.107 INFO:journalctl@ceph.mon.b.smithi121.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-25T00:24:38.107 INFO:journalctl@ceph.mon.b.smithi121.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-25T00:24:38.107 INFO:journalctl@ceph.mon.b.smithi121.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-25T00:24:38.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-25T00:24:38.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-25T00:24:38.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-25T00:24:38.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: 2024-04-25T00:24:38.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: ** Compaction Stats [default] ** 2024-04-25T00:24:38.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-25T00:24:38.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-25T00:24:38.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: L0 1/0 1.71 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.3 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-04-25T00:24:38.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: Sum 1/0 1.71 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.3 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-04-25T00:24:38.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.3 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-04-25T00:24:38.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: 2024-04-25T00:24:38.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: ** Compaction Stats [default] ** 2024-04-25T00:24:38.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-25T00:24:38.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-25T00:24:38.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.3 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-04-25T00:24:38.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: 2024-04-25T00:24:38.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-25T00:24:38.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: 2024-04-25T00:24:38.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-25T00:24:38.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-25T00:24:38.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-25T00:24:38.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-25T00:24:38.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-25T00:24:38.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-25T00:24:38.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: Cumulative compaction: 0.00 GB write, 0.71 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-25T00:24:38.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: Interval compaction: 0.00 GB write, 0.71 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-25T00:24:38.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-25T00:24:38.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: Block cache BinnedLRUCache@0x560a5e379090#2 capacity: 512.00 MB usage: 0.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 7e-06 secs_since: 0 2024-04-25T00:24:38.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2024-04-25T00:24:38.110 INFO:journalctl@ceph.mon.b.smithi121.stdout: 2024-04-25T00:24:38.110 INFO:journalctl@ceph.mon.b.smithi121.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-25T00:24:38.110 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-04-25T00:24:38.110 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: using public_addr v2:172.21.15.121:0/0 -> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] 2024-04-25T00:24:38.110 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: starting mon.b rank -1 at public addrs [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] at bind addrs [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:24:38.110 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: mon.b@-1(???) e0 preinit fsid cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:24:38.110 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: mon.b@-1(synchronizing).mds e1 new map 2024-04-25T00:24:38.110 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: mon.b@-1(synchronizing).mds e1 print_map 2024-04-25T00:24:38.110 INFO:journalctl@ceph.mon.b.smithi121.stdout: e1 2024-04-25T00:24:38.110 INFO:journalctl@ceph.mon.b.smithi121.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-25T00:24:38.110 INFO:journalctl@ceph.mon.b.smithi121.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2,11=minor log segments,12=quiesce subvolumes} 2024-04-25T00:24:38.110 INFO:journalctl@ceph.mon.b.smithi121.stdout: legacy client fscid: -1 2024-04-25T00:24:38.110 INFO:journalctl@ceph.mon.b.smithi121.stdout: 2024-04-25T00:24:38.110 INFO:journalctl@ceph.mon.b.smithi121.stdout: No filesystems configured 2024-04-25T00:24:38.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-25T00:24:38.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-25T00:24:38.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-25T00:24:38.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-25T00:24:38.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-25T00:24:38.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-25T00:24:38.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-25T00:24:38.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-25T00:24:38.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-25T00:24:38.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-25T00:24:38.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:38.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi130", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:24:38.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:38.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:38.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:38.112 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:38.112 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:24:38.112 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:38.112 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:24:38.112 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:38.112 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: Updating smithi121:/etc/ceph/ceph.conf 2024-04-25T00:24:38.112 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: Deploying cephadm binary to smithi130 2024-04-25T00:24:38.112 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: Updating smithi121:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:24:38.112 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-04-25T00:24:38.112 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:38.112 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: Updating smithi121:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T00:24:38.112 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:38.112 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:38.112 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:38.112 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:38.113 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:24:38.113 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:38.113 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:38.113 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: Added host smithi130 2024-04-25T00:24:38.113 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:38.113 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:38.113 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:24:38.113 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:38.113 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:38.113 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/4182769857' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-25T00:24:38.113 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/4182769857' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-25T00:24:38.113 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: osdmap e4: 0 total, 0 up, 0 in 2024-04-25T00:24:38.113 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:38.113 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:38.113 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:38.114 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:38.114 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi130", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:24:38.114 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:38.114 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:24:38.114 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-25T00:24:38.114 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:38.114 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:38.114 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:38.114 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: Updating smithi130:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:24:38.114 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:38.114 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-25T00:24:38.114 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: Updating smithi130:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T00:24:38.114 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:38.114 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi032:172.21.15.32=a;smithi121:172.21.15.121=b;smithi130:172.21.15.130=c", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:24:38.115 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: Saving service mon spec with placement smithi032:172.21.15.32=a;smithi121:172.21.15.121=b;smithi130:172.21.15.130=c;count:3 2024-04-25T00:24:38.115 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:38.115 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:24:38.115 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:38.115 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:24:38.115 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:38.115 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T00:24:38.115 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:38.115 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: Deploying daemon mon.c on smithi130 2024-04-25T00:24:38.115 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:38.115 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:38.115 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='client.? 172.21.15.130:0/2380851424' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-25T00:24:38.115 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:38.115 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-25T00:24:38.115 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: mon.a calling monitor election 2024-04-25T00:24:38.116 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:24:38.116 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:24:38.116 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:38.116 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: mon.c calling monitor election 2024-04-25T00:24:38.116 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:24:38.116 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:24:38.116 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:38.116 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:24:38.116 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-25T00:24:38.116 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: monmap epoch 2 2024-04-25T00:24:38.116 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: fsid cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:24:38.116 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: last_changed 2024-04-25T00:24:28.972598+0000 2024-04-25T00:24:38.116 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: created 2024-04-25T00:22:42.756376+0000 2024-04-25T00:24:38.116 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: min_mon_release 19 (squid) 2024-04-25T00:24:38.116 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: election_strategy: 1 2024-04-25T00:24:38.117 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: 0: [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] mon.a 2024-04-25T00:24:38.117 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: 1: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.c 2024-04-25T00:24:38.117 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: fsmap 2024-04-25T00:24:38.117 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: osdmap e4: 0 total, 0 up, 0 in 2024-04-25T00:24:38.117 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: mgrmap e13: x(active, since 51s) 2024-04-25T00:24:38.117 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: overall HEALTH_OK 2024-04-25T00:24:38.117 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:38.117 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:24:38.117 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:38.117 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:38.117 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T00:24:38.117 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:38.117 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: Deploying daemon mon.b on smithi121 2024-04-25T00:24:38.117 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: from='client.? 172.21.15.130:0/2129239003' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-25T00:24:38.118 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:38.118 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 ceph-mon[32231]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-25T00:24:38.118 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 bash[32211]: 61e6e5458627aac8f5215ee6a2564de11e7786b6511ad2d3b86a05969eddaf4e 2024-04-25T00:24:38.118 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:37 smithi121 systemd[1]: Started Ceph mon.b for cb997156-0299-11ef-bc93-c7b262605968. 2024-04-25T00:24:42.865 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:42.865+0000 7f8cd17fa700 1 -- 172.21.15.130:0/3495676906 <== mon.0 v2:172.21.15.32:3300/0 8 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 3 v3) ==== 95+0+1285 (secure 0 0 0) 0x7f8cc801a400 con 0x7f8cd419dc30 2024-04-25T00:24:42.866 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-25T00:24:42.866 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":3,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","modified":"2024-04-25T00:24:37.854195Z","created":"2024-04-25T00:22:42.756376Z","min_mon_release":19,"min_mon_release_name":"squid","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef","squid"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:3300","nonce":0},{"type":"v1","addr":"172.21.15.32:6789","nonce":0}]},"addr":"172.21.15.32:6789/0","public_addr":"172.21.15.32:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3300","nonce":0},{"type":"v1","addr":"172.21.15.130:6789","nonce":0}]},"addr":"172.21.15.130:6789/0","public_addr":"172.21.15.130:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:3300","nonce":0},{"type":"v1","addr":"172.21.15.121:6789","nonce":0}]},"addr":"172.21.15.121:6789/0","public_addr":"172.21.15.121:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-04-25T00:24:42.868 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:42.868+0000 7f8cda421700 1 -- 172.21.15.130:0/3495676906 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8cbc038e40 msgr2=0x7f8cbc03b300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:42.868 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:42.868+0000 7f8cda421700 1 --2- 172.21.15.130:0/3495676906 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8cbc038e40 0x7f8cbc03b300 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f8cc4000c00 tx=0x7f8cc400b580 comp rx=0 tx=0).stop 2024-04-25T00:24:42.869 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:42.868+0000 7f8cda421700 1 -- 172.21.15.130:0/3495676906 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8cd419dc30 msgr2=0x7f8cd41978b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:42.869 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:42.868+0000 7f8cda421700 1 --2- 172.21.15.130:0/3495676906 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8cd419dc30 0x7f8cd41978b0 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f8cc800b4d0 tx=0x7f8cc800b890 comp rx=0 tx=0).stop 2024-04-25T00:24:42.869 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:42.868+0000 7f8cda421700 1 -- 172.21.15.130:0/3495676906 shutdown_connections 2024-04-25T00:24:42.869 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:42.868+0000 7f8cda421700 1 --2- 172.21.15.130:0/3495676906 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8cbc038e40 0x7f8cbc03b300 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:42.869 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:42.868+0000 7f8cda421700 1 --2- 172.21.15.130:0/3495676906 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8cd4102ee0 0x7f8cd419d6f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:42.869 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:42.868+0000 7f8cda421700 1 --2- 172.21.15.130:0/3495676906 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8cd419dc30 0x7f8cd41978b0 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:42.869 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:42.868+0000 7f8cda421700 1 -- 172.21.15.130:0/3495676906 >> 172.21.15.130:0/3495676906 conn(0x7f8cd40fc710 msgr2=0x7f8cd4103d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:24:42.869 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:42.868+0000 7f8cda421700 1 -- 172.21.15.130:0/3495676906 shutdown_connections 2024-04-25T00:24:42.869 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:42.868+0000 7f8cda421700 1 -- 172.21.15.130:0/3495676906 wait complete. 2024-04-25T00:24:42.869 INFO:teuthology.orchestra.run.smithi130.stderr:dumped monmap epoch 3 2024-04-25T00:24:43.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-25T00:24:43.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:24:43.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:24:43.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: mon.a calling monitor election 2024-04-25T00:24:43.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: mon.c calling monitor election 2024-04-25T00:24:43.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: from='client.? 172.21.15.130:0/3495676906' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-25T00:24:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:24:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:24:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: mon.b calling monitor election 2024-04-25T00:24:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:24:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:24:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:24:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-25T00:24:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: monmap epoch 3 2024-04-25T00:24:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: fsid cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:24:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: last_changed 2024-04-25T00:24:37.854195+0000 2024-04-25T00:24:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: created 2024-04-25T00:22:42.756376+0000 2024-04-25T00:24:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: min_mon_release 19 (squid) 2024-04-25T00:24:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: election_strategy: 1 2024-04-25T00:24:43.144 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: 0: [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] mon.a 2024-04-25T00:24:43.144 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: 1: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.c 2024-04-25T00:24:43.144 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: 2: [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] mon.b 2024-04-25T00:24:43.144 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: fsmap 2024-04-25T00:24:43.144 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: osdmap e4: 0 total, 0 up, 0 in 2024-04-25T00:24:43.144 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: mgrmap e13: x(active, since 60s) 2024-04-25T00:24:43.144 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: overall HEALTH_OK 2024-04-25T00:24:43.144 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:43.144 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:43.144 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:43.144 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:42 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:43.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-25T00:24:43.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:24:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:24:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: mon.a calling monitor election 2024-04-25T00:24:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: mon.c calling monitor election 2024-04-25T00:24:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: from='client.? 172.21.15.130:0/3495676906' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-25T00:24:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:24:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:24:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: mon.b calling monitor election 2024-04-25T00:24:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:24:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:24:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:24:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-25T00:24:43.251 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: monmap epoch 3 2024-04-25T00:24:43.251 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: fsid cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:24:43.251 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: last_changed 2024-04-25T00:24:37.854195+0000 2024-04-25T00:24:43.251 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: created 2024-04-25T00:22:42.756376+0000 2024-04-25T00:24:43.251 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: min_mon_release 19 (squid) 2024-04-25T00:24:43.251 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: election_strategy: 1 2024-04-25T00:24:43.251 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: 0: [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] mon.a 2024-04-25T00:24:43.251 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: 1: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.c 2024-04-25T00:24:43.251 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: 2: [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] mon.b 2024-04-25T00:24:43.251 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: fsmap 2024-04-25T00:24:43.251 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: osdmap e4: 0 total, 0 up, 0 in 2024-04-25T00:24:43.251 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: mgrmap e13: x(active, since 60s) 2024-04-25T00:24:43.251 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: overall HEALTH_OK 2024-04-25T00:24:43.251 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:43.251 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:43.252 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:43.252 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:43.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-25T00:24:43.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:24:43.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:24:43.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: mon.a calling monitor election 2024-04-25T00:24:43.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: mon.c calling monitor election 2024-04-25T00:24:43.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: from='client.? 172.21.15.130:0/3495676906' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-25T00:24:43.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:43.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:24:43.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:24:43.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: mon.b calling monitor election 2024-04-25T00:24:43.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:43.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:24:43.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:24:43.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:24:43.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-25T00:24:43.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: monmap epoch 3 2024-04-25T00:24:43.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: fsid cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:24:43.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: last_changed 2024-04-25T00:24:37.854195+0000 2024-04-25T00:24:43.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: created 2024-04-25T00:22:42.756376+0000 2024-04-25T00:24:43.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: min_mon_release 19 (squid) 2024-04-25T00:24:43.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: election_strategy: 1 2024-04-25T00:24:43.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: 0: [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] mon.a 2024-04-25T00:24:43.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: 1: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.c 2024-04-25T00:24:43.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: 2: [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] mon.b 2024-04-25T00:24:43.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: fsmap 2024-04-25T00:24:43.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: osdmap e4: 0 total, 0 up, 0 in 2024-04-25T00:24:43.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: mgrmap e13: x(active, since 60s) 2024-04-25T00:24:43.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: overall HEALTH_OK 2024-04-25T00:24:43.340 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:43.340 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:43.340 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:43.340 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:42 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:43.535 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-25T00:24:43.535 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph config generate-minimal-conf 2024-04-25T00:24:43.793 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:24:44.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:43 smithi130 ceph-mon[32525]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:44.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:24:44.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:24:44.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:43 smithi032 ceph-mon[29357]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:44.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:24:44.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:24:44.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:43 smithi121 ceph-mon[32231]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:44.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:24:44.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:24:45.507 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.506+0000 7f3736ebb700 1 -- 172.21.15.32:0/2965984843 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3730101ea0 msgr2=0x7f3730102280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:45.508 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.506+0000 7f3736ebb700 1 --2- 172.21.15.32:0/2965984843 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3730101ea0 0x7f3730102280 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f3718009910 tx=0x7f3718009c20 comp rx=0 tx=0).stop 2024-04-25T00:24:45.508 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.506+0000 7f3736ebb700 1 -- 172.21.15.32:0/2965984843 shutdown_connections 2024-04-25T00:24:45.508 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.506+0000 7f3736ebb700 1 --2- 172.21.15.32:0/2965984843 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3730101ea0 0x7f3730102280 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:45.508 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.506+0000 7f3736ebb700 1 -- 172.21.15.32:0/2965984843 >> 172.21.15.32:0/2965984843 conn(0x7f37300fd750 msgr2=0x7f37300ffb70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:24:45.508 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.506+0000 7f3736ebb700 1 -- 172.21.15.32:0/2965984843 shutdown_connections 2024-04-25T00:24:45.508 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.506+0000 7f3736ebb700 1 -- 172.21.15.32:0/2965984843 wait complete. 2024-04-25T00:24:45.508 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.506+0000 7f3736ebb700 1 Processor -- start 2024-04-25T00:24:45.508 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.507+0000 7f3736ebb700 1 -- start start 2024-04-25T00:24:45.508 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.507+0000 7f3736ebb700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3730101ea0 0x7f373019d6b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:45.508 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.507+0000 7f3736ebb700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f373019dbf0 0x7f3730197830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:45.508 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.507+0000 7f3736ebb700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3730197d70 0x7f3730198220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:45.509 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.507+0000 7f3736ebb700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f373010c230 con 0x7f373019dbf0 2024-04-25T00:24:45.509 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.507+0000 7f3736ebb700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f373010c0b0 con 0x7f3730197d70 2024-04-25T00:24:45.509 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.507+0000 7f3736ebb700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f373010c3b0 con 0x7f3730101ea0 2024-04-25T00:24:45.509 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.509 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.509 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.509 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.509 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:45.509 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:24:45.509 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: Updating smithi032:/etc/ceph/ceph.conf 2024-04-25T00:24:45.509 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: Updating smithi121:/etc/ceph/ceph.conf 2024-04-25T00:24:45.510 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-25T00:24:45.510 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: Updating smithi121:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:24:45.510 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: Updating smithi130:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:24:45.510 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: Updating smithi032:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:24:45.510 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.510 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.510 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.510 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.510 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.510 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.510 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.510 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.510 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.510 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.511 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.511 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T00:24:45.511 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-25T00:24:45.511 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:45.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.507+0000 7f372ffff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f373019dbf0 0x7f3730197830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:45.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.507+0000 7f372ffff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f373019dbf0 0x7f3730197830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:49460/0 (socket says 172.21.15.32:49460) 2024-04-25T00:24:45.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.507+0000 7f372ffff700 1 -- 172.21.15.32:0/3334473306 learned_addr learned my addr 172.21.15.32:0/3334473306 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:24:45.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.507+0000 7f3734c57700 1 --2- 172.21.15.32:0/3334473306 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3730101ea0 0x7f373019d6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:45.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.507+0000 7f3735458700 1 --2- 172.21.15.32:0/3334473306 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3730197d70 0x7f3730198220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:45.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.507+0000 7f372ffff700 1 -- 172.21.15.32:0/3334473306 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3730101ea0 msgr2=0x7f373019d6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:45.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.507+0000 7f372ffff700 1 --2- 172.21.15.32:0/3334473306 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3730101ea0 0x7f373019d6b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:45.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.507+0000 7f372ffff700 1 -- 172.21.15.32:0/3334473306 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3730197d70 msgr2=0x7f3730198220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:45.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.507+0000 7f372ffff700 1 --2- 172.21.15.32:0/3334473306 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3730197d70 0x7f3730198220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:45.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.507+0000 7f372ffff700 1 -- 172.21.15.32:0/3334473306 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f37301a4510 con 0x7f373019dbf0 2024-04-25T00:24:45.514 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.508+0000 7f3735458700 1 --2- 172.21.15.32:0/3334473306 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3730197d70 0x7f3730198220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:24:45.514 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.508+0000 7f372ffff700 1 --2- 172.21.15.32:0/3334473306 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f373019dbf0 0x7f3730197830 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f372000b780 tx=0x7f372000ba90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:24:45.514 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.508+0000 7f372dffb700 1 -- 172.21.15.32:0/3334473306 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3720004230 con 0x7f373019dbf0 2024-04-25T00:24:45.514 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.508+0000 7f3736ebb700 1 -- 172.21.15.32:0/3334473306 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f37301a47a0 con 0x7f373019dbf0 2024-04-25T00:24:45.514 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.508+0000 7f372dffb700 1 -- 172.21.15.32:0/3334473306 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f37200043d0 con 0x7f373019dbf0 2024-04-25T00:24:45.514 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.508+0000 7f372dffb700 1 -- 172.21.15.32:0/3334473306 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3720011c90 con 0x7f373019dbf0 2024-04-25T00:24:45.514 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.508+0000 7f3736ebb700 1 -- 172.21.15.32:0/3334473306 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f37301a4d00 con 0x7f373019dbf0 2024-04-25T00:24:45.514 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.510+0000 7f3736ebb700 1 -- 172.21.15.32:0/3334473306 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f3730103380 con 0x7f373019dbf0 2024-04-25T00:24:45.514 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.510+0000 7f372dffb700 1 -- 172.21.15.32:0/3334473306 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 13) ==== 46343+0+0 (secure 0 0 0) 0x7f37200036a0 con 0x7f373019dbf0 2024-04-25T00:24:45.514 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.510+0000 7f372dffb700 1 --2- 172.21.15.32:0/3334473306 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f371c039580 0x7f371c03ba40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:45.514 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.510+0000 7f372dffb700 1 -- 172.21.15.32:0/3334473306 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(4..4 src has 1..4) ==== 1155+0+0 (secure 0 0 0) 0x7f372004bce0 con 0x7f373019dbf0 2024-04-25T00:24:45.514 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.511+0000 7f3734c57700 1 --2- 172.21.15.32:0/3334473306 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f371c039580 0x7f371c03ba40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:45.514 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.511+0000 7f3734c57700 1 --2- 172.21.15.32:0/3334473306 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f371c039580 0x7f371c03ba40 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f3718009750 tx=0x7f371800b540 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:24:45.516 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.516+0000 7f372dffb700 1 -- 172.21.15.32:0/3334473306 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f3720003d50 con 0x7f373019dbf0 2024-04-25T00:24:45.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:45.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:24:45.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: Updating smithi032:/etc/ceph/ceph.conf 2024-04-25T00:24:45.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: Updating smithi121:/etc/ceph/ceph.conf 2024-04-25T00:24:45.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-25T00:24:45.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: Updating smithi121:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:24:45.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: Updating smithi130:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:24:45.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: Updating smithi032:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:24:45.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.590 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.590 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.590 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.590 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.590 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.590 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.590 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.590 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T00:24:45.590 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-25T00:24:45.590 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:45.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:45.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:24:45.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: Updating smithi032:/etc/ceph/ceph.conf 2024-04-25T00:24:45.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: Updating smithi121:/etc/ceph/ceph.conf 2024-04-25T00:24:45.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-25T00:24:45.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: Updating smithi121:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:24:45.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: Updating smithi130:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:24:45.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: Updating smithi032:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:24:45.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:45.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T00:24:45.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-25T00:24:45.644 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:45.735 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.734+0000 7f3736ebb700 1 -- 172.21.15.32:0/3334473306 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "config generate-minimal-conf"} v 0) -- 0x7f37301a4ff0 con 0x7f373019dbf0 2024-04-25T00:24:45.735 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.735+0000 7f372dffb700 1 -- 172.21.15.32:0/3334473306 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "config generate-minimal-conf"}]=0 v9) ==== 76+0+275 (secure 0 0 0) 0x7f3720003990 con 0x7f373019dbf0 2024-04-25T00:24:45.736 INFO:teuthology.orchestra.run.smithi032.stdout:# minimal ceph.conf for cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:24:45.736 INFO:teuthology.orchestra.run.smithi032.stdout:[global] 2024-04-25T00:24:45.736 INFO:teuthology.orchestra.run.smithi032.stdout: fsid = cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:24:45.736 INFO:teuthology.orchestra.run.smithi032.stdout: mon_host = [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] 2024-04-25T00:24:45.738 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.737+0000 7f3736ebb700 1 -- 172.21.15.32:0/3334473306 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f371c039580 msgr2=0x7f371c03ba40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:45.738 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.737+0000 7f3736ebb700 1 --2- 172.21.15.32:0/3334473306 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f371c039580 0x7f371c03ba40 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f3718009750 tx=0x7f371800b540 comp rx=0 tx=0).stop 2024-04-25T00:24:45.738 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.737+0000 7f3736ebb700 1 -- 172.21.15.32:0/3334473306 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f373019dbf0 msgr2=0x7f3730197830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:45.738 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.737+0000 7f3736ebb700 1 --2- 172.21.15.32:0/3334473306 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f373019dbf0 0x7f3730197830 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f372000b780 tx=0x7f372000ba90 comp rx=0 tx=0).stop 2024-04-25T00:24:45.738 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.737+0000 7f3736ebb700 1 -- 172.21.15.32:0/3334473306 shutdown_connections 2024-04-25T00:24:45.738 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.737+0000 7f3736ebb700 1 --2- 172.21.15.32:0/3334473306 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f371c039580 0x7f371c03ba40 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:45.738 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.737+0000 7f3736ebb700 1 --2- 172.21.15.32:0/3334473306 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3730197d70 0x7f3730198220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:45.738 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.737+0000 7f3736ebb700 1 --2- 172.21.15.32:0/3334473306 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3730101ea0 0x7f373019d6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:45.738 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.737+0000 7f3736ebb700 1 --2- 172.21.15.32:0/3334473306 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f373019dbf0 0x7f3730197830 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:45.738 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.737+0000 7f3736ebb700 1 -- 172.21.15.32:0/3334473306 >> 172.21.15.32:0/3334473306 conn(0x7f37300fd750 msgr2=0x7f3730103870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:24:45.738 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.737+0000 7f3736ebb700 1 -- 172.21.15.32:0/3334473306 shutdown_connections 2024-04-25T00:24:45.738 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:45.737+0000 7f3736ebb700 1 -- 172.21.15.32:0/3334473306 wait complete. 2024-04-25T00:24:46.490 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:46 smithi032 ceph-mon[29357]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:46.491 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:46 smithi032 ceph-mon[29357]: Reconfiguring mon.a (unknown last config time)... 2024-04-25T00:24:46.491 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:46 smithi032 ceph-mon[29357]: Reconfiguring daemon mon.a on smithi032 2024-04-25T00:24:46.491 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:46 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3334473306' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:46.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:46 smithi121 ceph-mon[32231]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:46.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:46 smithi121 ceph-mon[32231]: Reconfiguring mon.a (unknown last config time)... 2024-04-25T00:24:46.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:46 smithi121 ceph-mon[32231]: Reconfiguring daemon mon.a on smithi032 2024-04-25T00:24:46.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:46 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3334473306' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:46.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:46 smithi130 ceph-mon[32525]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:46.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:46 smithi130 ceph-mon[32525]: Reconfiguring mon.a (unknown last config time)... 2024-04-25T00:24:46.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:46 smithi130 ceph-mon[32525]: Reconfiguring daemon mon.a on smithi032 2024-04-25T00:24:46.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:46 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3334473306' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:46.783 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-25T00:24:46.783 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-04-25T00:24:46.783 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd of=/etc/ceph/ceph.conf 2024-04-25T00:24:46.815 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-04-25T00:24:46.815 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-25T00:24:46.882 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-04-25T00:24:46.882 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/etc/ceph/ceph.conf 2024-04-25T00:24:46.914 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-04-25T00:24:46.914 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-25T00:24:46.981 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-04-25T00:24:46.981 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/ceph/ceph.conf 2024-04-25T00:24:47.012 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-04-25T00:24:47.012 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-25T00:24:47.079 INFO:tasks.cephadm:Adding mgr.x on smithi032 2024-04-25T00:24:47.079 INFO:tasks.cephadm:Adding mgr.y on smithi121 2024-04-25T00:24:47.079 INFO:tasks.cephadm:Adding mgr.z on smithi130 2024-04-25T00:24:47.079 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph orch apply mgr '3;smithi032=x;smithi121=y;smithi130=z' 2024-04-25T00:24:47.350 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.c/config 2024-04-25T00:24:48.489 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:48 smithi130 ceph-mon[32525]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:48.489 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:48 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:48.489 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:48 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:48.489 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:48 smithi130 ceph-mon[32525]: Reconfiguring mon.b (monmap changed)... 2024-04-25T00:24:48.489 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:48 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T00:24:48.489 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:48 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-25T00:24:48.489 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:48 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:48.490 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:48 smithi130 ceph-mon[32525]: Reconfiguring daemon mon.b on smithi121 2024-04-25T00:24:48.490 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:48 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:48.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:48 smithi032 ceph-mon[29357]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:48.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:48 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:48.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:48 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:48.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:48 smithi032 ceph-mon[29357]: Reconfiguring mon.b (monmap changed)... 2024-04-25T00:24:48.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:48 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T00:24:48.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:48 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-25T00:24:48.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:48 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:48.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:48 smithi032 ceph-mon[29357]: Reconfiguring daemon mon.b on smithi121 2024-04-25T00:24:48.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:48 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:48.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:48 smithi121 ceph-mon[32231]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:48.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:48 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:48.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:48 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:48.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:48 smithi121 ceph-mon[32231]: Reconfiguring mon.b (monmap changed)... 2024-04-25T00:24:48.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:48 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T00:24:48.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:48 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-25T00:24:48.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:48 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:48.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:48 smithi121 ceph-mon[32231]: Reconfiguring daemon mon.b on smithi121 2024-04-25T00:24:48.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:48 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:48.836 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.835+0000 7f33f95c6700 1 -- 172.21.15.130:0/2222039319 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f33f4102840 msgr2=0x7f33f4104c30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:48.836 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.835+0000 7f33f95c6700 1 --2- 172.21.15.130:0/2222039319 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f33f4102840 0x7f33f4104c30 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f33e4009910 tx=0x7f33e4009c20 comp rx=0 tx=0).stop 2024-04-25T00:24:48.836 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.835+0000 7f33f95c6700 1 -- 172.21.15.130:0/2222039319 shutdown_connections 2024-04-25T00:24:48.836 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.835+0000 7f33f95c6700 1 --2- 172.21.15.130:0/2222039319 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f33f4102840 0x7f33f4104c30 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:48.836 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.835+0000 7f33f95c6700 1 -- 172.21.15.130:0/2222039319 >> 172.21.15.130:0/2222039319 conn(0x7f33f40fc270 msgr2=0x7f33f40fe690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:24:48.837 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.835+0000 7f33f95c6700 1 -- 172.21.15.130:0/2222039319 shutdown_connections 2024-04-25T00:24:48.837 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.835+0000 7f33f95c6700 1 -- 172.21.15.130:0/2222039319 wait complete. 2024-04-25T00:24:48.837 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.836+0000 7f33f95c6700 1 Processor -- start 2024-04-25T00:24:48.837 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.836+0000 7f33f95c6700 1 -- start start 2024-04-25T00:24:48.839 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.836+0000 7f33f95c6700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f33f4102840 0x7f33f4197580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:48.839 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.836+0000 7f33f95c6700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f33f4197ac0 0x7f33f419cb30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:48.840 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.836+0000 7f33f95c6700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f33f4197f40 0x7f33f419f0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:48.840 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.836+0000 7f33f95c6700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f33f4107d90 con 0x7f33f4197f40 2024-04-25T00:24:48.840 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.836+0000 7f33f95c6700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f33f4107c10 con 0x7f33f4102840 2024-04-25T00:24:48.840 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.836+0000 7f33f95c6700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f33f4107f10 con 0x7f33f4197ac0 2024-04-25T00:24:48.840 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.836+0000 7f33f27fc700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f33f4197ac0 0x7f33f419cb30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:48.840 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.836+0000 7f33f27fc700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f33f4197ac0 0x7f33f419cb30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.130:55786/0 (socket says 172.21.15.130:55786) 2024-04-25T00:24:48.840 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.836+0000 7f33f27fc700 1 -- 172.21.15.130:0/587959475 learned_addr learned my addr 172.21.15.130:0/587959475 (peer_addr_for_me v2:172.21.15.130:0/0) 2024-04-25T00:24:48.840 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.836+0000 7f33f37fe700 1 --2- 172.21.15.130:0/587959475 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f33f4197f40 0x7f33f419f0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:48.840 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.836+0000 7f33f2ffd700 1 --2- 172.21.15.130:0/587959475 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f33f4102840 0x7f33f4197580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:48.840 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.837+0000 7f33f37fe700 1 -- 172.21.15.130:0/587959475 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f33f4197ac0 msgr2=0x7f33f419cb30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:48.840 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.837+0000 7f33f37fe700 1 --2- 172.21.15.130:0/587959475 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f33f4197ac0 0x7f33f419cb30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:48.840 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.837+0000 7f33f37fe700 1 -- 172.21.15.130:0/587959475 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f33f4102840 msgr2=0x7f33f4197580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:48.840 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.837+0000 7f33f37fe700 1 --2- 172.21.15.130:0/587959475 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f33f4102840 0x7f33f4197580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:48.841 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.837+0000 7f33f37fe700 1 -- 172.21.15.130:0/587959475 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f33f419f990 con 0x7f33f4197f40 2024-04-25T00:24:48.841 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.837+0000 7f33f27fc700 1 --2- 172.21.15.130:0/587959475 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f33f4197ac0 0x7f33f419cb30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:24:48.841 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.837+0000 7f33f2ffd700 1 --2- 172.21.15.130:0/587959475 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f33f4102840 0x7f33f4197580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:24:48.841 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.837+0000 7f33f37fe700 1 --2- 172.21.15.130:0/587959475 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f33f4197f40 0x7f33f419f0c0 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f33ec00bd50 tx=0x7f33ec00bd80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:24:48.841 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.837+0000 7f33dbfff700 1 -- 172.21.15.130:0/587959475 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f33ec019070 con 0x7f33f4197f40 2024-04-25T00:24:48.841 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.837+0000 7f33dbfff700 1 -- 172.21.15.130:0/587959475 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f33ec00cc90 con 0x7f33f4197f40 2024-04-25T00:24:48.841 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.837+0000 7f33f95c6700 1 -- 172.21.15.130:0/587959475 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f33f419fc80 con 0x7f33f4197f40 2024-04-25T00:24:48.841 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.837+0000 7f33dbfff700 1 -- 172.21.15.130:0/587959475 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f33ec012740 con 0x7f33f4197f40 2024-04-25T00:24:48.841 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.838+0000 7f33f95c6700 1 -- 172.21.15.130:0/587959475 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f33f41a01c0 con 0x7f33f4197f40 2024-04-25T00:24:48.841 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.838+0000 7f33dbfff700 1 -- 172.21.15.130:0/587959475 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 13) ==== 46343+0+0 (secure 0 0 0) 0x7f33ec01c070 con 0x7f33f4197f40 2024-04-25T00:24:48.841 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.838+0000 7f33dbfff700 1 --2- 172.21.15.130:0/587959475 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f33dc039580 0x7f33dc03ba40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:48.841 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.838+0000 7f33dbfff700 1 -- 172.21.15.130:0/587959475 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(4..4 src has 1..4) ==== 1155+0+0 (secure 0 0 0) 0x7f33ec04b900 con 0x7f33f4197f40 2024-04-25T00:24:48.841 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.839+0000 7f33f2ffd700 1 --2- 172.21.15.130:0/587959475 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f33dc039580 0x7f33dc03ba40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:48.841 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.839+0000 7f33f95c6700 1 -- 172.21.15.130:0/587959475 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f33f402db30 con 0x7f33f4197f40 2024-04-25T00:24:48.842 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.839+0000 7f33f2ffd700 1 --2- 172.21.15.130:0/587959475 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f33dc039580 0x7f33dc03ba40 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f33e400b5c0 tx=0x7f33e401a040 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:24:48.845 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:48.844+0000 7f33dbfff700 1 -- 172.21.15.130:0/587959475 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f33ec007290 con 0x7f33f4197f40 2024-04-25T00:24:49.060 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:49.059+0000 7f33f95c6700 1 -- 172.21.15.130:0/587959475 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mgr", "placement": "3;smithi032=x;smithi121=y;smithi130=z", "target": ["mon-mgr", ""]}) -- 0x7f33f4062670 con 0x7f33dc039580 2024-04-25T00:24:49.066 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:49.066+0000 7f33dbfff700 1 -- 172.21.15.130:0/587959475 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+24 (secure 0 0 0) 0x7f33f4062670 con 0x7f33dc039580 2024-04-25T00:24:49.067 INFO:teuthology.orchestra.run.smithi130.stdout:Scheduled mgr update... 2024-04-25T00:24:49.069 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:49.069+0000 7f33f95c6700 1 -- 172.21.15.130:0/587959475 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f33dc039580 msgr2=0x7f33dc03ba40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:49.070 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:49.069+0000 7f33f95c6700 1 --2- 172.21.15.130:0/587959475 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f33dc039580 0x7f33dc03ba40 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f33e400b5c0 tx=0x7f33e401a040 comp rx=0 tx=0).stop 2024-04-25T00:24:49.070 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:49.069+0000 7f33f95c6700 1 -- 172.21.15.130:0/587959475 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f33f4197f40 msgr2=0x7f33f419f0c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:49.070 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:49.069+0000 7f33f95c6700 1 --2- 172.21.15.130:0/587959475 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f33f4197f40 0x7f33f419f0c0 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f33ec00bd50 tx=0x7f33ec00bd80 comp rx=0 tx=0).stop 2024-04-25T00:24:49.070 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:49.069+0000 7f33f95c6700 1 -- 172.21.15.130:0/587959475 shutdown_connections 2024-04-25T00:24:49.070 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:49.069+0000 7f33f95c6700 1 --2- 172.21.15.130:0/587959475 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f33dc039580 0x7f33dc03ba40 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:49.070 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:49.069+0000 7f33f95c6700 1 --2- 172.21.15.130:0/587959475 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f33f4197f40 0x7f33f419f0c0 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:49.070 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:49.069+0000 7f33f95c6700 1 --2- 172.21.15.130:0/587959475 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f33f4102840 0x7f33f4197580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:49.070 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:49.069+0000 7f33f95c6700 1 --2- 172.21.15.130:0/587959475 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f33f4197ac0 0x7f33f419cb30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:49.070 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:49.069+0000 7f33f95c6700 1 -- 172.21.15.130:0/587959475 >> 172.21.15.130:0/587959475 conn(0x7f33f40fc270 msgr2=0x7f33f40fcee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:24:49.070 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:49.069+0000 7f33f95c6700 1 -- 172.21.15.130:0/587959475 shutdown_connections 2024-04-25T00:24:49.070 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:24:49.069+0000 7f33f95c6700 1 -- 172.21.15.130:0/587959475 wait complete. 2024-04-25T00:24:49.596 DEBUG:teuthology.orchestra.run.smithi121:mgr.y> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mgr.y.service 2024-04-25T00:24:49.599 DEBUG:teuthology.orchestra.run.smithi130:mgr.z> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mgr.z.service 2024-04-25T00:24:49.602 INFO:tasks.cephadm:Deploying OSDs... 2024-04-25T00:24:49.602 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-04-25T00:24:49.602 DEBUG:teuthology.orchestra.run.smithi032:> dd if=/scratch_devs of=/dev/stdout 2024-04-25T00:24:49.622 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-25T00:24:49.622 DEBUG:teuthology.orchestra.run.smithi032:> stat /dev/vg_nvme/lv_1 2024-04-25T00:24:49.681 INFO:teuthology.orchestra.run.smithi032.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-25T00:24:49.681 INFO:teuthology.orchestra.run.smithi032.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-25T00:24:49.681 INFO:teuthology.orchestra.run.smithi032.stdout:Device: 5h/5d Inode: 664 Links: 1 2024-04-25T00:24:49.681 INFO:teuthology.orchestra.run.smithi032.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-25T00:24:49.681 INFO:teuthology.orchestra.run.smithi032.stdout:Context: system_u:object_r:device_t:s0 2024-04-25T00:24:49.681 INFO:teuthology.orchestra.run.smithi032.stdout:Access: 2024-04-25 00:23:55.250534596 +0000 2024-04-25T00:24:49.681 INFO:teuthology.orchestra.run.smithi032.stdout:Modify: 2024-04-25 00:19:07.976861252 +0000 2024-04-25T00:24:49.681 INFO:teuthology.orchestra.run.smithi032.stdout:Change: 2024-04-25 00:19:07.976861252 +0000 2024-04-25T00:24:49.681 INFO:teuthology.orchestra.run.smithi032.stdout: Birth: 2024-04-25 00:19:07.976861252 +0000 2024-04-25T00:24:49.682 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-25T00:24:49.750 INFO:teuthology.orchestra.run.smithi032.stderr:1+0 records in 2024-04-25T00:24:49.750 INFO:teuthology.orchestra.run.smithi032.stderr:1+0 records out 2024-04-25T00:24:49.750 INFO:teuthology.orchestra.run.smithi032.stderr:512 bytes copied, 0.000124756 s, 4.1 MB/s 2024-04-25T00:24:49.751 DEBUG:teuthology.orchestra.run.smithi032:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-25T00:24:49.772 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:49 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:49.773 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:49 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:49.773 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:49 smithi130 ceph-mon[32525]: Reconfiguring mon.c (monmap changed)... 2024-04-25T00:24:49.773 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:49 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T00:24:49.773 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:49 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-25T00:24:49.773 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:49 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:49.773 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:49 smithi130 ceph-mon[32525]: Reconfiguring daemon mon.c on smithi130 2024-04-25T00:24:49.774 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:49 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:49.812 DEBUG:teuthology.orchestra.run.smithi032:> stat /dev/vg_nvme/lv_2 2024-04-25T00:24:49.870 INFO:teuthology.orchestra.run.smithi032.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-25T00:24:49.870 INFO:teuthology.orchestra.run.smithi032.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-25T00:24:49.871 INFO:teuthology.orchestra.run.smithi032.stdout:Device: 5h/5d Inode: 683 Links: 1 2024-04-25T00:24:49.871 INFO:teuthology.orchestra.run.smithi032.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-25T00:24:49.871 INFO:teuthology.orchestra.run.smithi032.stdout:Context: system_u:object_r:device_t:s0 2024-04-25T00:24:49.871 INFO:teuthology.orchestra.run.smithi032.stdout:Access: 2024-04-25 00:23:55.290533785 +0000 2024-04-25T00:24:49.871 INFO:teuthology.orchestra.run.smithi032.stdout:Modify: 2024-04-25 00:19:08.309854474 +0000 2024-04-25T00:24:49.871 INFO:teuthology.orchestra.run.smithi032.stdout:Change: 2024-04-25 00:19:08.309854474 +0000 2024-04-25T00:24:49.871 INFO:teuthology.orchestra.run.smithi032.stdout: Birth: 2024-04-25 00:19:08.309854474 +0000 2024-04-25T00:24:49.871 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-25T00:24:49.934 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:49 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:49.934 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:49 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:49.934 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:49 smithi032 ceph-mon[29357]: Reconfiguring mon.c (monmap changed)... 2024-04-25T00:24:49.934 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:49 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T00:24:49.934 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:49 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-25T00:24:49.934 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:49 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:49.934 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:49 smithi032 ceph-mon[29357]: Reconfiguring daemon mon.c on smithi130 2024-04-25T00:24:49.934 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:49 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:49.936 INFO:teuthology.orchestra.run.smithi032.stderr:1+0 records in 2024-04-25T00:24:49.936 INFO:teuthology.orchestra.run.smithi032.stderr:1+0 records out 2024-04-25T00:24:49.936 INFO:teuthology.orchestra.run.smithi032.stderr:512 bytes copied, 0.000117156 s, 4.4 MB/s 2024-04-25T00:24:49.938 DEBUG:teuthology.orchestra.run.smithi032:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-25T00:24:49.996 DEBUG:teuthology.orchestra.run.smithi032:> stat /dev/vg_nvme/lv_3 2024-04-25T00:24:50.054 INFO:teuthology.orchestra.run.smithi032.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-25T00:24:50.054 INFO:teuthology.orchestra.run.smithi032.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-25T00:24:50.055 INFO:teuthology.orchestra.run.smithi032.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-04-25T00:24:50.055 INFO:teuthology.orchestra.run.smithi032.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-25T00:24:50.055 INFO:teuthology.orchestra.run.smithi032.stdout:Context: system_u:object_r:device_t:s0 2024-04-25T00:24:50.055 INFO:teuthology.orchestra.run.smithi032.stdout:Access: 2024-04-25 00:23:55.328533014 +0000 2024-04-25T00:24:50.055 INFO:teuthology.orchestra.run.smithi032.stdout:Modify: 2024-04-25 00:19:08.684846841 +0000 2024-04-25T00:24:50.055 INFO:teuthology.orchestra.run.smithi032.stdout:Change: 2024-04-25 00:19:08.684846841 +0000 2024-04-25T00:24:50.055 INFO:teuthology.orchestra.run.smithi032.stdout: Birth: 2024-04-25 00:19:08.684846841 +0000 2024-04-25T00:24:50.055 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-25T00:24:50.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:49 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:50.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:49 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:50.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:49 smithi121 ceph-mon[32231]: Reconfiguring mon.c (monmap changed)... 2024-04-25T00:24:50.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:49 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T00:24:50.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:49 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-25T00:24:50.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:49 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:50.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:49 smithi121 ceph-mon[32231]: Reconfiguring daemon mon.c on smithi130 2024-04-25T00:24:50.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:49 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:50.119 INFO:teuthology.orchestra.run.smithi032.stderr:1+0 records in 2024-04-25T00:24:50.120 INFO:teuthology.orchestra.run.smithi032.stderr:1+0 records out 2024-04-25T00:24:50.120 INFO:teuthology.orchestra.run.smithi032.stderr:512 bytes copied, 0.000119088 s, 4.3 MB/s 2024-04-25T00:24:50.121 DEBUG:teuthology.orchestra.run.smithi032:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-25T00:24:50.179 DEBUG:teuthology.orchestra.run.smithi032:> stat /dev/vg_nvme/lv_4 2024-04-25T00:24:50.234 INFO:teuthology.orchestra.run.smithi032.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-25T00:24:50.234 INFO:teuthology.orchestra.run.smithi032.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-25T00:24:50.235 INFO:teuthology.orchestra.run.smithi032.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-04-25T00:24:50.235 INFO:teuthology.orchestra.run.smithi032.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-25T00:24:50.235 INFO:teuthology.orchestra.run.smithi032.stdout:Context: system_u:object_r:device_t:s0 2024-04-25T00:24:50.235 INFO:teuthology.orchestra.run.smithi032.stdout:Access: 2024-04-25 00:23:55.364532285 +0000 2024-04-25T00:24:50.235 INFO:teuthology.orchestra.run.smithi032.stdout:Modify: 2024-04-25 00:19:09.133837702 +0000 2024-04-25T00:24:50.235 INFO:teuthology.orchestra.run.smithi032.stdout:Change: 2024-04-25 00:19:09.133837702 +0000 2024-04-25T00:24:50.235 INFO:teuthology.orchestra.run.smithi032.stdout: Birth: 2024-04-25 00:19:09.133837702 +0000 2024-04-25T00:24:50.235 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-25T00:24:50.299 INFO:teuthology.orchestra.run.smithi032.stderr:1+0 records in 2024-04-25T00:24:50.299 INFO:teuthology.orchestra.run.smithi032.stderr:1+0 records out 2024-04-25T00:24:50.299 INFO:teuthology.orchestra.run.smithi032.stderr:512 bytes copied, 0.000115739 s, 4.4 MB/s 2024-04-25T00:24:50.300 DEBUG:teuthology.orchestra.run.smithi032:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-25T00:24:50.358 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-04-25T00:24:50.358 DEBUG:teuthology.orchestra.run.smithi121:> dd if=/scratch_devs of=/dev/stdout 2024-04-25T00:24:50.376 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-25T00:24:50.377 DEBUG:teuthology.orchestra.run.smithi121:> stat /dev/vg_nvme/lv_1 2024-04-25T00:24:50.434 INFO:teuthology.orchestra.run.smithi121.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-25T00:24:50.435 INFO:teuthology.orchestra.run.smithi121.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-25T00:24:50.435 INFO:teuthology.orchestra.run.smithi121.stdout:Device: 5h/5d Inode: 664 Links: 1 2024-04-25T00:24:50.435 INFO:teuthology.orchestra.run.smithi121.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-25T00:24:50.435 INFO:teuthology.orchestra.run.smithi121.stdout:Context: system_u:object_r:device_t:s0 2024-04-25T00:24:50.435 INFO:teuthology.orchestra.run.smithi121.stdout:Access: 2024-04-25 00:24:37.264468016 +0000 2024-04-25T00:24:50.435 INFO:teuthology.orchestra.run.smithi121.stdout:Modify: 2024-04-25 00:19:32.938215978 +0000 2024-04-25T00:24:50.435 INFO:teuthology.orchestra.run.smithi121.stdout:Change: 2024-04-25 00:19:32.938215978 +0000 2024-04-25T00:24:50.435 INFO:teuthology.orchestra.run.smithi121.stdout: Birth: 2024-04-25 00:19:32.938215978 +0000 2024-04-25T00:24:50.435 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-25T00:24:50.501 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records in 2024-04-25T00:24:50.501 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records out 2024-04-25T00:24:50.501 INFO:teuthology.orchestra.run.smithi121.stderr:512 bytes copied, 0.000267002 s, 1.9 MB/s 2024-04-25T00:24:50.503 DEBUG:teuthology.orchestra.run.smithi121:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-25T00:24:50.562 DEBUG:teuthology.orchestra.run.smithi121:> stat /dev/vg_nvme/lv_2 2024-04-25T00:24:50.619 INFO:teuthology.orchestra.run.smithi121.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-25T00:24:50.619 INFO:teuthology.orchestra.run.smithi121.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-25T00:24:50.619 INFO:teuthology.orchestra.run.smithi121.stdout:Device: 5h/5d Inode: 683 Links: 1 2024-04-25T00:24:50.619 INFO:teuthology.orchestra.run.smithi121.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-25T00:24:50.619 INFO:teuthology.orchestra.run.smithi121.stdout:Context: system_u:object_r:device_t:s0 2024-04-25T00:24:50.619 INFO:teuthology.orchestra.run.smithi121.stdout:Access: 2024-04-25 00:24:37.264468016 +0000 2024-04-25T00:24:50.619 INFO:teuthology.orchestra.run.smithi121.stdout:Modify: 2024-04-25 00:19:33.294211339 +0000 2024-04-25T00:24:50.620 INFO:teuthology.orchestra.run.smithi121.stdout:Change: 2024-04-25 00:19:33.294211339 +0000 2024-04-25T00:24:50.620 INFO:teuthology.orchestra.run.smithi121.stdout: Birth: 2024-04-25 00:19:33.294211339 +0000 2024-04-25T00:24:50.620 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-25T00:24:50.683 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records in 2024-04-25T00:24:50.683 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records out 2024-04-25T00:24:50.683 INFO:teuthology.orchestra.run.smithi121.stderr:512 bytes copied, 0.000201518 s, 2.5 MB/s 2024-04-25T00:24:50.685 DEBUG:teuthology.orchestra.run.smithi121:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-25T00:24:50.741 DEBUG:teuthology.orchestra.run.smithi121:> stat /dev/vg_nvme/lv_3 2024-04-25T00:24:50.796 INFO:teuthology.orchestra.run.smithi121.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-25T00:24:50.796 INFO:teuthology.orchestra.run.smithi121.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-25T00:24:50.797 INFO:teuthology.orchestra.run.smithi121.stdout:Device: 5h/5d Inode: 702 Links: 1 2024-04-25T00:24:50.797 INFO:teuthology.orchestra.run.smithi121.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-25T00:24:50.797 INFO:teuthology.orchestra.run.smithi121.stdout:Context: system_u:object_r:device_t:s0 2024-04-25T00:24:50.797 INFO:teuthology.orchestra.run.smithi121.stdout:Access: 2024-04-25 00:24:37.264468016 +0000 2024-04-25T00:24:50.797 INFO:teuthology.orchestra.run.smithi121.stdout:Modify: 2024-04-25 00:19:33.642206805 +0000 2024-04-25T00:24:50.797 INFO:teuthology.orchestra.run.smithi121.stdout:Change: 2024-04-25 00:19:33.642206805 +0000 2024-04-25T00:24:50.797 INFO:teuthology.orchestra.run.smithi121.stdout: Birth: 2024-04-25 00:19:33.642206805 +0000 2024-04-25T00:24:50.797 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-25T00:24:50.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:50 smithi121 ceph-mon[32231]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:50.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:50 smithi121 ceph-mon[32231]: from='client.14205 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "3;smithi032=x;smithi121=y;smithi130=z", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:24:50.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:50 smithi121 ceph-mon[32231]: Saving service mgr spec with placement smithi032=x;smithi121=y;smithi130=z;count:3 2024-04-25T00:24:50.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:50 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:50.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:50 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:50.859 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records in 2024-04-25T00:24:50.859 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records out 2024-04-25T00:24:50.860 INFO:teuthology.orchestra.run.smithi121.stderr:512 bytes copied, 0.000267387 s, 1.9 MB/s 2024-04-25T00:24:50.861 DEBUG:teuthology.orchestra.run.smithi121:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-25T00:24:50.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:50 smithi130 ceph-mon[32525]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:50.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:50 smithi130 ceph-mon[32525]: from='client.14205 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "3;smithi032=x;smithi121=y;smithi130=z", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:24:50.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:50 smithi130 ceph-mon[32525]: Saving service mgr spec with placement smithi032=x;smithi121=y;smithi130=z;count:3 2024-04-25T00:24:50.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:50 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:50.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:50 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:50.916 DEBUG:teuthology.orchestra.run.smithi121:> stat /dev/vg_nvme/lv_4 2024-04-25T00:24:50.972 INFO:teuthology.orchestra.run.smithi121.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-25T00:24:50.972 INFO:teuthology.orchestra.run.smithi121.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-25T00:24:50.972 INFO:teuthology.orchestra.run.smithi121.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-04-25T00:24:50.972 INFO:teuthology.orchestra.run.smithi121.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-25T00:24:50.972 INFO:teuthology.orchestra.run.smithi121.stdout:Context: system_u:object_r:device_t:s0 2024-04-25T00:24:50.972 INFO:teuthology.orchestra.run.smithi121.stdout:Access: 2024-04-25 00:24:37.265468003 +0000 2024-04-25T00:24:50.972 INFO:teuthology.orchestra.run.smithi121.stdout:Modify: 2024-04-25 00:19:33.981202388 +0000 2024-04-25T00:24:50.972 INFO:teuthology.orchestra.run.smithi121.stdout:Change: 2024-04-25 00:19:33.981202388 +0000 2024-04-25T00:24:50.972 INFO:teuthology.orchestra.run.smithi121.stdout: Birth: 2024-04-25 00:19:33.981202388 +0000 2024-04-25T00:24:50.972 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-25T00:24:50.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:50 smithi032 ceph-mon[29357]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:50.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:50 smithi032 ceph-mon[29357]: from='client.14205 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "3;smithi032=x;smithi121=y;smithi130=z", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:24:50.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:50 smithi032 ceph-mon[29357]: Saving service mgr spec with placement smithi032=x;smithi121=y;smithi130=z;count:3 2024-04-25T00:24:50.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:50 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:50.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:50 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:51.035 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records in 2024-04-25T00:24:51.035 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records out 2024-04-25T00:24:51.035 INFO:teuthology.orchestra.run.smithi121.stderr:512 bytes copied, 0.000204919 s, 2.5 MB/s 2024-04-25T00:24:51.036 DEBUG:teuthology.orchestra.run.smithi121:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-25T00:24:51.092 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-04-25T00:24:51.092 DEBUG:teuthology.orchestra.run.smithi130:> dd if=/scratch_devs of=/dev/stdout 2024-04-25T00:24:51.109 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-25T00:24:51.110 DEBUG:teuthology.orchestra.run.smithi130:> stat /dev/vg_nvme/lv_1 2024-04-25T00:24:51.166 INFO:teuthology.orchestra.run.smithi130.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-25T00:24:51.166 INFO:teuthology.orchestra.run.smithi130.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-25T00:24:51.166 INFO:teuthology.orchestra.run.smithi130.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-04-25T00:24:51.166 INFO:teuthology.orchestra.run.smithi130.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-25T00:24:51.166 INFO:teuthology.orchestra.run.smithi130.stdout:Context: system_u:object_r:device_t:s0 2024-04-25T00:24:51.167 INFO:teuthology.orchestra.run.smithi130.stdout:Access: 2024-04-25 00:24:27.841772986 +0000 2024-04-25T00:24:51.167 INFO:teuthology.orchestra.run.smithi130.stdout:Modify: 2024-04-25 00:19:19.630517235 +0000 2024-04-25T00:24:51.167 INFO:teuthology.orchestra.run.smithi130.stdout:Change: 2024-04-25 00:19:19.630517235 +0000 2024-04-25T00:24:51.167 INFO:teuthology.orchestra.run.smithi130.stdout: Birth: 2024-04-25 00:19:19.630517235 +0000 2024-04-25T00:24:51.167 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-25T00:24:51.234 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records in 2024-04-25T00:24:51.234 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records out 2024-04-25T00:24:51.234 INFO:teuthology.orchestra.run.smithi130.stderr:512 bytes copied, 0.000277949 s, 1.8 MB/s 2024-04-25T00:24:51.234 DEBUG:teuthology.orchestra.run.smithi130:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-25T00:24:51.291 DEBUG:teuthology.orchestra.run.smithi130:> stat /dev/vg_nvme/lv_2 2024-04-25T00:24:51.346 INFO:teuthology.orchestra.run.smithi130.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-25T00:24:51.346 INFO:teuthology.orchestra.run.smithi130.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-25T00:24:51.346 INFO:teuthology.orchestra.run.smithi130.stdout:Device: 5h/5d Inode: 683 Links: 1 2024-04-25T00:24:51.346 INFO:teuthology.orchestra.run.smithi130.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-25T00:24:51.346 INFO:teuthology.orchestra.run.smithi130.stdout:Context: system_u:object_r:device_t:s0 2024-04-25T00:24:51.346 INFO:teuthology.orchestra.run.smithi130.stdout:Access: 2024-04-25 00:24:27.841772986 +0000 2024-04-25T00:24:51.346 INFO:teuthology.orchestra.run.smithi130.stdout:Modify: 2024-04-25 00:19:19.979510825 +0000 2024-04-25T00:24:51.346 INFO:teuthology.orchestra.run.smithi130.stdout:Change: 2024-04-25 00:19:19.979510825 +0000 2024-04-25T00:24:51.346 INFO:teuthology.orchestra.run.smithi130.stdout: Birth: 2024-04-25 00:19:19.979510825 +0000 2024-04-25T00:24:51.347 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-25T00:24:51.410 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records in 2024-04-25T00:24:51.410 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records out 2024-04-25T00:24:51.410 INFO:teuthology.orchestra.run.smithi130.stderr:512 bytes copied, 0.000220863 s, 2.3 MB/s 2024-04-25T00:24:51.411 DEBUG:teuthology.orchestra.run.smithi130:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-25T00:24:51.467 DEBUG:teuthology.orchestra.run.smithi130:> stat /dev/vg_nvme/lv_3 2024-04-25T00:24:51.523 INFO:teuthology.orchestra.run.smithi130.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-25T00:24:51.523 INFO:teuthology.orchestra.run.smithi130.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-25T00:24:51.523 INFO:teuthology.orchestra.run.smithi130.stdout:Device: 5h/5d Inode: 702 Links: 1 2024-04-25T00:24:51.523 INFO:teuthology.orchestra.run.smithi130.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-25T00:24:51.523 INFO:teuthology.orchestra.run.smithi130.stdout:Context: system_u:object_r:device_t:s0 2024-04-25T00:24:51.523 INFO:teuthology.orchestra.run.smithi130.stdout:Access: 2024-04-25 00:24:27.841772986 +0000 2024-04-25T00:24:51.523 INFO:teuthology.orchestra.run.smithi130.stdout:Modify: 2024-04-25 00:19:20.324504488 +0000 2024-04-25T00:24:51.523 INFO:teuthology.orchestra.run.smithi130.stdout:Change: 2024-04-25 00:19:20.324504488 +0000 2024-04-25T00:24:51.523 INFO:teuthology.orchestra.run.smithi130.stdout: Birth: 2024-04-25 00:19:20.324504488 +0000 2024-04-25T00:24:51.524 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-25T00:24:51.587 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records in 2024-04-25T00:24:51.587 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records out 2024-04-25T00:24:51.587 INFO:teuthology.orchestra.run.smithi130.stderr:512 bytes copied, 0.000258678 s, 2.0 MB/s 2024-04-25T00:24:51.588 DEBUG:teuthology.orchestra.run.smithi130:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-25T00:24:51.644 DEBUG:teuthology.orchestra.run.smithi130:> stat /dev/vg_nvme/lv_4 2024-04-25T00:24:51.700 INFO:teuthology.orchestra.run.smithi130.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-25T00:24:51.700 INFO:teuthology.orchestra.run.smithi130.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-25T00:24:51.700 INFO:teuthology.orchestra.run.smithi130.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-04-25T00:24:51.700 INFO:teuthology.orchestra.run.smithi130.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-25T00:24:51.701 INFO:teuthology.orchestra.run.smithi130.stdout:Context: system_u:object_r:device_t:s0 2024-04-25T00:24:51.701 INFO:teuthology.orchestra.run.smithi130.stdout:Access: 2024-04-25 00:24:27.842772968 +0000 2024-04-25T00:24:51.701 INFO:teuthology.orchestra.run.smithi130.stdout:Modify: 2024-04-25 00:19:20.668498170 +0000 2024-04-25T00:24:51.701 INFO:teuthology.orchestra.run.smithi130.stdout:Change: 2024-04-25 00:19:20.668498170 +0000 2024-04-25T00:24:51.701 INFO:teuthology.orchestra.run.smithi130.stdout: Birth: 2024-04-25 00:19:20.668498170 +0000 2024-04-25T00:24:51.701 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-25T00:24:51.763 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records in 2024-04-25T00:24:51.763 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records out 2024-04-25T00:24:51.763 INFO:teuthology.orchestra.run.smithi130.stderr:512 bytes copied, 0.000199862 s, 2.6 MB/s 2024-04-25T00:24:51.764 DEBUG:teuthology.orchestra.run.smithi130:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-25T00:24:51.821 INFO:tasks.cephadm:Deploying osd.0 on smithi032 with /dev/vg_nvme/lv_4... 2024-04-25T00:24:51.822 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-25T00:24:51.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:51 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:24:51.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:51 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:51.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:51 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:24:51.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:51 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:51.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:51 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T00:24:51.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:51 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-25T00:24:51.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:51 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-25T00:24:51.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:51 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:51.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:51 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:24:51.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:51 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:51.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:51 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:24:51.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:51 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:51.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:51 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T00:24:51.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:51 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-25T00:24:51.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:51 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-25T00:24:51.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:51 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:52.082 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:24:52.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:51 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:24:52.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:51 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:52.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:51 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:24:52.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:51 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:52.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:51 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T00:24:52.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:51 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-25T00:24:52.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:51 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-25T00:24:52.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:51 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:52.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:52 smithi032 ceph-mon[29357]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:52.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:52 smithi032 ceph-mon[29357]: Deploying daemon mgr.z on smithi130 2024-04-25T00:24:52.984 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:52 smithi130 ceph-mon[32525]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:52.984 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:52 smithi130 ceph-mon[32525]: Deploying daemon mgr.z on smithi130 2024-04-25T00:24:53.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:52 smithi121 ceph-mon[32231]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:53.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:52 smithi121 ceph-mon[32231]: Deploying daemon mgr.z on smithi130 2024-04-25T00:24:53.391 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:53 smithi130 systemd[1]: Starting Ceph mgr.z for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:24:53.662 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:53 smithi130 podman[34350]: 2024-04-25 00:24:53.303976298 +0000 UTC m=+0.017792749 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901 2024-04-25T00:24:53.662 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:53 smithi130 podman[34350]: 2024-04-25 00:24:53.401692914 +0000 UTC m=+0.115509339 container create 681d0365cccfb1e43d1036dfbf361fb3d3ebb03cd85c97f4431c73a89cc9e8e7 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z, GIT_CLEAN=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-d2c45d9, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T00:24:53.662 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:53 smithi130 podman[34350]: 2024-04-25 00:24:53.587143418 +0000 UTC m=+0.300959836 container init 681d0365cccfb1e43d1036dfbf361fb3d3ebb03cd85c97f4431c73a89cc9e8e7 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-d2c45d9, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-25T00:24:53.662 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:53 smithi130 podman[34350]: 2024-04-25 00:24:53.589751493 +0000 UTC m=+0.303567913 container start 681d0365cccfb1e43d1036dfbf361fb3d3ebb03cd85c97f4431c73a89cc9e8e7 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z, RELEASE=squid-d2c45d9, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-25T00:24:53.662 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:53 smithi130 ceph-mgr[34370]: -- 172.21.15.130:0/1569153892 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56148c8545a0 con 0x56148c828c00 2024-04-25T00:24:53.662 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:53 smithi130 bash[34350]: 681d0365cccfb1e43d1036dfbf361fb3d3ebb03cd85c97f4431c73a89cc9e8e7 2024-04-25T00:24:53.933 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:53 smithi130 systemd[1]: Started Ceph mgr.z for cb997156-0299-11ef-bc93-c7b262605968. 2024-04-25T00:24:53.933 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:53 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:24:53.749+0000 7fc5f63ee200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-25T00:24:53.933 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:53 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:24:53.932+0000 7fc5f63ee200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-25T00:24:54.007 INFO:teuthology.orchestra.run.smithi032.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-25T00:24:54.007 INFO:teuthology.orchestra.run.smithi032.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-25T00:24:54.007 INFO:teuthology.orchestra.run.smithi032.stderr: stderr: 10+0 records in 2024-04-25T00:24:54.008 INFO:teuthology.orchestra.run.smithi032.stderr:10+0 records out 2024-04-25T00:24:54.008 INFO:teuthology.orchestra.run.smithi032.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00824736 s, 1.3 GB/s 2024-04-25T00:24:54.008 INFO:teuthology.orchestra.run.smithi032.stderr:--> Zapping successful for: 2024-04-25T00:24:54.340 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:54 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:24:54.034+0000 7fc5f63ee200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-25T00:24:54.340 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:54 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:24:54.339+0000 7fc5f63ee200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-25T00:24:54.600 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi032:vg_nvme/lv_4 2024-04-25T00:24:54.641 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:54 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:24:54.423+0000 7fc5f63ee200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-25T00:24:54.845 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:24:54.896 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:54 smithi121 ceph-mon[32231]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:54.896 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:54 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:54.896 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:54 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:54.896 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:54 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:54.896 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:54 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T00:24:54.896 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:54 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-25T00:24:54.896 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:54 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-25T00:24:54.897 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:54 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:54.967 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:54 smithi032 ceph-mon[29357]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:54.968 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:54 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:54.968 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:54 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:54.968 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:54 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:54.968 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:54 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T00:24:54.968 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:54 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-25T00:24:54.968 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:54 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-25T00:24:54.968 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:54 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:55.022 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:54 smithi130 ceph-mon[32525]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:55.022 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:54 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:55.022 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:54 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:55.022 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:54 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:55.022 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:54 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T00:24:55.022 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:54 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-25T00:24:55.022 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:54 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-25T00:24:55.022 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:54 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:55.866 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:55 smithi121 ceph-mon[32231]: Deploying daemon mgr.y on smithi121 2024-04-25T00:24:55.945 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:55 smithi032 ceph-mon[29357]: Deploying daemon mgr.y on smithi121 2024-04-25T00:24:55.982 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:55 smithi130 ceph-mon[32525]: Deploying daemon mgr.y on smithi121 2024-04-25T00:24:55.982 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:55 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:24:55.647+0000 7fc5f63ee200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-25T00:24:55.982 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:55 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:24:55.821+0000 7fc5f63ee200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-25T00:24:56.284 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:55 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:24:55.998+0000 7fc5f63ee200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-25T00:24:56.284 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:56 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:24:56.283+0000 7fc5f63ee200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-25T00:24:56.287 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.287+0000 7f89925e6700 1 -- 172.21.15.32:0/3060793942 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f898c1022b0 msgr2=0x7f898c1026d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:56.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.287+0000 7f89925e6700 1 --2- 172.21.15.32:0/3060793942 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f898c1022b0 0x7f898c1026d0 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f897c0099c0 tx=0x7f897c009cd0 comp rx=0 tx=0).stop 2024-04-25T00:24:56.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.287+0000 7f89925e6700 1 -- 172.21.15.32:0/3060793942 shutdown_connections 2024-04-25T00:24:56.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.287+0000 7f89925e6700 1 --2- 172.21.15.32:0/3060793942 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f898c103e50 0x7f898c10a700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:56.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.287+0000 7f89925e6700 1 --2- 172.21.15.32:0/3060793942 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f898c1022b0 0x7f898c1026d0 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:56.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.287+0000 7f89925e6700 1 --2- 172.21.15.32:0/3060793942 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f898c1034b0 0x7f898c103910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:56.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.287+0000 7f89925e6700 1 -- 172.21.15.32:0/3060793942 >> 172.21.15.32:0/3060793942 conn(0x7f898c0fd850 msgr2=0x7f898c0ffc90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:24:56.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.287+0000 7f89925e6700 1 -- 172.21.15.32:0/3060793942 shutdown_connections 2024-04-25T00:24:56.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.287+0000 7f89925e6700 1 -- 172.21.15.32:0/3060793942 wait complete. 2024-04-25T00:24:56.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.287+0000 7f89925e6700 1 Processor -- start 2024-04-25T00:24:56.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.288+0000 7f89925e6700 1 -- start start 2024-04-25T00:24:56.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.288+0000 7f89925e6700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f898c1022b0 0x7f898c1984e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:56.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.288+0000 7f89925e6700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f898c1034b0 0x7f898c198a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:56.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.288+0000 7f89925e6700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f898c103e50 0x7f898c19faa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:56.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.288+0000 7f89925e6700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f898c10de70 con 0x7f898c1022b0 2024-04-25T00:24:56.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.288+0000 7f89925e6700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f898c10dcf0 con 0x7f898c1034b0 2024-04-25T00:24:56.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.288+0000 7f89925e6700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f898c10dff0 con 0x7f898c103e50 2024-04-25T00:24:56.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.288+0000 7f898bfff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f898c1022b0 0x7f898c1984e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:56.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.288+0000 7f898bfff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f898c1022b0 0x7f898c1984e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:41466/0 (socket says 172.21.15.32:41466) 2024-04-25T00:24:56.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.288+0000 7f898bfff700 1 -- 172.21.15.32:0/443037048 learned_addr learned my addr 172.21.15.32:0/443037048 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:24:56.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.288+0000 7f898b7fe700 1 --2- 172.21.15.32:0/443037048 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f898c1034b0 0x7f898c198a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:56.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.288+0000 7f8990b83700 1 --2- 172.21.15.32:0/443037048 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f898c103e50 0x7f898c19faa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:56.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.288+0000 7f898bfff700 1 -- 172.21.15.32:0/443037048 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f898c103e50 msgr2=0x7f898c19faa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:56.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.288+0000 7f898bfff700 1 --2- 172.21.15.32:0/443037048 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f898c103e50 0x7f898c19faa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.288+0000 7f898bfff700 1 -- 172.21.15.32:0/443037048 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f898c1034b0 msgr2=0x7f898c198a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:24:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.288+0000 7f898bfff700 1 --2- 172.21.15.32:0/443037048 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f898c1034b0 0x7f898c198a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:24:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.288+0000 7f898bfff700 1 -- 172.21.15.32:0/443037048 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f898c1a0140 con 0x7f898c1022b0 2024-04-25T00:24:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.289+0000 7f898bfff700 1 --2- 172.21.15.32:0/443037048 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f898c1022b0 0x7f898c1984e0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f897c00f710 tx=0x7f897c009580 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:24:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.289+0000 7f89897fa700 1 -- 172.21.15.32:0/443037048 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f897c009e60 con 0x7f898c1022b0 2024-04-25T00:24:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.289+0000 7f89897fa700 1 -- 172.21.15.32:0/443037048 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f897c003960 con 0x7f898c1022b0 2024-04-25T00:24:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.289+0000 7f89925e6700 1 -- 172.21.15.32:0/443037048 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f898c1a03d0 con 0x7f898c1022b0 2024-04-25T00:24:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.289+0000 7f89897fa700 1 -- 172.21.15.32:0/443037048 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f897c005560 con 0x7f898c1022b0 2024-04-25T00:24:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.289+0000 7f89925e6700 1 -- 172.21.15.32:0/443037048 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f898c1a0900 con 0x7f898c1022b0 2024-04-25T00:24:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.290+0000 7f89897fa700 1 -- 172.21.15.32:0/443037048 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 13) ==== 46343+0+0 (secure 0 0 0) 0x7f897c00fb00 con 0x7f898c1022b0 2024-04-25T00:24:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.290+0000 7f89925e6700 1 -- 172.21.15.32:0/443037048 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f898c068f20 con 0x7f898c1022b0 2024-04-25T00:24:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.290+0000 7f89897fa700 1 --2- 172.21.15.32:0/443037048 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8978039330 0x7f897803b7f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:24:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.290+0000 7f89897fa700 1 -- 172.21.15.32:0/443037048 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(4..4 src has 1..4) ==== 1155+0+0 (secure 0 0 0) 0x7f897c04d040 con 0x7f898c1022b0 2024-04-25T00:24:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.290+0000 7f898b7fe700 1 --2- 172.21.15.32:0/443037048 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8978039330 0x7f897803b7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:24:56.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.290+0000 7f898b7fe700 1 --2- 172.21.15.32:0/443037048 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8978039330 0x7f897803b7f0 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f8974009c60 tx=0x7f8974009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:24:56.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.295+0000 7f89897fa700 1 -- 172.21.15.32:0/443037048 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f897c013070 con 0x7f898c1022b0 2024-04-25T00:24:56.390 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:24:56 smithi121 systemd[1]: Started Ceph mgr.y for cb997156-0299-11ef-bc93-c7b262605968. 2024-04-25T00:24:56.516 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:24:56.515+0000 7f89925e6700 1 -- 172.21.15.32:0/443037048 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_4", "target": ["mon-mgr", ""]}) -- 0x7f898c062670 con 0x7f8978039330 2024-04-25T00:24:56.583 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:56 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:24:56.374+0000 7fc5f63ee200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-25T00:24:56.663 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:24:56 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:24:56.389+0000 7ffa91472200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-25T00:24:56.663 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:24:56 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:24:56.577+0000 7ffa91472200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-25T00:24:56.826 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:56 smithi032 ceph-mon[29357]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:56.827 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:56 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:56.827 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:56 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:56.827 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:56 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:56.827 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:56 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:56.827 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:56 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:24:56.827 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:56 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:24:56.827 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:56 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:56.827 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:56 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:24:56.891 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:56 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:24:56.583+0000 7fc5f63ee200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-25T00:24:56.892 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:56 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:24:56.782+0000 7fc5f63ee200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-25T00:24:56.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:56 smithi130 ceph-mon[32525]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:56.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:56 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:56.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:56 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:56.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:56 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:56.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:56 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:56.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:56 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:24:56.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:56 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:24:56.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:56 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:56.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:56 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:24:57.012 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:24:56 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:24:56.686+0000 7ffa91472200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-25T00:24:57.012 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:56 smithi121 ceph-mon[32231]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:57.012 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:56 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:57.012 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:56 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:57.012 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:56 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:57.012 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:56 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:57.013 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:56 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:24:57.013 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:56 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:24:57.013 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:56 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:57.013 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:56 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:24:57.322 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:24:57 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:24:57.011+0000 7ffa91472200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-25T00:24:57.322 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:24:57 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:24:57.109+0000 7ffa91472200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-25T00:24:57.642 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:57 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:24:57.412+0000 7fc5f63ee200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-25T00:24:57.946 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:57 smithi130 ceph-mon[32525]: from='client.14223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:24:57.946 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:57 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:24:57.852+0000 7fc5f63ee200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-25T00:24:57.946 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:57 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:24:57.945+0000 7fc5f63ee200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-25T00:24:57.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:57 smithi032 ceph-mon[29357]: from='client.14223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:24:58.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:57 smithi121 ceph-mon[32231]: from='client.14223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:24:58.391 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:58 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:24:58.145+0000 7fc5f63ee200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-25T00:24:58.587 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:24:58 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:24:58.328+0000 7ffa91472200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-25T00:24:58.587 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:24:58 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:24:58.495+0000 7ffa91472200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-25T00:24:58.951 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:24:58 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:24:58.664+0000 7ffa91472200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-25T00:24:58.951 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:24:58 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:24:58.950+0000 7ffa91472200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-25T00:24:58.952 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:58 smithi121 ceph-mon[32231]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:58.952 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:58 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:58.952 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:58 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:58.952 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:58 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:58.952 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:58 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:58.953 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:58 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:58.953 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:58 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:24:58.953 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:58 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:58.953 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:58 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:58.953 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:58 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T00:24:58.953 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:58 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-25T00:24:58.954 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:58 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:58.984 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:58 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:24:58.659+0000 7fc5f63ee200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-25T00:24:58.984 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:58 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:24:58.748+0000 7fc5f63ee200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-25T00:24:58.984 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:58 smithi130 ceph-mon[32525]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:58.984 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:58 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:58.985 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:58 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:58.985 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:58 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:58.985 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:58 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:58.985 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:58 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:58.985 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:58 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:24:58.985 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:58 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:58.985 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:58 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:58.985 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:58 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T00:24:58.985 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:58 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-25T00:24:58.985 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:58 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:58.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:58 smithi032 ceph-mon[29357]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:24:58.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:58 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:58.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:58 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:58.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:58 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:58.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:58 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:58.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:58 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:58.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:58 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:24:58.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:58 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:58.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:58 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:24:58.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:58 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T00:24:58.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:58 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-25T00:24:58.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:58 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:24:59.241 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:24:59 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:24:59.041+0000 7ffa91472200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-25T00:24:59.242 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:24:59 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:24:59.241+0000 7ffa91472200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-25T00:24:59.325 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:58 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:24:58.983+0000 7fc5f63ee200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-25T00:24:59.325 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:59 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:24:59.116+0000 7fc5f63ee200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-25T00:24:59.325 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:59 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:24:59.209+0000 7fc5f63ee200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-25T00:24:59.325 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:59 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:24:59.324+0000 7fc5f63ee200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-25T00:24:59.587 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:24:59 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:24:59.430+0000 7ffa91472200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-25T00:24:59.642 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:24:59 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:24:59.416+0000 7fc5f63ee200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-25T00:24:59.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:59 smithi032 ceph-mon[29357]: Reconfiguring mgr.x (unknown last config time)... 2024-04-25T00:24:59.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:59 smithi032 ceph-mon[29357]: Reconfiguring daemon mgr.x on smithi032 2024-04-25T00:24:59.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:59 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3268536258' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "074681d6-1be7-49a1-9518-9968ad62dc08"}]: dispatch 2024-04-25T00:24:59.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:59 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3268536258' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "074681d6-1be7-49a1-9518-9968ad62dc08"}]': finished 2024-04-25T00:24:59.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:59 smithi032 ceph-mon[29357]: osdmap e5: 1 total, 0 up, 1 in 2024-04-25T00:24:59.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:24:59 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T00:25:00.028 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:59 smithi121 ceph-mon[32231]: Reconfiguring mgr.x (unknown last config time)... 2024-04-25T00:25:00.028 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:59 smithi121 ceph-mon[32231]: Reconfiguring daemon mgr.x on smithi032 2024-04-25T00:25:00.028 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:59 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3268536258' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "074681d6-1be7-49a1-9518-9968ad62dc08"}]: dispatch 2024-04-25T00:25:00.028 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:59 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3268536258' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "074681d6-1be7-49a1-9518-9968ad62dc08"}]': finished 2024-04-25T00:25:00.029 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:59 smithi121 ceph-mon[32231]: osdmap e5: 1 total, 0 up, 1 in 2024-04-25T00:25:00.029 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:24:59 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T00:25:00.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:59 smithi130 ceph-mon[32525]: Reconfiguring mgr.x (unknown last config time)... 2024-04-25T00:25:00.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:59 smithi130 ceph-mon[32525]: Reconfiguring daemon mgr.x on smithi032 2024-04-25T00:25:00.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:59 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3268536258' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "074681d6-1be7-49a1-9518-9968ad62dc08"}]: dispatch 2024-04-25T00:25:00.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:59 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3268536258' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "074681d6-1be7-49a1-9518-9968ad62dc08"}]': finished 2024-04-25T00:25:00.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:59 smithi130 ceph-mon[32525]: osdmap e5: 1 total, 0 up, 1 in 2024-04-25T00:25:00.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:24:59 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T00:25:00.337 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:25:00 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:25:00.027+0000 7ffa91472200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-25T00:25:00.642 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:25:00 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:25:00.236+0000 7fc5f63ee200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-25T00:25:00.642 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:25:00 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:25:00.448+0000 7fc5f63ee200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-25T00:25:00.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:25:00.681+0000 7f89897fa700 1 -- 172.21.15.32:0/443037048 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mgrmap(e 14) ==== 92060+0+0 (secure 0 0 0) 0x7f897c02b7c0 con 0x7f898c1022b0 2024-04-25T00:25:00.743 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:00 smithi121 ceph-mon[32231]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:00.743 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:00 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3832003547' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:25:00.743 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:00 smithi121 ceph-mon[32231]: Standby manager daemon z started 2024-04-25T00:25:00.743 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:00 smithi121 ceph-mon[32231]: from='mgr.? 172.21.15.130:0/3922576091' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2024-04-25T00:25:00.743 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:00 smithi121 ceph-mon[32231]: from='mgr.? 172.21.15.130:0/3922576091' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T00:25:00.743 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:00 smithi121 ceph-mon[32231]: from='mgr.? 172.21.15.130:0/3922576091' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2024-04-25T00:25:00.743 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:00 smithi121 ceph-mon[32231]: from='mgr.? 172.21.15.130:0/3922576091' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T00:25:00.743 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:25:00 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:25:00.447+0000 7ffa91472200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-25T00:25:00.744 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:25:00 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:25:00.543+0000 7ffa91472200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-25T00:25:00.744 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:25:00 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:25:00.742+0000 7ffa91472200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-25T00:25:00.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:00 smithi032 ceph-mon[29357]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:00.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:00 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3832003547' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:25:00.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:00 smithi032 ceph-mon[29357]: Standby manager daemon z started 2024-04-25T00:25:00.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:00 smithi032 ceph-mon[29357]: from='mgr.? 172.21.15.130:0/3922576091' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2024-04-25T00:25:00.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:00 smithi032 ceph-mon[29357]: from='mgr.? 172.21.15.130:0/3922576091' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T00:25:00.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:00 smithi032 ceph-mon[29357]: from='mgr.? 172.21.15.130:0/3922576091' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2024-04-25T00:25:00.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:00 smithi032 ceph-mon[29357]: from='mgr.? 172.21.15.130:0/3922576091' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T00:25:01.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:00 smithi130 ceph-mon[32525]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:01.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:00 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3832003547' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:25:01.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:00 smithi130 ceph-mon[32525]: Standby manager daemon z started 2024-04-25T00:25:01.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:00 smithi130 ceph-mon[32525]: from='mgr.? 172.21.15.130:0/3922576091' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2024-04-25T00:25:01.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:00 smithi130 ceph-mon[32525]: from='mgr.? 172.21.15.130:0/3922576091' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T00:25:01.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:00 smithi130 ceph-mon[32525]: from='mgr.? 172.21.15.130:0/3922576091' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2024-04-25T00:25:01.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:00 smithi130 ceph-mon[32525]: from='mgr.? 172.21.15.130:0/3922576091' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T00:25:01.587 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:25:01 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:25:01.263+0000 7ffa91472200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-25T00:25:01.588 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:25:01 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:25:01.352+0000 7ffa91472200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-25T00:25:01.934 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:01 smithi121 ceph-mon[32231]: mgrmap e14: x(active, since 77s), standbys: z 2024-04-25T00:25:01.934 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:01 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2024-04-25T00:25:01.934 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:25:01 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:25:01.593+0000 7ffa91472200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-25T00:25:01.934 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:25:01 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:25:01.725+0000 7ffa91472200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-25T00:25:01.934 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:25:01 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:25:01.819+0000 7ffa91472200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-25T00:25:01.934 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:25:01 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:25:01.933+0000 7ffa91472200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-25T00:25:01.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:01 smithi032 ceph-mon[29357]: mgrmap e14: x(active, since 77s), standbys: z 2024-04-25T00:25:01.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:01 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2024-04-25T00:25:02.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:01 smithi130 ceph-mon[32525]: mgrmap e14: x(active, since 77s), standbys: z 2024-04-25T00:25:02.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:01 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2024-04-25T00:25:02.337 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:25:02 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:25:02.023+0000 7ffa91472200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-25T00:25:02.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:02 smithi032 ceph-mon[29357]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:03.060 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:02 smithi121 ceph-mon[32231]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:03.060 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:25:02 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:25:02.847+0000 7ffa91472200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-25T00:25:03.060 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:25:03 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:25:03.059+0000 7ffa91472200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-25T00:25:03.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:02 smithi130 ceph-mon[32525]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:03.695 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:25:03.694+0000 7f89897fa700 1 -- 172.21.15.32:0/443037048 <== mon.0 v2:172.21.15.32:3300/0 8 ==== mgrmap(e 15) ==== 137777+0+0 (secure 0 0 0) 0x7f897c0213c0 con 0x7f898c1022b0 2024-04-25T00:25:03.894 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:03 smithi032 ceph-mon[29357]: Standby manager daemon y started 2024-04-25T00:25:03.895 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:03 smithi032 ceph-mon[29357]: from='mgr.? 172.21.15.121:0/3321834448' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-25T00:25:03.895 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:03 smithi032 ceph-mon[29357]: from='mgr.? 172.21.15.121:0/3321834448' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T00:25:03.895 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:03 smithi032 ceph-mon[29357]: from='mgr.? 172.21.15.121:0/3321834448' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-25T00:25:03.895 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:03 smithi032 ceph-mon[29357]: from='mgr.? 172.21.15.121:0/3321834448' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T00:25:04.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:03 smithi121 ceph-mon[32231]: Standby manager daemon y started 2024-04-25T00:25:04.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:03 smithi121 ceph-mon[32231]: from='mgr.? 172.21.15.121:0/3321834448' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-25T00:25:04.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:03 smithi121 ceph-mon[32231]: from='mgr.? 172.21.15.121:0/3321834448' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T00:25:04.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:03 smithi121 ceph-mon[32231]: from='mgr.? 172.21.15.121:0/3321834448' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-25T00:25:04.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:03 smithi121 ceph-mon[32231]: from='mgr.? 172.21.15.121:0/3321834448' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T00:25:04.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:03 smithi130 ceph-mon[32525]: Standby manager daemon y started 2024-04-25T00:25:04.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:03 smithi130 ceph-mon[32525]: from='mgr.? 172.21.15.121:0/3321834448' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-25T00:25:04.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:03 smithi130 ceph-mon[32525]: from='mgr.? 172.21.15.121:0/3321834448' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T00:25:04.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:03 smithi130 ceph-mon[32525]: from='mgr.? 172.21.15.121:0/3321834448' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-25T00:25:04.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:03 smithi130 ceph-mon[32525]: from='mgr.? 172.21.15.121:0/3321834448' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T00:25:04.893 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:04 smithi032 ceph-mon[29357]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:04.894 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:04 smithi032 ceph-mon[29357]: mgrmap e15: x(active, since 80s), standbys: z, y 2024-04-25T00:25:04.894 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:04 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-25T00:25:04.894 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:04 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:04.894 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:04 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:05.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:04 smithi121 ceph-mon[32231]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:05.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:04 smithi121 ceph-mon[32231]: mgrmap e15: x(active, since 80s), standbys: z, y 2024-04-25T00:25:05.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:04 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-25T00:25:05.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:04 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:05.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:04 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:05.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:04 smithi130 ceph-mon[32525]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:05.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:04 smithi130 ceph-mon[32525]: mgrmap e15: x(active, since 80s), standbys: z, y 2024-04-25T00:25:05.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:04 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-25T00:25:05.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:04 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:05.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:04 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:05.816 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:05 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:25:05.816 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:05 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:05.816 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:05 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:25:05.816 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:05 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:06.036 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:05 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:25:06.036 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:05 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:06.036 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:05 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:25:06.036 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:05 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:06.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:05 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:25:06.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:05 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:06.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:05 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:25:06.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:05 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:06.871 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:06 smithi032 ceph-mon[29357]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:07.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:06 smithi121 ceph-mon[32231]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:07.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:06 smithi130 ceph-mon[32525]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:07.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-25T00:25:07.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:08.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:07 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-25T00:25:08.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:07 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:08.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:07 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-25T00:25:08.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:07 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:08.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:08 smithi032 ceph-mon[29357]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:08.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:08 smithi032 ceph-mon[29357]: Deploying daemon osd.0 on smithi032 2024-04-25T00:25:09.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:08 smithi121 ceph-mon[32231]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:09.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:08 smithi121 ceph-mon[32231]: Deploying daemon osd.0 on smithi032 2024-04-25T00:25:09.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:08 smithi130 ceph-mon[32525]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:09.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:08 smithi130 ceph-mon[32525]: Deploying daemon osd.0 on smithi032 2024-04-25T00:25:10.986 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:10 smithi032 ceph-mon[29357]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:11.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:10 smithi121 ceph-mon[32231]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:11.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:10 smithi130 ceph-mon[32525]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:11.859 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:11 smithi032 ceph-mon[29357]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:12.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:11 smithi121 ceph-mon[32231]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:12.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:11 smithi130 ceph-mon[32525]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:13.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:13 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:13.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:13 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:13.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:13 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:25:13.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:13 smithi032 ceph-mon[29357]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:14.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:13 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:14.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:13 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:14.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:13 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:25:14.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:13 smithi121 ceph-mon[32231]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:14.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:13 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:14.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:13 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:14.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:13 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:25:14.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:13 smithi130 ceph-mon[32525]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:16.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:15 smithi130 ceph-mon[32525]: pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:16.163 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:15 smithi032 ceph-mon[29357]: pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:16.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:25:16.224+0000 7f89897fa700 1 -- 172.21.15.32:0/443037048 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7f898c062670 con 0x7f8978039330 2024-04-25T00:25:16.226 INFO:teuthology.orchestra.run.smithi032.stdout:Created osd(s) 0 on host 'smithi032' 2024-04-25T00:25:16.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:25:16.227+0000 7f89925e6700 1 -- 172.21.15.32:0/443037048 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8978039330 msgr2=0x7f897803b7f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:25:16.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:25:16.227+0000 7f89925e6700 1 --2- 172.21.15.32:0/443037048 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8978039330 0x7f897803b7f0 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f8974009c60 tx=0x7f8974009260 comp rx=0 tx=0).stop 2024-04-25T00:25:16.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:25:16.227+0000 7f89925e6700 1 -- 172.21.15.32:0/443037048 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f898c1022b0 msgr2=0x7f898c1984e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:25:16.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:25:16.227+0000 7f89925e6700 1 --2- 172.21.15.32:0/443037048 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f898c1022b0 0x7f898c1984e0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f897c00f710 tx=0x7f897c009580 comp rx=0 tx=0).stop 2024-04-25T00:25:16.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:25:16.227+0000 7f89925e6700 1 -- 172.21.15.32:0/443037048 shutdown_connections 2024-04-25T00:25:16.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:25:16.227+0000 7f89925e6700 1 --2- 172.21.15.32:0/443037048 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8978039330 0x7f897803b7f0 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:25:16.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:25:16.227+0000 7f89925e6700 1 --2- 172.21.15.32:0/443037048 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f898c103e50 0x7f898c19faa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:25:16.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:25:16.227+0000 7f89925e6700 1 --2- 172.21.15.32:0/443037048 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f898c1022b0 0x7f898c1984e0 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:25:16.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:25:16.227+0000 7f89925e6700 1 --2- 172.21.15.32:0/443037048 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f898c1034b0 0x7f898c198a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:25:16.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:25:16.227+0000 7f89925e6700 1 -- 172.21.15.32:0/443037048 >> 172.21.15.32:0/443037048 conn(0x7f898c0fd850 msgr2=0x7f898c1067d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:25:16.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:25:16.227+0000 7f89925e6700 1 -- 172.21.15.32:0/443037048 shutdown_connections 2024-04-25T00:25:16.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:25:16.227+0000 7f89925e6700 1 -- 172.21.15.32:0/443037048 wait complete. 2024-04-25T00:25:16.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:15 smithi121 ceph-mon[32231]: pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:16.862 DEBUG:teuthology.orchestra.run.smithi032:osd.0> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.0.service 2024-04-25T00:25:16.865 INFO:tasks.cephadm:Deploying osd.1 on smithi121 with /dev/vg_nvme/lv_4... 2024-04-25T00:25:16.865 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-25T00:25:17.106 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.b/config 2024-04-25T00:25:17.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:17 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:17.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:17 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:17.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:17 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:17.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:17 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:17.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:17 smithi032 ceph-mon[29357]: from='osd.0 [v2:172.21.15.32:6802/3984695193,v1:172.21.15.32:6803/3984695193]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-25T00:25:17.510 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:17 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:17.510 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:17 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:17.510 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:17 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:17.510 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:17 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:17.510 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:17 smithi121 ceph-mon[32231]: from='osd.0 [v2:172.21.15.32:6802/3984695193,v1:172.21.15.32:6803/3984695193]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-25T00:25:17.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:17 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:17.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:17 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:17.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:17 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:17.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:17 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:17.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:17 smithi130 ceph-mon[32525]: from='osd.0 [v2:172.21.15.32:6802/3984695193,v1:172.21.15.32:6803/3984695193]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-25T00:25:18.421 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:18 smithi121 ceph-mon[32231]: pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:18.421 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:18 smithi121 ceph-mon[32231]: from='osd.0 [v2:172.21.15.32:6802/3984695193,v1:172.21.15.32:6803/3984695193]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-25T00:25:18.421 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:18 smithi121 ceph-mon[32231]: osdmap e6: 1 total, 0 up, 1 in 2024-04-25T00:25:18.421 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:18 smithi121 ceph-mon[32231]: from='osd.0 [v2:172.21.15.32:6802/3984695193,v1:172.21.15.32:6803/3984695193]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-04-25T00:25:18.421 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:18 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T00:25:18.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:18 smithi032 ceph-mon[29357]: pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:18.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:18 smithi032 ceph-mon[29357]: from='osd.0 [v2:172.21.15.32:6802/3984695193,v1:172.21.15.32:6803/3984695193]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-25T00:25:18.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:18 smithi032 ceph-mon[29357]: osdmap e6: 1 total, 0 up, 1 in 2024-04-25T00:25:18.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:18 smithi032 ceph-mon[29357]: from='osd.0 [v2:172.21.15.32:6802/3984695193,v1:172.21.15.32:6803/3984695193]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-04-25T00:25:18.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:18 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T00:25:18.499 INFO:journalctl@ceph.osd.0.smithi032.stdout:Apr 25 00:25:18 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-0[44152]: 2024-04-25T00:25:18.302+0000 7f6b610b4700 -1 osd.0 0 waiting for initial osdmap 2024-04-25T00:25:18.499 INFO:journalctl@ceph.osd.0.smithi032.stdout:Apr 25 00:25:18 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-0[44152]: 2024-04-25T00:25:18.309+0000 7f6b5cea7700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T00:25:18.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:18 smithi130 ceph-mon[32525]: pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:18.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:18 smithi130 ceph-mon[32525]: from='osd.0 [v2:172.21.15.32:6802/3984695193,v1:172.21.15.32:6803/3984695193]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-25T00:25:18.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:18 smithi130 ceph-mon[32525]: osdmap e6: 1 total, 0 up, 1 in 2024-04-25T00:25:18.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:18 smithi130 ceph-mon[32525]: from='osd.0 [v2:172.21.15.32:6802/3984695193,v1:172.21.15.32:6803/3984695193]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-04-25T00:25:18.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:18 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T00:25:18.973 INFO:teuthology.orchestra.run.smithi121.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-25T00:25:18.974 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-25T00:25:18.974 INFO:teuthology.orchestra.run.smithi121.stderr: stderr: 10+0 records in 2024-04-25T00:25:18.974 INFO:teuthology.orchestra.run.smithi121.stderr:10+0 records out 2024-04-25T00:25:18.974 INFO:teuthology.orchestra.run.smithi121.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0130973 s, 801 MB/s 2024-04-25T00:25:18.974 INFO:teuthology.orchestra.run.smithi121.stderr:--> Zapping successful for: 2024-04-25T00:25:19.537 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:19 smithi121 ceph-mon[32231]: from='osd.0 [v2:172.21.15.32:6802/3984695193,v1:172.21.15.32:6803/3984695193]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-04-25T00:25:19.537 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:19 smithi121 ceph-mon[32231]: osdmap e7: 1 total, 0 up, 1 in 2024-04-25T00:25:19.537 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:19 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T00:25:19.538 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:19 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T00:25:19.564 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi121:vg_nvme/lv_4 2024-04-25T00:25:19.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:19 smithi130 ceph-mon[32525]: from='osd.0 [v2:172.21.15.32:6802/3984695193,v1:172.21.15.32:6803/3984695193]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-04-25T00:25:19.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:19 smithi130 ceph-mon[32525]: osdmap e7: 1 total, 0 up, 1 in 2024-04-25T00:25:19.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:19 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T00:25:19.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:19 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T00:25:19.677 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:19 smithi032 ceph-mon[29357]: from='osd.0 [v2:172.21.15.32:6802/3984695193,v1:172.21.15.32:6803/3984695193]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-04-25T00:25:19.677 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:19 smithi032 ceph-mon[29357]: osdmap e7: 1 total, 0 up, 1 in 2024-04-25T00:25:19.677 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:19 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T00:25:19.677 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:19 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T00:25:19.807 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.b/config 2024-04-25T00:25:20.586 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:20 smithi121 ceph-mon[32231]: purged_snaps scrub starts 2024-04-25T00:25:20.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:20 smithi121 ceph-mon[32231]: purged_snaps scrub ok 2024-04-25T00:25:20.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:20 smithi121 ceph-mon[32231]: pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:20.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:20 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T00:25:20.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:20 smithi121 ceph-mon[32231]: osd.0 [v2:172.21.15.32:6802/3984695193,v1:172.21.15.32:6803/3984695193] boot 2024-04-25T00:25:20.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:20 smithi121 ceph-mon[32231]: osdmap e8: 1 total, 1 up, 1 in 2024-04-25T00:25:20.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:20 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T00:25:20.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:20 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:20.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:20 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:20.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:20 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:25:20.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:20 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:20.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:20 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:20.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:20 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:25:20.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:20 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:20.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:20 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:25:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:20 smithi130 ceph-mon[32525]: purged_snaps scrub starts 2024-04-25T00:25:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:20 smithi130 ceph-mon[32525]: purged_snaps scrub ok 2024-04-25T00:25:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:20 smithi130 ceph-mon[32525]: pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:20 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T00:25:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:20 smithi130 ceph-mon[32525]: osd.0 [v2:172.21.15.32:6802/3984695193,v1:172.21.15.32:6803/3984695193] boot 2024-04-25T00:25:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:20 smithi130 ceph-mon[32525]: osdmap e8: 1 total, 1 up, 1 in 2024-04-25T00:25:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:20 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T00:25:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:20 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:20 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:20 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:25:20.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:20 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:20.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:20 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:20.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:20 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:25:20.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:20 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:20.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:20 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:25:20.647 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:20 smithi032 ceph-mon[29357]: purged_snaps scrub starts 2024-04-25T00:25:20.647 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:20 smithi032 ceph-mon[29357]: purged_snaps scrub ok 2024-04-25T00:25:20.647 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:20 smithi032 ceph-mon[29357]: pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T00:25:20.647 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:20 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T00:25:20.647 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:20 smithi032 ceph-mon[29357]: osd.0 [v2:172.21.15.32:6802/3984695193,v1:172.21.15.32:6803/3984695193] boot 2024-04-25T00:25:20.647 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:20 smithi032 ceph-mon[29357]: osdmap e8: 1 total, 1 up, 1 in 2024-04-25T00:25:20.647 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:20 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T00:25:20.647 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:20 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:20.647 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:20 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:20.647 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:20 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:25:20.647 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:20 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:20.647 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:20 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:20.647 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:20 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:25:20.647 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:20 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:20.648 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:20 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:25:21.395 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.392+0000 7fb75a57d700 1 -- 172.21.15.121:0/3832112439 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb7541011d0 msgr2=0x7fb7541015f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:25:21.395 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.392+0000 7fb75a57d700 1 --2- 172.21.15.121:0/3832112439 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb7541011d0 0x7fb7541015f0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7fb7440099c0 tx=0x7fb744009cd0 comp rx=0 tx=0).stop 2024-04-25T00:25:21.395 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.392+0000 7fb75a57d700 1 -- 172.21.15.121:0/3832112439 shutdown_connections 2024-04-25T00:25:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.392+0000 7fb75a57d700 1 --2- 172.21.15.121:0/3832112439 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb754102d70 0x7fb754109620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:25:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.392+0000 7fb75a57d700 1 --2- 172.21.15.121:0/3832112439 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb7541011d0 0x7fb7541015f0 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:25:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.392+0000 7fb75a57d700 1 --2- 172.21.15.121:0/3832112439 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb7541023d0 0x7fb754102830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:25:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.392+0000 7fb75a57d700 1 -- 172.21.15.121:0/3832112439 >> 172.21.15.121:0/3832112439 conn(0x7fb7540fc790 msgr2=0x7fb7540febb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:25:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.392+0000 7fb75a57d700 1 -- 172.21.15.121:0/3832112439 shutdown_connections 2024-04-25T00:25:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.393+0000 7fb75a57d700 1 -- 172.21.15.121:0/3832112439 wait complete. 2024-04-25T00:25:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.393+0000 7fb75a57d700 1 Processor -- start 2024-04-25T00:25:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.393+0000 7fb75a57d700 1 -- start start 2024-04-25T00:25:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.393+0000 7fb75a57d700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb7541011d0 0x7fb754198560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:25:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.393+0000 7fb75a57d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb7541023d0 0x7fb754198aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:25:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.393+0000 7fb75a57d700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb754102d70 0x7fb75419fb20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:25:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.393+0000 7fb75a57d700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fb75410cdb0 con 0x7fb7541011d0 2024-04-25T00:25:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.393+0000 7fb75a57d700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fb75410cc30 con 0x7fb754102d70 2024-04-25T00:25:21.397 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.393+0000 7fb75a57d700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fb75410cf30 con 0x7fb7541023d0 2024-04-25T00:25:21.397 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.394+0000 7fb758b1a700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb754102d70 0x7fb75419fb20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:25:21.397 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.394+0000 7fb758b1a700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb754102d70 0x7fb75419fb20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.121:38360/0 (socket says 172.21.15.121:38360) 2024-04-25T00:25:21.397 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.394+0000 7fb758b1a700 1 -- 172.21.15.121:0/2546127475 learned_addr learned my addr 172.21.15.121:0/2546127475 (peer_addr_for_me v2:172.21.15.121:0/0) 2024-04-25T00:25:21.397 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.394+0000 7fb753fff700 1 --2- 172.21.15.121:0/2546127475 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb7541011d0 0x7fb754198560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:25:21.397 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.394+0000 7fb7537fe700 1 --2- 172.21.15.121:0/2546127475 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb7541023d0 0x7fb754198aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:25:21.397 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.394+0000 7fb758b1a700 1 -- 172.21.15.121:0/2546127475 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb7541023d0 msgr2=0x7fb754198aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:25:21.397 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.394+0000 7fb758b1a700 1 --2- 172.21.15.121:0/2546127475 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb7541023d0 0x7fb754198aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:25:21.397 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.394+0000 7fb758b1a700 1 -- 172.21.15.121:0/2546127475 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb7541011d0 msgr2=0x7fb754198560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:25:21.397 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.394+0000 7fb758b1a700 1 --2- 172.21.15.121:0/2546127475 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb7541011d0 0x7fb754198560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:25:21.397 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.394+0000 7fb758b1a700 1 -- 172.21.15.121:0/2546127475 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb7541a01c0 con 0x7fb754102d70 2024-04-25T00:25:21.397 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.394+0000 7fb7537fe700 1 --2- 172.21.15.121:0/2546127475 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb7541023d0 0x7fb754198aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:25:21.397 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.394+0000 7fb753fff700 1 --2- 172.21.15.121:0/2546127475 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb7541011d0 0x7fb754198560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:25:21.397 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.394+0000 7fb758b1a700 1 --2- 172.21.15.121:0/2546127475 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb754102d70 0x7fb75419fb20 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fb74c00ada0 tx=0x7fb74c00c330 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:25:21.397 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.394+0000 7fb7517fa700 1 -- 172.21.15.121:0/2546127475 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb74c017020 con 0x7fb754102d70 2024-04-25T00:25:21.397 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.394+0000 7fb75a57d700 1 -- 172.21.15.121:0/2546127475 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb7541a04b0 con 0x7fb754102d70 2024-04-25T00:25:21.398 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.394+0000 7fb7517fa700 1 -- 172.21.15.121:0/2546127475 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb74c004bb0 con 0x7fb754102d70 2024-04-25T00:25:21.398 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.394+0000 7fb7517fa700 1 -- 172.21.15.121:0/2546127475 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb74c005520 con 0x7fb754102d70 2024-04-25T00:25:21.398 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.394+0000 7fb75a57d700 1 -- 172.21.15.121:0/2546127475 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb7541a09a0 con 0x7fb754102d70 2024-04-25T00:25:21.401 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.395+0000 7fb75a57d700 1 -- 172.21.15.121:0/2546127475 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb754068f20 con 0x7fb754102d70 2024-04-25T00:25:21.401 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.396+0000 7fb7517fa700 1 -- 172.21.15.121:0/2546127475 <== mon.2 v2:172.21.15.121:3300/0 4 ==== mgrmap(e 15) ==== 137777+0+0 (secure 0 0 0) 0x7fb74c0056c0 con 0x7fb754102d70 2024-04-25T00:25:21.402 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.397+0000 7fb7517fa700 1 --2- 172.21.15.121:0/2546127475 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb73c0a2b90 0x7fb73c0a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:25:21.402 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.397+0000 7fb7517fa700 1 -- 172.21.15.121:0/2546127475 <== mon.2 v2:172.21.15.121:3300/0 5 ==== osd_map(9..9 src has 1..9) ==== 1767+0+0 (secure 0 0 0) 0x7fb74c0ccca0 con 0x7fb754102d70 2024-04-25T00:25:21.402 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.397+0000 7fb753fff700 1 --2- 172.21.15.121:0/2546127475 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb73c0a2b90 0x7fb73c0a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:25:21.402 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.397+0000 7fb753fff700 1 --2- 172.21.15.121:0/2546127475 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb73c0a2b90 0x7fb73c0a5050 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7fb744000c00 tx=0x7fb744005fd0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:25:21.402 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.401+0000 7fb7517fa700 1 -- 172.21.15.121:0/2546127475 <== mon.2 v2:172.21.15.121:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fb74c021020 con 0x7fb754102d70 2024-04-25T00:25:21.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:21 smithi121 ceph-mon[32231]: Detected new or changed devices on smithi032 2024-04-25T00:25:21.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:21 smithi121 ceph-mon[32231]: Adjusting osd_memory_target on smithi032 to 17093M 2024-04-25T00:25:21.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:21 smithi121 ceph-mon[32231]: osdmap e9: 1 total, 1 up, 1 in 2024-04-25T00:25:21.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:21 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:21.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:21 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:25:21.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:21 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:21.610 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:21.606+0000 7fb75a57d700 1 -- 172.21.15.121:0/2546127475 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_4", "target": ["mon-mgr", ""]}) -- 0x7fb7540fe9a0 con 0x7fb73c0a2b90 2024-04-25T00:25:21.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:21 smithi130 ceph-mon[32525]: Detected new or changed devices on smithi032 2024-04-25T00:25:21.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:21 smithi130 ceph-mon[32525]: Adjusting osd_memory_target on smithi032 to 17093M 2024-04-25T00:25:21.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:21 smithi130 ceph-mon[32525]: osdmap e9: 1 total, 1 up, 1 in 2024-04-25T00:25:21.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:21 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:21.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:21 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:25:21.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:21 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:21.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:21 smithi032 ceph-mon[29357]: Detected new or changed devices on smithi032 2024-04-25T00:25:21.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:21 smithi032 ceph-mon[29357]: Adjusting osd_memory_target on smithi032 to 17093M 2024-04-25T00:25:21.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:21 smithi032 ceph-mon[29357]: osdmap e9: 1 total, 1 up, 1 in 2024-04-25T00:25:21.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:21 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:21.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:21 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:25:21.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:21 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:22.508 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:22 smithi121 ceph-mon[32231]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:22.508 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:22 smithi121 ceph-mon[32231]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:25:22.508 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:22 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:25:22.508 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:22 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:25:22.508 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:22 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:22.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:22 smithi130 ceph-mon[32525]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:22.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:22 smithi130 ceph-mon[32525]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:25:22.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:22 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:25:22.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:22 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:25:22.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:22 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:22.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:22 smithi032 ceph-mon[29357]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:22.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:22 smithi032 ceph-mon[29357]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:25:22.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:22 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:25:22.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:22 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:25:22.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:22 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:24.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:24 smithi121 ceph-mon[32231]: pgmap v49: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:24.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:24 smithi121 ceph-mon[32231]: from='client.? 172.21.15.121:0/1736850678' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e7ea9910-062b-4801-b1a5-a90a56d73b66"}]: dispatch 2024-04-25T00:25:24.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:24 smithi121 ceph-mon[32231]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e7ea9910-062b-4801-b1a5-a90a56d73b66"}]: dispatch 2024-04-25T00:25:24.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:24 smithi121 ceph-mon[32231]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e7ea9910-062b-4801-b1a5-a90a56d73b66"}]': finished 2024-04-25T00:25:24.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:24 smithi121 ceph-mon[32231]: osdmap e10: 2 total, 1 up, 2 in 2024-04-25T00:25:24.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:24 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T00:25:24.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:24 smithi130 ceph-mon[32525]: pgmap v49: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:24.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:24 smithi130 ceph-mon[32525]: from='client.? 172.21.15.121:0/1736850678' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e7ea9910-062b-4801-b1a5-a90a56d73b66"}]: dispatch 2024-04-25T00:25:24.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:24 smithi130 ceph-mon[32525]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e7ea9910-062b-4801-b1a5-a90a56d73b66"}]: dispatch 2024-04-25T00:25:24.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:24 smithi130 ceph-mon[32525]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e7ea9910-062b-4801-b1a5-a90a56d73b66"}]': finished 2024-04-25T00:25:24.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:24 smithi130 ceph-mon[32525]: osdmap e10: 2 total, 1 up, 2 in 2024-04-25T00:25:24.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:24 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T00:25:24.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:24 smithi032 ceph-mon[29357]: pgmap v49: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:24.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:24 smithi032 ceph-mon[29357]: from='client.? 172.21.15.121:0/1736850678' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e7ea9910-062b-4801-b1a5-a90a56d73b66"}]: dispatch 2024-04-25T00:25:24.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:24 smithi032 ceph-mon[29357]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e7ea9910-062b-4801-b1a5-a90a56d73b66"}]: dispatch 2024-04-25T00:25:24.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:24 smithi032 ceph-mon[29357]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e7ea9910-062b-4801-b1a5-a90a56d73b66"}]': finished 2024-04-25T00:25:24.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:24 smithi032 ceph-mon[29357]: osdmap e10: 2 total, 1 up, 2 in 2024-04-25T00:25:24.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:24 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T00:25:25.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:25 smithi121 ceph-mon[32231]: from='client.? 172.21.15.121:0/3861549097' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:25:25.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:25 smithi130 ceph-mon[32525]: from='client.? 172.21.15.121:0/3861549097' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:25:25.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:25 smithi032 ceph-mon[29357]: from='client.? 172.21.15.121:0/3861549097' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:25:26.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:26 smithi121 ceph-mon[32231]: pgmap v51: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:26.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:26 smithi130 ceph-mon[32525]: pgmap v51: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:26.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:26 smithi032 ceph-mon[29357]: pgmap v51: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:28.503 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:28 smithi121 ceph-mon[32231]: pgmap v52: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:28.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:28 smithi130 ceph-mon[32525]: pgmap v52: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:28.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:28 smithi032 ceph-mon[29357]: pgmap v52: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:30.382 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:30 smithi121 ceph-mon[32231]: pgmap v53: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:30.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:30 smithi032 ceph-mon[29357]: pgmap v53: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:30.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:30 smithi130 ceph-mon[32525]: pgmap v53: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:32.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:32 smithi121 ceph-mon[32231]: pgmap v54: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:32.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:32 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-25T00:25:32.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:32 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:32.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:32 smithi121 ceph-mon[32231]: Deploying daemon osd.1 on smithi121 2024-04-25T00:25:32.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:32 smithi130 ceph-mon[32525]: pgmap v54: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:32.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:32 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-25T00:25:32.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:32 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:32.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:32 smithi130 ceph-mon[32525]: Deploying daemon osd.1 on smithi121 2024-04-25T00:25:32.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:32 smithi032 ceph-mon[29357]: pgmap v54: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:32.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:32 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-25T00:25:32.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:32 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:32.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:32 smithi032 ceph-mon[29357]: Deploying daemon osd.1 on smithi121 2024-04-25T00:25:34.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:34 smithi121 ceph-mon[32231]: pgmap v55: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:34.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:34 smithi130 ceph-mon[32525]: pgmap v55: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:34.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:34 smithi032 ceph-mon[29357]: pgmap v55: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:36.488 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:36 smithi121 ceph-mon[32231]: pgmap v56: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:36.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:36 smithi130 ceph-mon[32525]: pgmap v56: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:36.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:36 smithi032 ceph-mon[29357]: pgmap v56: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:37.804 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:37 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:37.804 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:37 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:37.804 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:37 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:25:37.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:37.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:37.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:25:37.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:37 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:37.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:37 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:37.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:37 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:25:38.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:38 smithi121 ceph-mon[32231]: pgmap v57: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:38.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:38 smithi130 ceph-mon[32525]: pgmap v57: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:38.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:38 smithi032 ceph-mon[29357]: pgmap v57: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:39.574 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:39.574+0000 7fb7517fa700 1 -- 172.21.15.121:0/2546127475 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7fb7540fe9a0 con 0x7fb73c0a2b90 2024-04-25T00:25:39.575 INFO:teuthology.orchestra.run.smithi121.stdout:Created osd(s) 1 on host 'smithi121' 2024-04-25T00:25:39.578 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:39.576+0000 7fb75a57d700 1 -- 172.21.15.121:0/2546127475 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb73c0a2b90 msgr2=0x7fb73c0a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:25:39.578 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:39.576+0000 7fb75a57d700 1 --2- 172.21.15.121:0/2546127475 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb73c0a2b90 0x7fb73c0a5050 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7fb744000c00 tx=0x7fb744005fd0 comp rx=0 tx=0).stop 2024-04-25T00:25:39.578 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:39.576+0000 7fb75a57d700 1 -- 172.21.15.121:0/2546127475 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb754102d70 msgr2=0x7fb75419fb20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:25:39.579 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:39.576+0000 7fb75a57d700 1 --2- 172.21.15.121:0/2546127475 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb754102d70 0x7fb75419fb20 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fb74c00ada0 tx=0x7fb74c00c330 comp rx=0 tx=0).stop 2024-04-25T00:25:39.579 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:39.576+0000 7fb75a57d700 1 -- 172.21.15.121:0/2546127475 shutdown_connections 2024-04-25T00:25:39.579 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:39.576+0000 7fb75a57d700 1 --2- 172.21.15.121:0/2546127475 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb73c0a2b90 0x7fb73c0a5050 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:25:39.579 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:39.576+0000 7fb75a57d700 1 --2- 172.21.15.121:0/2546127475 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb754102d70 0x7fb75419fb20 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:25:39.579 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:39.576+0000 7fb75a57d700 1 --2- 172.21.15.121:0/2546127475 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb7541011d0 0x7fb754198560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:25:39.579 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:39.576+0000 7fb75a57d700 1 --2- 172.21.15.121:0/2546127475 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb7541023d0 0x7fb754198aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:25:39.579 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:39.576+0000 7fb75a57d700 1 -- 172.21.15.121:0/2546127475 >> 172.21.15.121:0/2546127475 conn(0x7fb7540fc790 msgr2=0x7fb7540fe290 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:25:39.579 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:39.576+0000 7fb75a57d700 1 -- 172.21.15.121:0/2546127475 shutdown_connections 2024-04-25T00:25:39.579 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:25:39.576+0000 7fb75a57d700 1 -- 172.21.15.121:0/2546127475 wait complete. 2024-04-25T00:25:40.262 DEBUG:teuthology.orchestra.run.smithi121:osd.1> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.1.service 2024-04-25T00:25:40.264 INFO:tasks.cephadm:Deploying osd.2 on smithi130 with /dev/vg_nvme/lv_4... 2024-04-25T00:25:40.264 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-25T00:25:40.512 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.c/config 2024-04-25T00:25:40.736 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:40 smithi121 ceph-mon[32231]: pgmap v58: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:40.736 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:40 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:40.737 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:40 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:40.737 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:40 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:40.737 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:40 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:40.737 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:40 smithi121 ceph-mon[32231]: from='osd.1 [v2:172.21.15.121:6800/431442351,v1:172.21.15.121:6801/431442351]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-25T00:25:40.737 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:40 smithi121 ceph-mon[32231]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-25T00:25:40.737 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:25:40 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-1[36801]: 2024-04-25T00:25:40.434+0000 7f97915697c0 -1 osd.1 0 log_to_monitors true 2024-04-25T00:25:40.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:40 smithi130 ceph-mon[32525]: pgmap v58: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:40.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:40 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:40.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:40 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:40.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:40 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:40.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:40 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:40.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:40 smithi130 ceph-mon[32525]: from='osd.1 [v2:172.21.15.121:6800/431442351,v1:172.21.15.121:6801/431442351]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-25T00:25:40.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:40 smithi130 ceph-mon[32525]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-25T00:25:40.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:40 smithi032 ceph-mon[29357]: pgmap v58: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:40.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:40 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:40.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:40 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:40.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:40 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:40.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:40 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:40.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:40 smithi032 ceph-mon[29357]: from='osd.1 [v2:172.21.15.121:6800/431442351,v1:172.21.15.121:6801/431442351]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-25T00:25:40.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:40 smithi032 ceph-mon[29357]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-25T00:25:41.784 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:41 smithi121 ceph-mon[32231]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-25T00:25:41.784 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:41 smithi121 ceph-mon[32231]: from='osd.1 [v2:172.21.15.121:6800/431442351,v1:172.21.15.121:6801/431442351]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-25T00:25:41.784 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:41 smithi121 ceph-mon[32231]: osdmap e11: 2 total, 1 up, 2 in 2024-04-25T00:25:41.784 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:41 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T00:25:41.784 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:41 smithi121 ceph-mon[32231]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-25T00:25:41.785 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:25:41 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-1[36801]: 2024-04-25T00:25:41.635+0000 7f9786bce700 -1 osd.1 0 waiting for initial osdmap 2024-04-25T00:25:41.785 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:25:41 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-1[36801]: 2024-04-25T00:25:41.640+0000 7f97829c1700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T00:25:41.838 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:41 smithi130 ceph-mon[32525]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-25T00:25:41.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:41 smithi130 ceph-mon[32525]: from='osd.1 [v2:172.21.15.121:6800/431442351,v1:172.21.15.121:6801/431442351]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-25T00:25:41.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:41 smithi130 ceph-mon[32525]: osdmap e11: 2 total, 1 up, 2 in 2024-04-25T00:25:41.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:41 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T00:25:41.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:41 smithi130 ceph-mon[32525]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-25T00:25:41.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:41 smithi032 ceph-mon[29357]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-25T00:25:41.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:41 smithi032 ceph-mon[29357]: from='osd.1 [v2:172.21.15.121:6800/431442351,v1:172.21.15.121:6801/431442351]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-25T00:25:41.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:41 smithi032 ceph-mon[29357]: osdmap e11: 2 total, 1 up, 2 in 2024-04-25T00:25:41.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:41 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T00:25:41.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:41 smithi032 ceph-mon[29357]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-25T00:25:42.409 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-25T00:25:42.409 INFO:teuthology.orchestra.run.smithi130.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-25T00:25:42.409 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10+0 records in 2024-04-25T00:25:42.410 INFO:teuthology.orchestra.run.smithi130.stderr:10+0 records out 2024-04-25T00:25:42.410 INFO:teuthology.orchestra.run.smithi130.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0147892 s, 709 MB/s 2024-04-25T00:25:42.410 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 2024-04-25T00:25:42.410 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping successful for: 2024-04-25T00:25:42.722 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:42 smithi130 ceph-mon[32525]: pgmap v60: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:42.722 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:42 smithi130 ceph-mon[32525]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-04-25T00:25:42.722 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:42 smithi130 ceph-mon[32525]: osdmap e12: 2 total, 1 up, 2 in 2024-04-25T00:25:42.722 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:42 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T00:25:42.722 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:42 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T00:25:42.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:42 smithi121 ceph-mon[32231]: pgmap v60: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:42.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:42 smithi121 ceph-mon[32231]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-04-25T00:25:42.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:42 smithi121 ceph-mon[32231]: osdmap e12: 2 total, 1 up, 2 in 2024-04-25T00:25:42.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:42 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T00:25:42.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:42 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T00:25:42.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:42 smithi032 ceph-mon[29357]: pgmap v60: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-25T00:25:42.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:42 smithi032 ceph-mon[29357]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-04-25T00:25:42.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:42 smithi032 ceph-mon[29357]: osdmap e12: 2 total, 1 up, 2 in 2024-04-25T00:25:42.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T00:25:42.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T00:25:43.001 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi130:vg_nvme/lv_4 2024-04-25T00:25:43.234 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.c/config 2024-04-25T00:25:43.953 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:43 smithi130 ceph-mon[32525]: purged_snaps scrub starts 2024-04-25T00:25:43.953 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:43 smithi130 ceph-mon[32525]: purged_snaps scrub ok 2024-04-25T00:25:43.953 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:43 smithi130 ceph-mon[32525]: osd.1 [v2:172.21.15.121:6800/431442351,v1:172.21.15.121:6801/431442351] boot 2024-04-25T00:25:43.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:43 smithi130 ceph-mon[32525]: osdmap e13: 2 total, 2 up, 2 in 2024-04-25T00:25:43.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T00:25:43.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:43.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:43.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:25:43.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:43.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:43.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:25:43.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:43.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:25:43.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:43.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:25:43.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:43.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:43 smithi032 ceph-mon[29357]: purged_snaps scrub starts 2024-04-25T00:25:43.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:43 smithi032 ceph-mon[29357]: purged_snaps scrub ok 2024-04-25T00:25:43.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:43 smithi032 ceph-mon[29357]: osd.1 [v2:172.21.15.121:6800/431442351,v1:172.21.15.121:6801/431442351] boot 2024-04-25T00:25:43.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:43 smithi032 ceph-mon[29357]: osdmap e13: 2 total, 2 up, 2 in 2024-04-25T00:25:43.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T00:25:43.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:43.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:43.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:25:43.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:43.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:43.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:25:43.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:43.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:25:43.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:43.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:25:43.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:44.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:43 smithi121 ceph-mon[32231]: purged_snaps scrub starts 2024-04-25T00:25:44.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:43 smithi121 ceph-mon[32231]: purged_snaps scrub ok 2024-04-25T00:25:44.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:43 smithi121 ceph-mon[32231]: osd.1 [v2:172.21.15.121:6800/431442351,v1:172.21.15.121:6801/431442351] boot 2024-04-25T00:25:44.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:43 smithi121 ceph-mon[32231]: osdmap e13: 2 total, 2 up, 2 in 2024-04-25T00:25:44.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T00:25:44.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:44.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:44.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:25:44.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:44.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:44.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:25:44.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:44.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:25:44.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:44.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:25:44.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:25:44.740 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.728+0000 7f387d5a6700 1 -- 172.21.15.130:0/1468018574 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3878107f70 msgr2=0x7f387810a3e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:25:44.740 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.728+0000 7f387d5a6700 1 --2- 172.21.15.130:0/1468018574 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3878107f70 0x7f387810a3e0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f3868009990 tx=0x7f3868009ca0 comp rx=0 tx=0).stop 2024-04-25T00:25:44.740 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.728+0000 7f387d5a6700 1 -- 172.21.15.130:0/1468018574 shutdown_connections 2024-04-25T00:25:44.740 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.728+0000 7f387d5a6700 1 --2- 172.21.15.130:0/1468018574 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3878107f70 0x7f387810a3e0 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:25:44.740 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.728+0000 7f387d5a6700 1 --2- 172.21.15.130:0/1468018574 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3878102d10 0x7f3878105100 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:25:44.740 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.728+0000 7f387d5a6700 1 --2- 172.21.15.130:0/1468018574 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3878105640 0x7f3878107a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:25:44.740 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.728+0000 7f387d5a6700 1 -- 172.21.15.130:0/1468018574 >> 172.21.15.130:0/1468018574 conn(0x7f38780fc720 msgr2=0x7f38780feb60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:25:44.740 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.728+0000 7f387d5a6700 1 -- 172.21.15.130:0/1468018574 shutdown_connections 2024-04-25T00:25:44.740 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.728+0000 7f387d5a6700 1 -- 172.21.15.130:0/1468018574 wait complete. 2024-04-25T00:25:44.740 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.728+0000 7f387d5a6700 1 Processor -- start 2024-04-25T00:25:44.740 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.729+0000 7f387d5a6700 1 -- start start 2024-04-25T00:25:44.741 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.729+0000 7f387d5a6700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3878102d10 0x7f3878102670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:25:44.741 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.729+0000 7f387d5a6700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3878105640 0x7f3878100cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:25:44.741 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.729+0000 7f387d5a6700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3878107f70 0x7f3878101200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:25:44.741 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.729+0000 7f387d5a6700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f387810df00 con 0x7f3878107f70 2024-04-25T00:25:44.741 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.729+0000 7f387d5a6700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f387810dd80 con 0x7f3878105640 2024-04-25T00:25:44.741 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.729+0000 7f387d5a6700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f387810e080 con 0x7f3878102d10 2024-04-25T00:25:44.741 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.729+0000 7f3876ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3878102d10 0x7f3878102670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:25:44.741 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.729+0000 7f3876ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3878102d10 0x7f3878102670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.130:57096/0 (socket says 172.21.15.130:57096) 2024-04-25T00:25:44.741 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.729+0000 7f3876ffd700 1 -- 172.21.15.130:0/2328629453 learned_addr learned my addr 172.21.15.130:0/2328629453 (peer_addr_for_me v2:172.21.15.130:0/0) 2024-04-25T00:25:44.741 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.729+0000 7f38777fe700 1 --2- 172.21.15.130:0/2328629453 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3878107f70 0x7f3878101200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:25:44.741 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.729+0000 7f38767fc700 1 --2- 172.21.15.130:0/2328629453 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3878105640 0x7f3878100cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:25:44.741 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.729+0000 7f3876ffd700 1 -- 172.21.15.130:0/2328629453 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3878105640 msgr2=0x7f3878100cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:25:44.741 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.729+0000 7f3876ffd700 1 --2- 172.21.15.130:0/2328629453 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3878105640 0x7f3878100cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:25:44.742 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.729+0000 7f3876ffd700 1 -- 172.21.15.130:0/2328629453 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3878107f70 msgr2=0x7f3878101200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:25:44.742 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.729+0000 7f3876ffd700 1 --2- 172.21.15.130:0/2328629453 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3878107f70 0x7f3878101200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:25:44.742 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.729+0000 7f3876ffd700 1 -- 172.21.15.130:0/2328629453 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3878101ab0 con 0x7f3878102d10 2024-04-25T00:25:44.742 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.729+0000 7f38767fc700 1 --2- 172.21.15.130:0/2328629453 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3878105640 0x7f3878100cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:25:44.742 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.729+0000 7f3876ffd700 1 --2- 172.21.15.130:0/2328629453 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3878102d10 0x7f3878102670 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f387000ba50 tx=0x7f387000bd60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:25:44.742 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.730+0000 7f385ffff700 1 -- 172.21.15.130:0/2328629453 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3870019070 con 0x7f3878102d10 2024-04-25T00:25:44.742 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.730+0000 7f387d5a6700 1 -- 172.21.15.130:0/2328629453 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f38781a2680 con 0x7f3878102d10 2024-04-25T00:25:44.742 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.730+0000 7f385ffff700 1 -- 172.21.15.130:0/2328629453 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3870014440 con 0x7f3878102d10 2024-04-25T00:25:44.742 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.730+0000 7f385ffff700 1 -- 172.21.15.130:0/2328629453 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f38700075d0 con 0x7f3878102d10 2024-04-25T00:25:44.742 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.730+0000 7f387d5a6700 1 -- 172.21.15.130:0/2328629453 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f38781a2bc0 con 0x7f3878102d10 2024-04-25T00:25:44.742 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.731+0000 7f387d5a6700 1 -- 172.21.15.130:0/2328629453 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f3878068f20 con 0x7f3878102d10 2024-04-25T00:25:44.742 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.732+0000 7f385ffff700 1 -- 172.21.15.130:0/2328629453 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 15) ==== 137777+0+0 (secure 0 0 0) 0x7f387001c070 con 0x7f3878102d10 2024-04-25T00:25:44.742 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.732+0000 7f385ffff700 1 --2- 172.21.15.130:0/2328629453 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f38600a2b90 0x7f38600a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:25:44.743 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.732+0000 7f385ffff700 1 -- 172.21.15.130:0/2328629453 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(14..14 src has 1..14) ==== 2309+0+0 (secure 0 0 0) 0x7f38700cd050 con 0x7f3878102d10 2024-04-25T00:25:44.743 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.732+0000 7f38767fc700 1 --2- 172.21.15.130:0/2328629453 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f38600a2b90 0x7f38600a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:25:44.743 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.733+0000 7f38767fc700 1 --2- 172.21.15.130:0/2328629453 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f38600a2b90 0x7f38600a5050 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f386c005950 tx=0x7f386c0058e0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:25:44.743 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.736+0000 7f385ffff700 1 -- 172.21.15.130:0/2328629453 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f387009b800 con 0x7f3878102d10 2024-04-25T00:25:44.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:44 smithi130 ceph-mon[32525]: pgmap v63: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:25:44.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:44 smithi130 ceph-mon[32525]: Detected new or changed devices on smithi121 2024-04-25T00:25:44.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:44 smithi130 ceph-mon[32525]: Adjusting osd_memory_target on smithi121 to 17088M 2024-04-25T00:25:44.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:44 smithi130 ceph-mon[32525]: osdmap e14: 2 total, 2 up, 2 in 2024-04-25T00:25:44.942 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:25:44.941+0000 7f387d5a6700 1 -- 172.21.15.130:0/2328629453 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_4", "target": ["mon-mgr", ""]}) -- 0x7f3878062670 con 0x7f38600a2b90 2024-04-25T00:25:44.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:44 smithi032 ceph-mon[29357]: pgmap v63: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:25:44.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:44 smithi032 ceph-mon[29357]: Detected new or changed devices on smithi121 2024-04-25T00:25:44.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:44 smithi032 ceph-mon[29357]: Adjusting osd_memory_target on smithi121 to 17088M 2024-04-25T00:25:44.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:44 smithi032 ceph-mon[29357]: osdmap e14: 2 total, 2 up, 2 in 2024-04-25T00:25:45.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:44 smithi121 ceph-mon[32231]: pgmap v63: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:25:45.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:44 smithi121 ceph-mon[32231]: Detected new or changed devices on smithi121 2024-04-25T00:25:45.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:44 smithi121 ceph-mon[32231]: Adjusting osd_memory_target on smithi121 to 17088M 2024-04-25T00:25:45.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:44 smithi121 ceph-mon[32231]: osdmap e14: 2 total, 2 up, 2 in 2024-04-25T00:25:45.858 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:25:45.858 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:25:45.859 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:45.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:25:45.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:25:45.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:46.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:25:46.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:25:46.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:46.753 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:46 smithi130 ceph-mon[32525]: pgmap v65: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:25:46.754 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:46 smithi130 ceph-mon[32525]: from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:25:46.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:46 smithi032 ceph-mon[29357]: pgmap v65: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:25:46.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:46 smithi032 ceph-mon[29357]: from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:25:47.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:46 smithi121 ceph-mon[32231]: pgmap v65: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:25:47.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:46 smithi121 ceph-mon[32231]: from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:25:47.952 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:47 smithi130 ceph-mon[32525]: from='client.? 172.21.15.130:0/3882944279' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8b49a8b4-65a2-4b96-8ab0-350f338cd6f3"}]: dispatch 2024-04-25T00:25:47.952 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:47 smithi130 ceph-mon[32525]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8b49a8b4-65a2-4b96-8ab0-350f338cd6f3"}]: dispatch 2024-04-25T00:25:47.952 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:47 smithi130 ceph-mon[32525]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8b49a8b4-65a2-4b96-8ab0-350f338cd6f3"}]': finished 2024-04-25T00:25:47.952 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:47 smithi130 ceph-mon[32525]: osdmap e15: 3 total, 2 up, 3 in 2024-04-25T00:25:47.952 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:47 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T00:25:47.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:47 smithi032 ceph-mon[29357]: from='client.? 172.21.15.130:0/3882944279' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8b49a8b4-65a2-4b96-8ab0-350f338cd6f3"}]: dispatch 2024-04-25T00:25:47.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:47 smithi032 ceph-mon[29357]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8b49a8b4-65a2-4b96-8ab0-350f338cd6f3"}]: dispatch 2024-04-25T00:25:47.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:47 smithi032 ceph-mon[29357]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8b49a8b4-65a2-4b96-8ab0-350f338cd6f3"}]': finished 2024-04-25T00:25:47.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:47 smithi032 ceph-mon[29357]: osdmap e15: 3 total, 2 up, 3 in 2024-04-25T00:25:47.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:47 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T00:25:48.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:47 smithi121 ceph-mon[32231]: from='client.? 172.21.15.130:0/3882944279' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8b49a8b4-65a2-4b96-8ab0-350f338cd6f3"}]: dispatch 2024-04-25T00:25:48.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:47 smithi121 ceph-mon[32231]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8b49a8b4-65a2-4b96-8ab0-350f338cd6f3"}]: dispatch 2024-04-25T00:25:48.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:47 smithi121 ceph-mon[32231]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8b49a8b4-65a2-4b96-8ab0-350f338cd6f3"}]': finished 2024-04-25T00:25:48.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:47 smithi121 ceph-mon[32231]: osdmap e15: 3 total, 2 up, 3 in 2024-04-25T00:25:48.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:47 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T00:25:48.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:48 smithi032 ceph-mon[29357]: pgmap v66: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:25:48.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:48 smithi032 ceph-mon[29357]: from='client.? 172.21.15.130:0/3489212537' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:25:49.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:48 smithi121 ceph-mon[32231]: pgmap v66: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:25:49.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:48 smithi121 ceph-mon[32231]: from='client.? 172.21.15.130:0/3489212537' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:25:49.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:48 smithi130 ceph-mon[32525]: pgmap v66: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:25:49.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:48 smithi130 ceph-mon[32525]: from='client.? 172.21.15.130:0/3489212537' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:25:50.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:50 smithi032 ceph-mon[29357]: pgmap v68: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:25:51.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:50 smithi121 ceph-mon[32231]: pgmap v68: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:25:51.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:50 smithi130 ceph-mon[32525]: pgmap v68: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:25:52.814 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:52 smithi130 ceph-mon[32525]: pgmap v69: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:25:52.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:52 smithi032 ceph-mon[29357]: pgmap v69: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:25:53.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:52 smithi121 ceph-mon[32231]: pgmap v69: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:25:54.758 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:54 smithi130 ceph-mon[32525]: pgmap v70: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:25:54.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:54 smithi032 ceph-mon[29357]: pgmap v70: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:25:55.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:54 smithi121 ceph-mon[32231]: pgmap v70: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:25:55.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:55 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-25T00:25:55.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:55 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:56.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:55 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-25T00:25:56.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:55 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:56.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:55 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-25T00:25:56.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:55 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:25:56.824 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:56 smithi130 ceph-mon[32525]: Deploying daemon osd.2 on smithi130 2024-04-25T00:25:56.824 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:56 smithi130 ceph-mon[32525]: pgmap v71: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:25:56.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:56 smithi032 ceph-mon[29357]: Deploying daemon osd.2 on smithi130 2024-04-25T00:25:56.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:56 smithi032 ceph-mon[29357]: pgmap v71: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:25:57.079 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:56 smithi121 ceph-mon[32231]: Deploying daemon osd.2 on smithi130 2024-04-25T00:25:57.080 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:56 smithi121 ceph-mon[32231]: pgmap v71: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:25:58.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:25:58 smithi032 ceph-mon[29357]: pgmap v72: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:25:59.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:25:58 smithi121 ceph-mon[32231]: pgmap v72: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:25:59.090 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:25:58 smithi130 ceph-mon[32525]: pgmap v72: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:26:00.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:00 smithi032 ceph-mon[29357]: pgmap v73: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:26:00.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:00 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:00.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:00 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:00.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:00 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:26:01.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:00 smithi121 ceph-mon[32231]: pgmap v73: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:26:01.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:00 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:01.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:00 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:01.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:00 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:26:01.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:00 smithi130 ceph-mon[32525]: pgmap v73: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:26:01.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:00 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:01.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:00 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:01.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:00 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:26:02.915 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:02 smithi130 ceph-mon[32525]: pgmap v74: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:26:02.960 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:02.960+0000 7f385ffff700 1 -- 172.21.15.130:0/2328629453 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7f3878062670 con 0x7f38600a2b90 2024-04-25T00:26:02.960 INFO:teuthology.orchestra.run.smithi130.stdout:Created osd(s) 2 on host 'smithi130' 2024-04-25T00:26:02.962 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:02.961+0000 7f387d5a6700 1 -- 172.21.15.130:0/2328629453 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f38600a2b90 msgr2=0x7f38600a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:26:02.962 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:02.961+0000 7f387d5a6700 1 --2- 172.21.15.130:0/2328629453 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f38600a2b90 0x7f38600a5050 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f386c005950 tx=0x7f386c0058e0 comp rx=0 tx=0).stop 2024-04-25T00:26:02.962 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:02.961+0000 7f387d5a6700 1 -- 172.21.15.130:0/2328629453 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3878102d10 msgr2=0x7f3878102670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:26:02.962 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:02.961+0000 7f387d5a6700 1 --2- 172.21.15.130:0/2328629453 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3878102d10 0x7f3878102670 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f387000ba50 tx=0x7f387000bd60 comp rx=0 tx=0).stop 2024-04-25T00:26:02.962 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:02.961+0000 7f387d5a6700 1 -- 172.21.15.130:0/2328629453 shutdown_connections 2024-04-25T00:26:02.962 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:02.961+0000 7f387d5a6700 1 --2- 172.21.15.130:0/2328629453 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f38600a2b90 0x7f38600a5050 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:02.962 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:02.961+0000 7f387d5a6700 1 --2- 172.21.15.130:0/2328629453 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3878107f70 0x7f3878101200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:02.963 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:02.961+0000 7f387d5a6700 1 --2- 172.21.15.130:0/2328629453 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3878102d10 0x7f3878102670 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:02.963 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:02.961+0000 7f387d5a6700 1 --2- 172.21.15.130:0/2328629453 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3878105640 0x7f3878100cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:02.963 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:02.961+0000 7f387d5a6700 1 -- 172.21.15.130:0/2328629453 >> 172.21.15.130:0/2328629453 conn(0x7f38780fc720 msgr2=0x7f38781035d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:26:02.963 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:02.961+0000 7f387d5a6700 1 -- 172.21.15.130:0/2328629453 shutdown_connections 2024-04-25T00:26:02.963 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:02.961+0000 7f387d5a6700 1 -- 172.21.15.130:0/2328629453 wait complete. 2024-04-25T00:26:02.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:02 smithi032 ceph-mon[29357]: pgmap v74: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:26:03.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:02 smithi121 ceph-mon[32231]: pgmap v74: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:26:03.556 DEBUG:teuthology.orchestra.run.smithi130:osd.2> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.2.service 2024-04-25T00:26:03.558 INFO:tasks.cephadm:Deploying osd.3 on smithi032 with /dev/vg_nvme/lv_3... 2024-04-25T00:26:03.559 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-25T00:26:03.816 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:26:04.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:03 smithi130 ceph-mon[32525]: pgmap v75: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:26:04.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:03 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:04.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:03 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:04.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:03 smithi130 ceph-mon[32525]: from='osd.2 [v2:172.21.15.130:6800/1393844110,v1:172.21.15.130:6801/1393844110]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-25T00:26:04.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:03 smithi130 ceph-mon[32525]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-25T00:26:04.143 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:26:03 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-2[37663]: 2024-04-25T00:26:03.785+0000 7f4b6c5787c0 -1 osd.2 0 log_to_monitors true 2024-04-25T00:26:04.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:03 smithi032 ceph-mon[29357]: pgmap v75: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:26:04.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:03 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:04.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:03 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:04.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:03 smithi032 ceph-mon[29357]: from='osd.2 [v2:172.21.15.130:6800/1393844110,v1:172.21.15.130:6801/1393844110]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-25T00:26:04.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:03 smithi032 ceph-mon[29357]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-25T00:26:04.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:03 smithi121 ceph-mon[32231]: pgmap v75: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:26:04.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:03 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:04.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:03 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:04.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:03 smithi121 ceph-mon[32231]: from='osd.2 [v2:172.21.15.130:6800/1393844110,v1:172.21.15.130:6801/1393844110]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-25T00:26:04.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:03 smithi121 ceph-mon[32231]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-25T00:26:05.317 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:05 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:05.317 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:05 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:05.317 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:05 smithi032 ceph-mon[29357]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-25T00:26:05.317 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:05 smithi032 ceph-mon[29357]: osdmap e16: 3 total, 2 up, 3 in 2024-04-25T00:26:05.318 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:05 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T00:26:05.318 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:05 smithi032 ceph-mon[29357]: from='osd.2 [v2:172.21.15.130:6800/1393844110,v1:172.21.15.130:6801/1393844110]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-25T00:26:05.318 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:05 smithi032 ceph-mon[29357]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-25T00:26:05.333 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:05 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:05.334 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:05 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:05.334 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:05 smithi130 ceph-mon[32525]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-25T00:26:05.334 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:05 smithi130 ceph-mon[32525]: osdmap e16: 3 total, 2 up, 3 in 2024-04-25T00:26:05.334 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:05 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T00:26:05.334 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:05 smithi130 ceph-mon[32525]: from='osd.2 [v2:172.21.15.130:6800/1393844110,v1:172.21.15.130:6801/1393844110]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-25T00:26:05.334 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:05 smithi130 ceph-mon[32525]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-25T00:26:05.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:05 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:05.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:05 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:05.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:05 smithi121 ceph-mon[32231]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-25T00:26:05.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:05 smithi121 ceph-mon[32231]: osdmap e16: 3 total, 2 up, 3 in 2024-04-25T00:26:05.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:05 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T00:26:05.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:05 smithi121 ceph-mon[32231]: from='osd.2 [v2:172.21.15.130:6800/1393844110,v1:172.21.15.130:6801/1393844110]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-25T00:26:05.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:05 smithi121 ceph-mon[32231]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-25T00:26:05.886 INFO:teuthology.orchestra.run.smithi032.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-25T00:26:05.887 INFO:teuthology.orchestra.run.smithi032.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-25T00:26:05.887 INFO:teuthology.orchestra.run.smithi032.stderr: stderr: 10+0 records in 2024-04-25T00:26:05.887 INFO:teuthology.orchestra.run.smithi032.stderr:10+0 records out 2024-04-25T00:26:05.887 INFO:teuthology.orchestra.run.smithi032.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0085046 s, 1.2 GB/s 2024-04-25T00:26:05.887 INFO:teuthology.orchestra.run.smithi032.stderr:--> Zapping successful for: 2024-04-25T00:26:05.891 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:26:05 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-2[37663]: 2024-04-25T00:26:05.690+0000 7f4b61bdd700 -1 osd.2 0 waiting for initial osdmap 2024-04-25T00:26:05.891 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:26:05 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-2[37663]: 2024-04-25T00:26:05.694+0000 7f4b5d9d0700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T00:26:06.649 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi032:vg_nvme/lv_3 2024-04-25T00:26:06.915 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:26:06.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:06 smithi032 ceph-mon[29357]: pgmap v77: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:26:06.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:06 smithi032 ceph-mon[29357]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-04-25T00:26:06.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:06 smithi032 ceph-mon[29357]: osdmap e17: 3 total, 2 up, 3 in 2024-04-25T00:26:06.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:06 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T00:26:06.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:06 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T00:26:07.040 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:06 smithi130 ceph-mon[32525]: pgmap v77: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:26:07.040 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:06 smithi130 ceph-mon[32525]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-04-25T00:26:07.040 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:06 smithi130 ceph-mon[32525]: osdmap e17: 3 total, 2 up, 3 in 2024-04-25T00:26:07.040 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:06 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T00:26:07.040 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:06 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T00:26:07.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:06 smithi121 ceph-mon[32231]: pgmap v77: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:26:07.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:06 smithi121 ceph-mon[32231]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-04-25T00:26:07.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:06 smithi121 ceph-mon[32231]: osdmap e17: 3 total, 2 up, 3 in 2024-04-25T00:26:07.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:06 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T00:26:07.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:06 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T00:26:07.948 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:07 smithi032 ceph-mon[29357]: purged_snaps scrub starts 2024-04-25T00:26:07.948 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:07 smithi032 ceph-mon[29357]: purged_snaps scrub ok 2024-04-25T00:26:07.948 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T00:26:07.948 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:07 smithi032 ceph-mon[29357]: osd.2 [v2:172.21.15.130:6800/1393844110,v1:172.21.15.130:6801/1393844110] boot 2024-04-25T00:26:07.948 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:07 smithi032 ceph-mon[29357]: osdmap e18: 3 total, 3 up, 3 in 2024-04-25T00:26:07.948 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T00:26:07.948 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-04-25T00:26:07.948 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:07.949 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:07.949 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:26:07.949 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:07.949 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:07.949 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:26:07.949 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:07.949 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:26:07.949 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:07.949 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:26:07.949 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:08.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:07 smithi121 ceph-mon[32231]: purged_snaps scrub starts 2024-04-25T00:26:08.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:07 smithi121 ceph-mon[32231]: purged_snaps scrub ok 2024-04-25T00:26:08.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:07 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T00:26:08.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:07 smithi121 ceph-mon[32231]: osd.2 [v2:172.21.15.130:6800/1393844110,v1:172.21.15.130:6801/1393844110] boot 2024-04-25T00:26:08.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:07 smithi121 ceph-mon[32231]: osdmap e18: 3 total, 3 up, 3 in 2024-04-25T00:26:08.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:07 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T00:26:08.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:07 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-04-25T00:26:08.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:07 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:08.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:07 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:08.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:07 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:26:08.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:07 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:08.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:07 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:08.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:07 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:26:08.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:07 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:08.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:07 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:26:08.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:07 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:08.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:07 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:26:08.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:07 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:08.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:07 smithi130 ceph-mon[32525]: purged_snaps scrub starts 2024-04-25T00:26:08.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:07 smithi130 ceph-mon[32525]: purged_snaps scrub ok 2024-04-25T00:26:08.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:07 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T00:26:08.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:07 smithi130 ceph-mon[32525]: osd.2 [v2:172.21.15.130:6800/1393844110,v1:172.21.15.130:6801/1393844110] boot 2024-04-25T00:26:08.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:07 smithi130 ceph-mon[32525]: osdmap e18: 3 total, 3 up, 3 in 2024-04-25T00:26:08.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:07 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T00:26:08.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:07 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-04-25T00:26:08.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:07 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:08.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:07 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:08.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:07 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:26:08.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:07 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:08.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:07 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:08.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:07 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:26:08.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:07 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:08.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:07 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:26:08.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:07 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:08.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:07 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:26:08.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:07 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:08.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.673+0000 7ff301d6a700 1 -- 172.21.15.32:0/2004549526 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff2fc103620 msgr2=0x7ff2fc103a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:26:08.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.673+0000 7ff301d6a700 1 --2- 172.21.15.32:0/2004549526 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff2fc103620 0x7ff2fc103a80 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7ff2f00099c0 tx=0x7ff2f0009cd0 comp rx=0 tx=0).stop 2024-04-25T00:26:08.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.673+0000 7ff301d6a700 1 -- 172.21.15.32:0/2004549526 shutdown_connections 2024-04-25T00:26:08.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.673+0000 7ff301d6a700 1 --2- 172.21.15.32:0/2004549526 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff2fc103fc0 0x7ff2fc10a870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:08.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.673+0000 7ff301d6a700 1 --2- 172.21.15.32:0/2004549526 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff2fc1005a0 0x7ff2fc1009c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:08.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.673+0000 7ff301d6a700 1 --2- 172.21.15.32:0/2004549526 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff2fc103620 0x7ff2fc103a80 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:08.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.673+0000 7ff301d6a700 1 -- 172.21.15.32:0/2004549526 >> 172.21.15.32:0/2004549526 conn(0x7ff2fc0fbee0 msgr2=0x7ff2fc0fe320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:26:08.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.673+0000 7ff301d6a700 1 -- 172.21.15.32:0/2004549526 shutdown_connections 2024-04-25T00:26:08.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.673+0000 7ff301d6a700 1 -- 172.21.15.32:0/2004549526 wait complete. 2024-04-25T00:26:08.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.673+0000 7ff301d6a700 1 Processor -- start 2024-04-25T00:26:08.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.674+0000 7ff301d6a700 1 -- start start 2024-04-25T00:26:08.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.674+0000 7ff301d6a700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff2fc1005a0 0x7ff2fc102f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:26:08.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.674+0000 7ff301d6a700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff2fc103620 0x7ff2fc1015d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:26:08.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.674+0000 7ff301d6a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff2fc103fc0 0x7ff2fc101b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:26:08.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.674+0000 7ff301d6a700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7ff2fc10de80 con 0x7ff2fc103620 2024-04-25T00:26:08.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.674+0000 7ff301d6a700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7ff2fc10dd00 con 0x7ff2fc1005a0 2024-04-25T00:26:08.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.674+0000 7ff301d6a700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7ff2fc10e000 con 0x7ff2fc103fc0 2024-04-25T00:26:08.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.674+0000 7ff2faffd700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff2fc103620 0x7ff2fc1015d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:26:08.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.674+0000 7ff2faffd700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff2fc103620 0x7ff2fc1015d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:51226/0 (socket says 172.21.15.32:51226) 2024-04-25T00:26:08.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.674+0000 7ff2faffd700 1 -- 172.21.15.32:0/295220143 learned_addr learned my addr 172.21.15.32:0/295220143 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:26:08.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.674+0000 7ff2fbfff700 1 --2- 172.21.15.32:0/295220143 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff2fc103fc0 0x7ff2fc101b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:26:08.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.674+0000 7ff2fb7fe700 1 --2- 172.21.15.32:0/295220143 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff2fc1005a0 0x7ff2fc102f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:26:08.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.674+0000 7ff2faffd700 1 -- 172.21.15.32:0/295220143 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff2fc103fc0 msgr2=0x7ff2fc101b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:26:08.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.674+0000 7ff2faffd700 1 --2- 172.21.15.32:0/295220143 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff2fc103fc0 0x7ff2fc101b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:08.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.674+0000 7ff2faffd700 1 -- 172.21.15.32:0/295220143 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff2fc1005a0 msgr2=0x7ff2fc102f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:26:08.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.674+0000 7ff2faffd700 1 --2- 172.21.15.32:0/295220143 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff2fc1005a0 0x7ff2fc102f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:08.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.674+0000 7ff2faffd700 1 -- 172.21.15.32:0/295220143 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff2fc1023c0 con 0x7ff2fc103620 2024-04-25T00:26:08.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.675+0000 7ff2faffd700 1 --2- 172.21.15.32:0/295220143 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff2fc103620 0x7ff2fc1015d0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7ff2f0005910 tx=0x7ff2f0009580 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:26:08.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.675+0000 7ff2f8ff9700 1 -- 172.21.15.32:0/295220143 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff2f0027070 con 0x7ff2fc103620 2024-04-25T00:26:08.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.675+0000 7ff2f8ff9700 1 -- 172.21.15.32:0/295220143 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff2f0004150 con 0x7ff2fc103620 2024-04-25T00:26:08.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.675+0000 7ff301d6a700 1 -- 172.21.15.32:0/295220143 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff2fc1a24d0 con 0x7ff2fc103620 2024-04-25T00:26:08.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.675+0000 7ff2f8ff9700 1 -- 172.21.15.32:0/295220143 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff2f0022da0 con 0x7ff2fc103620 2024-04-25T00:26:08.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.675+0000 7ff301d6a700 1 -- 172.21.15.32:0/295220143 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff2fc1a2920 con 0x7ff2fc103620 2024-04-25T00:26:08.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.676+0000 7ff301d6a700 1 -- 172.21.15.32:0/295220143 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff2fc068f20 con 0x7ff2fc103620 2024-04-25T00:26:08.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.677+0000 7ff2f8ff9700 1 -- 172.21.15.32:0/295220143 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 15) ==== 137777+0+0 (secure 0 0 0) 0x7ff2f002a070 con 0x7ff2fc103620 2024-04-25T00:26:08.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.677+0000 7ff2f8ff9700 1 --2- 172.21.15.32:0/295220143 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff2e40a2b90 0x7ff2e40a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:26:08.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.678+0000 7ff2fb7fe700 1 --2- 172.21.15.32:0/295220143 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff2e40a2b90 0x7ff2e40a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:26:08.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.678+0000 7ff2f8ff9700 1 -- 172.21.15.32:0/295220143 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 3226+0+0 (secure 0 0 0) 0x7ff2f00ced20 con 0x7ff2fc103620 2024-04-25T00:26:08.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.678+0000 7ff2fb7fe700 1 --2- 172.21.15.32:0/295220143 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff2e40a2b90 0x7ff2e40a5050 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7ff2fc06a6e0 tx=0x7ff2ec007400 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:26:08.682 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.681+0000 7ff2f8ff9700 1 -- 172.21.15.32:0/295220143 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7ff2f0031020 con 0x7ff2fc103620 2024-04-25T00:26:08.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:08.911+0000 7ff301d6a700 1 -- 172.21.15.32:0/295220143 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_3", "target": ["mon-mgr", ""]}) -- 0x7ff2fc062670 con 0x7ff2e40a2b90 2024-04-25T00:26:08.912 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:08 smithi032 ceph-mon[29357]: pgmap v80: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:26:08.912 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:08 smithi032 ceph-mon[29357]: Detected new or changed devices on smithi130 2024-04-25T00:26:08.913 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:08 smithi032 ceph-mon[29357]: Adjusting osd_memory_target on smithi130 to 17088M 2024-04-25T00:26:08.913 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:08 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-04-25T00:26:08.913 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:08 smithi032 ceph-mon[29357]: osdmap e19: 3 total, 3 up, 3 in 2024-04-25T00:26:08.913 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:08 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-25T00:26:09.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:08 smithi121 ceph-mon[32231]: pgmap v80: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:26:09.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:08 smithi121 ceph-mon[32231]: Detected new or changed devices on smithi130 2024-04-25T00:26:09.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:08 smithi121 ceph-mon[32231]: Adjusting osd_memory_target on smithi130 to 17088M 2024-04-25T00:26:09.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:08 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-04-25T00:26:09.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:08 smithi121 ceph-mon[32231]: osdmap e19: 3 total, 3 up, 3 in 2024-04-25T00:26:09.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:08 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-25T00:26:09.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:08 smithi130 ceph-mon[32525]: pgmap v80: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-25T00:26:09.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:08 smithi130 ceph-mon[32525]: Detected new or changed devices on smithi130 2024-04-25T00:26:09.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:08 smithi130 ceph-mon[32525]: Adjusting osd_memory_target on smithi130 to 17088M 2024-04-25T00:26:09.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:08 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-04-25T00:26:09.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:08 smithi130 ceph-mon[32525]: osdmap e19: 3 total, 3 up, 3 in 2024-04-25T00:26:09.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:08 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-25T00:26:09.498 INFO:journalctl@ceph.osd.0.smithi032.stdout:Apr 25 00:26:09 smithi032 sudo[48750]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-25T00:26:09.498 INFO:journalctl@ceph.osd.0.smithi032.stdout:Apr 25 00:26:09 smithi032 sudo[48750]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-25T00:26:09.498 INFO:journalctl@ceph.osd.0.smithi032.stdout:Apr 25 00:26:09 smithi032 sudo[48750]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-25T00:26:09.498 INFO:journalctl@ceph.osd.0.smithi032.stdout:Apr 25 00:26:09 smithi032 sudo[48750]: pam_unix(sudo:session): session closed for user root 2024-04-25T00:26:09.714 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:09.713+0000 7ff2f8ff9700 1 -- 172.21.15.32:0/295220143 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7ff2f0099140 con 0x7ff2fc103620 2024-04-25T00:26:09.827 INFO:journalctl@ceph.osd.0.smithi032.stdout:Apr 25 00:26:09 smithi032 sudo[48806]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-25T00:26:09.828 INFO:journalctl@ceph.osd.0.smithi032.stdout:Apr 25 00:26:09 smithi032 sudo[48806]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-25T00:26:09.828 INFO:journalctl@ceph.osd.0.smithi032.stdout:Apr 25 00:26:09 smithi032 sudo[48806]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-25T00:26:09.828 INFO:journalctl@ceph.osd.0.smithi032.stdout:Apr 25 00:26:09 smithi032 sudo[48806]: pam_unix(sudo:session): session closed for user root 2024-04-25T00:26:09.828 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:09 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-25T00:26:09.828 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:09 smithi032 ceph-mon[29357]: osdmap e20: 3 total, 3 up, 3 in 2024-04-25T00:26:09.828 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:09 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:26:09.828 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:09 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:26:09.828 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:09 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:09.972 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:09 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-25T00:26:09.973 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:09 smithi121 ceph-mon[32231]: osdmap e20: 3 total, 3 up, 3 in 2024-04-25T00:26:09.973 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:09 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:26:09.973 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:09 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:26:09.973 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:09 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:09.973 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:26:09 smithi121 sudo[40495]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-25T00:26:09.973 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:26:09 smithi121 sudo[40495]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-25T00:26:09.973 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:26:09 smithi121 sudo[40495]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-25T00:26:09.973 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:26:09 smithi121 sudo[40495]: pam_unix(sudo:session): session closed for user root 2024-04-25T00:26:10.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:09 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-25T00:26:10.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:09 smithi130 ceph-mon[32525]: osdmap e20: 3 total, 3 up, 3 in 2024-04-25T00:26:10.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:09 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:26:10.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:09 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:26:10.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:09 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:10.337 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:26:10 smithi121 sudo[40498]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-25T00:26:10.337 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:26:10 smithi121 sudo[40498]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-25T00:26:10.338 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:26:10 smithi121 sudo[40498]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-25T00:26:10.338 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:26:10 smithi121 sudo[40498]: pam_unix(sudo:session): session closed for user root 2024-04-25T00:26:10.733 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:26:10 smithi130 sudo[41718]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-25T00:26:10.733 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:26:10 smithi130 sudo[41718]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-25T00:26:10.733 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:26:10 smithi130 sudo[41718]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-25T00:26:10.733 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:26:10 smithi130 sudo[41718]: pam_unix(sudo:session): session closed for user root 2024-04-25T00:26:10.733 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:10 smithi130 ceph-mon[32525]: pgmap v83: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-25T00:26:10.733 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:10 smithi130 ceph-mon[32525]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:26:10.733 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:10 smithi130 ceph-mon[32525]: mgrmap e16: x(active, since 2m), standbys: z, y 2024-04-25T00:26:10.733 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:10 smithi130 ceph-mon[32525]: osdmap e21: 3 total, 3 up, 3 in 2024-04-25T00:26:10.734 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:10 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:10.734 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:10 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:10.874 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:10 smithi032 ceph-mon[29357]: pgmap v83: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-25T00:26:10.874 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:10 smithi032 ceph-mon[29357]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:26:10.874 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:10 smithi032 ceph-mon[29357]: mgrmap e16: x(active, since 2m), standbys: z, y 2024-04-25T00:26:10.874 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:10 smithi032 ceph-mon[29357]: osdmap e21: 3 total, 3 up, 3 in 2024-04-25T00:26:10.874 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:10 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:10.874 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:10 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:11.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:10 smithi121 ceph-mon[32231]: pgmap v83: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-25T00:26:11.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:10 smithi121 ceph-mon[32231]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:26:11.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:10 smithi121 ceph-mon[32231]: mgrmap e16: x(active, since 2m), standbys: z, y 2024-04-25T00:26:11.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:10 smithi121 ceph-mon[32231]: osdmap e21: 3 total, 3 up, 3 in 2024-04-25T00:26:11.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:10 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:11.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:10 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:11.141 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:26:10 smithi130 sudo[41721]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-25T00:26:11.142 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:26:10 smithi130 sudo[41721]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-25T00:26:11.142 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:26:10 smithi130 sudo[41721]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-25T00:26:11.142 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:26:10 smithi130 sudo[41721]: pam_unix(sudo:session): session closed for user root 2024-04-25T00:26:11.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:11 smithi032 sudo[48862]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-25T00:26:11.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:11 smithi032 sudo[48862]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-25T00:26:11.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:11 smithi032 sudo[48862]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-25T00:26:11.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:11 smithi032 sudo[48862]: pam_unix(sudo:session): session closed for user root 2024-04-25T00:26:11.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:11 smithi032 sudo[48905]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-25T00:26:11.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:11 smithi032 sudo[48905]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-25T00:26:11.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:11 smithi032 sudo[48905]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-25T00:26:11.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:11 smithi032 sudo[48905]: pam_unix(sudo:session): session closed for user root 2024-04-25T00:26:12.162 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:11 smithi130 sudo[41724]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-25T00:26:12.162 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:11 smithi130 sudo[41724]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-25T00:26:12.162 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:11 smithi130 sudo[41724]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-25T00:26:12.162 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:11 smithi130 sudo[41724]: pam_unix(sudo:session): session closed for user root 2024-04-25T00:26:12.162 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:12 smithi130 sudo[41727]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-25T00:26:12.162 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:12 smithi130 sudo[41727]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-25T00:26:12.162 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:12 smithi130 sudo[41727]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-25T00:26:12.162 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:12 smithi130 sudo[41727]: pam_unix(sudo:session): session closed for user root 2024-04-25T00:26:12.162 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:12 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:12.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:12 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:12.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:12 smithi032 ceph-mon[29357]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-25T00:26:12.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:12 smithi032 ceph-mon[29357]: pgmap v85: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-25T00:26:12.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:12 smithi032 ceph-mon[29357]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-25T00:26:12.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:12 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-25T00:26:12.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:12 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:26:12.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:12 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:26:12.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:12 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-25T00:26:12.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:12 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:26:12.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:12 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:26:12.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:12 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3992059213' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3956963a-c4a2-41ee-a138-3c5acef75dcb"}]: dispatch 2024-04-25T00:26:12.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:12 smithi032 ceph-mon[29357]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3956963a-c4a2-41ee-a138-3c5acef75dcb"}]: dispatch 2024-04-25T00:26:12.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:12 smithi130 ceph-mon[32525]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-25T00:26:12.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:12 smithi130 ceph-mon[32525]: pgmap v85: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-25T00:26:12.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:12 smithi130 ceph-mon[32525]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-25T00:26:12.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:12 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-25T00:26:12.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:12 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:26:12.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:12 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:26:12.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:12 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-25T00:26:12.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:12 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:26:12.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:12 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:26:12.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:12 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3992059213' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3956963a-c4a2-41ee-a138-3c5acef75dcb"}]: dispatch 2024-04-25T00:26:12.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:12 smithi130 ceph-mon[32525]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3956963a-c4a2-41ee-a138-3c5acef75dcb"}]: dispatch 2024-04-25T00:26:12.646 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:12 smithi121 sudo[40501]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-25T00:26:12.646 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:12 smithi121 sudo[40501]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-25T00:26:12.646 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:12 smithi121 sudo[40501]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-25T00:26:12.646 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:12 smithi121 sudo[40501]: pam_unix(sudo:session): session closed for user root 2024-04-25T00:26:12.646 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:12 smithi121 sudo[40504]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-25T00:26:12.646 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:12 smithi121 sudo[40504]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-25T00:26:12.646 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:12 smithi121 sudo[40504]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-25T00:26:12.646 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:12 smithi121 sudo[40504]: pam_unix(sudo:session): session closed for user root 2024-04-25T00:26:12.647 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:12 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:12.647 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:12 smithi121 ceph-mon[32231]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-25T00:26:12.647 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:12 smithi121 ceph-mon[32231]: pgmap v85: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-25T00:26:12.647 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:12 smithi121 ceph-mon[32231]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-25T00:26:12.647 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:12 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-25T00:26:12.647 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:12 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:26:12.647 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:12 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:26:12.647 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:12 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-25T00:26:12.647 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:12 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:26:12.647 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:12 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:26:12.647 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:12 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3992059213' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3956963a-c4a2-41ee-a138-3c5acef75dcb"}]: dispatch 2024-04-25T00:26:12.647 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:12 smithi121 ceph-mon[32231]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3956963a-c4a2-41ee-a138-3c5acef75dcb"}]: dispatch 2024-04-25T00:26:13.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:13 smithi032 ceph-mon[29357]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-25T00:26:13.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:13 smithi032 ceph-mon[29357]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-25T00:26:13.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:13 smithi032 ceph-mon[29357]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3956963a-c4a2-41ee-a138-3c5acef75dcb"}]': finished 2024-04-25T00:26:13.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:13 smithi032 ceph-mon[29357]: osdmap e22: 4 total, 3 up, 4 in 2024-04-25T00:26:13.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:13 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T00:26:13.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:13 smithi032 ceph-mon[29357]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-25T00:26:13.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:13 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-25T00:26:13.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:13 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:26:13.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:13 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:26:13.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:13 smithi032 ceph-mon[29357]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-25T00:26:13.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:13 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/430069571' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:26:13.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:13 smithi121 ceph-mon[32231]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-25T00:26:13.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:13 smithi121 ceph-mon[32231]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-25T00:26:13.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:13 smithi121 ceph-mon[32231]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3956963a-c4a2-41ee-a138-3c5acef75dcb"}]': finished 2024-04-25T00:26:13.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:13 smithi121 ceph-mon[32231]: osdmap e22: 4 total, 3 up, 4 in 2024-04-25T00:26:13.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:13 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T00:26:13.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:13 smithi121 ceph-mon[32231]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-25T00:26:13.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:13 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-25T00:26:13.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:13 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:26:13.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:13 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:26:13.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:13 smithi121 ceph-mon[32231]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-25T00:26:13.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:13 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/430069571' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:26:13.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:13 smithi130 ceph-mon[32525]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-25T00:26:13.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:13 smithi130 ceph-mon[32525]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-25T00:26:13.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:13 smithi130 ceph-mon[32525]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3956963a-c4a2-41ee-a138-3c5acef75dcb"}]': finished 2024-04-25T00:26:13.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:13 smithi130 ceph-mon[32525]: osdmap e22: 4 total, 3 up, 4 in 2024-04-25T00:26:13.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:13 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T00:26:13.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:13 smithi130 ceph-mon[32525]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-25T00:26:13.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:13 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-25T00:26:13.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:13 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:26:13.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:13 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:26:13.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:13 smithi130 ceph-mon[32525]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-25T00:26:13.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:13 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/430069571' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:26:14.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:14 smithi032 ceph-mon[29357]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-04-25T00:26:14.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:14 smithi121 ceph-mon[32231]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-04-25T00:26:14.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:14 smithi130 ceph-mon[32525]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-04-25T00:26:16.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:16 smithi032 ceph-mon[29357]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-04-25T00:26:16.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:16 smithi121 ceph-mon[32231]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-04-25T00:26:16.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:16 smithi130 ceph-mon[32525]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-04-25T00:26:18.312 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:18 smithi032 ceph-mon[29357]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 83 MiB used, 268 GiB / 268 GiB avail 2024-04-25T00:26:18.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:18 smithi121 ceph-mon[32231]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 83 MiB used, 268 GiB / 268 GiB avail 2024-04-25T00:26:18.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:18 smithi130 ceph-mon[32525]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 83 MiB used, 268 GiB / 268 GiB avail 2024-04-25T00:26:20.468 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:20 smithi032 ceph-mon[29357]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 44 KiB/s wr, 4 op/s 2024-04-25T00:26:20.469 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:20 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-25T00:26:20.469 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:20 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:20.469 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:20 smithi032 ceph-mon[29357]: Deploying daemon osd.3 on smithi032 2024-04-25T00:26:20.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:20 smithi121 ceph-mon[32231]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 44 KiB/s wr, 4 op/s 2024-04-25T00:26:20.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:20 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-25T00:26:20.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:20 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:20.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:20 smithi121 ceph-mon[32231]: Deploying daemon osd.3 on smithi032 2024-04-25T00:26:20.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:20 smithi130 ceph-mon[32525]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 44 KiB/s wr, 4 op/s 2024-04-25T00:26:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:20 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-25T00:26:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:20 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:20 smithi130 ceph-mon[32525]: Deploying daemon osd.3 on smithi032 2024-04-25T00:26:22.371 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:22 smithi032 ceph-mon[29357]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.2 KiB/s rd, 33 KiB/s wr, 3 op/s 2024-04-25T00:26:22.444 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:22 smithi130 ceph-mon[32525]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.2 KiB/s rd, 33 KiB/s wr, 3 op/s 2024-04-25T00:26:22.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:22 smithi121 ceph-mon[32231]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.2 KiB/s rd, 33 KiB/s wr, 3 op/s 2024-04-25T00:26:24.439 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:24 smithi032 ceph-mon[29357]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1023 B/s rd, 26 KiB/s wr, 2 op/s 2024-04-25T00:26:24.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:24 smithi121 ceph-mon[32231]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1023 B/s rd, 26 KiB/s wr, 2 op/s 2024-04-25T00:26:24.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:24 smithi130 ceph-mon[32525]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1023 B/s rd, 26 KiB/s wr, 2 op/s 2024-04-25T00:26:26.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:26 smithi130 ceph-mon[32525]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 853 B/s rd, 22 KiB/s wr, 2 op/s 2024-04-25T00:26:26.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:26 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:26.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:26 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:26.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:26 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:26:26.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:26 smithi032 ceph-mon[29357]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 853 B/s rd, 22 KiB/s wr, 2 op/s 2024-04-25T00:26:26.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:26 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:26.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:26 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:26.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:26 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:26:26.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:26 smithi121 ceph-mon[32231]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 853 B/s rd, 22 KiB/s wr, 2 op/s 2024-04-25T00:26:26.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:26 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:26.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:26 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:26.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:26 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:26:28.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:28 smithi130 ceph-mon[32525]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 853 B/s rd, 22 KiB/s wr, 2 op/s 2024-04-25T00:26:28.654 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:28 smithi032 ceph-mon[29357]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 853 B/s rd, 22 KiB/s wr, 2 op/s 2024-04-25T00:26:28.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:28 smithi121 ceph-mon[32231]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 853 B/s rd, 22 KiB/s wr, 2 op/s 2024-04-25T00:26:28.977 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:28.976+0000 7ff2f8ff9700 1 -- 172.21.15.32:0/295220143 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7ff2fc062670 con 0x7ff2e40a2b90 2024-04-25T00:26:28.977 INFO:teuthology.orchestra.run.smithi032.stdout:Created osd(s) 3 on host 'smithi032' 2024-04-25T00:26:28.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:28.979+0000 7ff301d6a700 1 -- 172.21.15.32:0/295220143 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff2e40a2b90 msgr2=0x7ff2e40a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:26:28.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:28.979+0000 7ff301d6a700 1 --2- 172.21.15.32:0/295220143 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff2e40a2b90 0x7ff2e40a5050 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7ff2fc06a6e0 tx=0x7ff2ec007400 comp rx=0 tx=0).stop 2024-04-25T00:26:28.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:28.979+0000 7ff301d6a700 1 -- 172.21.15.32:0/295220143 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff2fc103620 msgr2=0x7ff2fc1015d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:26:28.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:28.979+0000 7ff301d6a700 1 --2- 172.21.15.32:0/295220143 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff2fc103620 0x7ff2fc1015d0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7ff2f0005910 tx=0x7ff2f0009580 comp rx=0 tx=0).stop 2024-04-25T00:26:28.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:28.979+0000 7ff301d6a700 1 -- 172.21.15.32:0/295220143 shutdown_connections 2024-04-25T00:26:28.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:28.979+0000 7ff301d6a700 1 --2- 172.21.15.32:0/295220143 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff2e40a2b90 0x7ff2e40a5050 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:28.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:28.979+0000 7ff301d6a700 1 --2- 172.21.15.32:0/295220143 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff2fc103fc0 0x7ff2fc101b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:28.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:28.979+0000 7ff301d6a700 1 --2- 172.21.15.32:0/295220143 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff2fc1005a0 0x7ff2fc102f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:28.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:28.979+0000 7ff301d6a700 1 --2- 172.21.15.32:0/295220143 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff2fc103620 0x7ff2fc1015d0 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:28.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:28.979+0000 7ff301d6a700 1 -- 172.21.15.32:0/295220143 >> 172.21.15.32:0/295220143 conn(0x7ff2fc0fbee0 msgr2=0x7ff2fc106850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:26:28.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:28.979+0000 7ff301d6a700 1 -- 172.21.15.32:0/295220143 shutdown_connections 2024-04-25T00:26:28.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:26:28.979+0000 7ff301d6a700 1 -- 172.21.15.32:0/295220143 wait complete. 2024-04-25T00:26:29.698 DEBUG:teuthology.orchestra.run.smithi032:osd.3> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.3.service 2024-04-25T00:26:29.701 INFO:tasks.cephadm:Deploying osd.4 on smithi121 with /dev/vg_nvme/lv_3... 2024-04-25T00:26:29.701 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-25T00:26:29.946 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.b/config 2024-04-25T00:26:30.068 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:29 smithi121 ceph-mon[32231]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 853 B/s rd, 22 KiB/s wr, 2 op/s 2024-04-25T00:26:30.068 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:29 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:30.068 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:29 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:30.068 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:29 smithi121 ceph-mon[32231]: from='osd.3 [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-25T00:26:30.068 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:29 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:30.068 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:29 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:30.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:29 smithi032 ceph-mon[29357]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 853 B/s rd, 22 KiB/s wr, 2 op/s 2024-04-25T00:26:30.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:29 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:30.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:29 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:30.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:29 smithi032 ceph-mon[29357]: from='osd.3 [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-25T00:26:30.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:29 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:30.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:29 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:30.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:29 smithi130 ceph-mon[32525]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 853 B/s rd, 22 KiB/s wr, 2 op/s 2024-04-25T00:26:30.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:29 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:30.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:29 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:30.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:29 smithi130 ceph-mon[32525]: from='osd.3 [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-25T00:26:30.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:29 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:30.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:29 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:31.218 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:30 smithi121 ceph-mon[32231]: from='osd.3 [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-25T00:26:31.218 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:30 smithi121 ceph-mon[32231]: osdmap e23: 4 total, 3 up, 4 in 2024-04-25T00:26:31.218 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:30 smithi121 ceph-mon[32231]: from='osd.3 [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-04-25T00:26:31.218 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:30 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T00:26:31.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:30 smithi032 ceph-mon[29357]: from='osd.3 [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-25T00:26:31.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:30 smithi032 ceph-mon[29357]: osdmap e23: 4 total, 3 up, 4 in 2024-04-25T00:26:31.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:30 smithi032 ceph-mon[29357]: from='osd.3 [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-04-25T00:26:31.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:30 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T00:26:31.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:30 smithi130 ceph-mon[32525]: from='osd.3 [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-25T00:26:31.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:30 smithi130 ceph-mon[32525]: osdmap e23: 4 total, 3 up, 4 in 2024-04-25T00:26:31.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:30 smithi130 ceph-mon[32525]: from='osd.3 [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-04-25T00:26:31.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:30 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T00:26:31.768 INFO:journalctl@ceph.osd.3.smithi032.stdout:Apr 25 00:26:31 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-3[50928]: 2024-04-25T00:26:31.502+0000 7f4ac46a0700 -1 osd.3 0 waiting for initial osdmap 2024-04-25T00:26:31.768 INFO:journalctl@ceph.osd.3.smithi032.stdout:Apr 25 00:26:31 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-3[50928]: 2024-04-25T00:26:31.508+0000 7f4abc48b700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T00:26:31.801 INFO:teuthology.orchestra.run.smithi121.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-25T00:26:31.801 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-25T00:26:31.801 INFO:teuthology.orchestra.run.smithi121.stderr: stderr: 10+0 records in 2024-04-25T00:26:31.801 INFO:teuthology.orchestra.run.smithi121.stderr:10+0 records out 2024-04-25T00:26:31.801 INFO:teuthology.orchestra.run.smithi121.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0113587 s, 923 MB/s 2024-04-25T00:26:31.801 INFO:teuthology.orchestra.run.smithi121.stderr:--> Zapping successful for: 2024-04-25T00:26:32.104 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:31 smithi032 ceph-mon[29357]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-04-25T00:26:32.104 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:31 smithi032 ceph-mon[29357]: from='osd.3 [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-04-25T00:26:32.104 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:31 smithi032 ceph-mon[29357]: osdmap e24: 4 total, 3 up, 4 in 2024-04-25T00:26:32.104 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:31 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T00:26:32.104 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:31 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T00:26:32.104 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:31 smithi032 ceph-mon[29357]: from='osd.3 [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708]' entity='osd.3' 2024-04-25T00:26:32.105 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:31 smithi032 ceph-mon[29357]: osd.3 [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708] boot 2024-04-25T00:26:32.105 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:31 smithi032 ceph-mon[29357]: osdmap e25: 4 total, 4 up, 4 in 2024-04-25T00:26:32.105 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:31 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T00:26:32.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:31 smithi121 ceph-mon[32231]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-04-25T00:26:32.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:31 smithi121 ceph-mon[32231]: from='osd.3 [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-04-25T00:26:32.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:31 smithi121 ceph-mon[32231]: osdmap e24: 4 total, 3 up, 4 in 2024-04-25T00:26:32.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:31 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T00:26:32.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:31 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T00:26:32.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:31 smithi121 ceph-mon[32231]: from='osd.3 [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708]' entity='osd.3' 2024-04-25T00:26:32.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:31 smithi121 ceph-mon[32231]: osd.3 [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708] boot 2024-04-25T00:26:32.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:31 smithi121 ceph-mon[32231]: osdmap e25: 4 total, 4 up, 4 in 2024-04-25T00:26:32.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:31 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T00:26:32.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:31 smithi130 ceph-mon[32525]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-04-25T00:26:32.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:31 smithi130 ceph-mon[32525]: from='osd.3 [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-04-25T00:26:32.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:31 smithi130 ceph-mon[32525]: osdmap e24: 4 total, 3 up, 4 in 2024-04-25T00:26:32.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:31 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T00:26:32.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:31 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T00:26:32.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:31 smithi130 ceph-mon[32525]: from='osd.3 [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708]' entity='osd.3' 2024-04-25T00:26:32.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:31 smithi130 ceph-mon[32525]: osd.3 [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708] boot 2024-04-25T00:26:32.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:31 smithi130 ceph-mon[32525]: osdmap e25: 4 total, 4 up, 4 in 2024-04-25T00:26:32.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:31 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T00:26:32.500 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi121:vg_nvme/lv_3 2024-04-25T00:26:32.742 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.b/config 2024-04-25T00:26:32.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:32 smithi032 ceph-mon[29357]: purged_snaps scrub starts 2024-04-25T00:26:32.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:32 smithi032 ceph-mon[29357]: purged_snaps scrub ok 2024-04-25T00:26:32.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:32 smithi032 ceph-mon[29357]: osdmap e26: 4 total, 4 up, 4 in 2024-04-25T00:26:33.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:32 smithi121 ceph-mon[32231]: purged_snaps scrub starts 2024-04-25T00:26:33.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:32 smithi121 ceph-mon[32231]: purged_snaps scrub ok 2024-04-25T00:26:33.067 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:32 smithi121 ceph-mon[32231]: osdmap e26: 4 total, 4 up, 4 in 2024-04-25T00:26:33.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:32 smithi130 ceph-mon[32525]: purged_snaps scrub starts 2024-04-25T00:26:33.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:32 smithi130 ceph-mon[32525]: purged_snaps scrub ok 2024-04-25T00:26:33.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:32 smithi130 ceph-mon[32525]: osdmap e26: 4 total, 4 up, 4 in 2024-04-25T00:26:34.122 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.117+0000 7f84e92c3700 1 -- 172.21.15.121:0/341758395 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f84e4103ec0 msgr2=0x7f84e410a770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:26:34.122 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.117+0000 7f84e92c3700 1 --2- 172.21.15.121:0/341758395 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f84e4103ec0 0x7f84e410a770 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f84dc00b0a0 tx=0x7f84dc00b3b0 comp rx=0 tx=0).stop 2024-04-25T00:26:34.122 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.117+0000 7f84e92c3700 1 -- 172.21.15.121:0/341758395 shutdown_connections 2024-04-25T00:26:34.122 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.117+0000 7f84e92c3700 1 --2- 172.21.15.121:0/341758395 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f84e4103ec0 0x7f84e410a770 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:34.122 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.117+0000 7f84e92c3700 1 --2- 172.21.15.121:0/341758395 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f84e4100540 0x7f84e4100960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:34.122 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.117+0000 7f84e92c3700 1 --2- 172.21.15.121:0/341758395 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f84e4103520 0x7f84e4103980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:34.122 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.117+0000 7f84e92c3700 1 -- 172.21.15.121:0/341758395 >> 172.21.15.121:0/341758395 conn(0x7f84e40fbec0 msgr2=0x7f84e40fe2e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:26:34.122 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.117+0000 7f84e92c3700 1 -- 172.21.15.121:0/341758395 shutdown_connections 2024-04-25T00:26:34.122 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.117+0000 7f84e92c3700 1 -- 172.21.15.121:0/341758395 wait complete. 2024-04-25T00:26:34.122 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.118+0000 7f84e92c3700 1 Processor -- start 2024-04-25T00:26:34.122 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.118+0000 7f84e92c3700 1 -- start start 2024-04-25T00:26:34.123 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.118+0000 7f84e92c3700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f84e4100540 0x7f84e41983e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:26:34.123 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.118+0000 7f84e92c3700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f84e4103520 0x7f84e4198920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:26:34.123 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.118+0000 7f84e92c3700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f84e4103ec0 0x7f84e419f9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:26:34.123 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.118+0000 7f84e92c3700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f84e410de50 con 0x7f84e4100540 2024-04-25T00:26:34.123 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.118+0000 7f84e92c3700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f84e410dcd0 con 0x7f84e4103520 2024-04-25T00:26:34.123 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.118+0000 7f84e92c3700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f84e410dfd0 con 0x7f84e4103ec0 2024-04-25T00:26:34.123 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.118+0000 7f84da7fc700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f84e4103520 0x7f84e4198920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:26:34.123 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.118+0000 7f84da7fc700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f84e4103520 0x7f84e4198920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.121:58584/0 (socket says 172.21.15.121:58584) 2024-04-25T00:26:34.123 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.118+0000 7f84da7fc700 1 -- 172.21.15.121:0/883103502 learned_addr learned my addr 172.21.15.121:0/883103502 (peer_addr_for_me v2:172.21.15.121:0/0) 2024-04-25T00:26:34.123 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.118+0000 7f84e2ffd700 1 --2- 172.21.15.121:0/883103502 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f84e4100540 0x7f84e41983e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:26:34.123 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.118+0000 7f84e37fe700 1 --2- 172.21.15.121:0/883103502 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f84e4103ec0 0x7f84e419f9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:26:34.123 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.118+0000 7f84da7fc700 1 -- 172.21.15.121:0/883103502 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f84e4103ec0 msgr2=0x7f84e419f9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:26:34.123 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.118+0000 7f84da7fc700 1 --2- 172.21.15.121:0/883103502 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f84e4103ec0 0x7f84e419f9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:34.124 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.118+0000 7f84da7fc700 1 -- 172.21.15.121:0/883103502 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f84e4100540 msgr2=0x7f84e41983e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:26:34.124 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.118+0000 7f84da7fc700 1 --2- 172.21.15.121:0/883103502 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f84e4100540 0x7f84e41983e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:34.124 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.118+0000 7f84da7fc700 1 -- 172.21.15.121:0/883103502 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f84e41a0040 con 0x7f84e4103520 2024-04-25T00:26:34.124 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.119+0000 7f84da7fc700 1 --2- 172.21.15.121:0/883103502 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f84e4103520 0x7f84e4198920 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f84cc00e9c0 tx=0x7f84cc00ecd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:26:34.124 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.119+0000 7f84e0ff9700 1 -- 172.21.15.121:0/883103502 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f84cc00cbd0 con 0x7f84e4103520 2024-04-25T00:26:34.124 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.119+0000 7f84e92c3700 1 -- 172.21.15.121:0/883103502 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f84e41a0330 con 0x7f84e4103520 2024-04-25T00:26:34.124 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.119+0000 7f84e92c3700 1 -- 172.21.15.121:0/883103502 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f84e41a0870 con 0x7f84e4103520 2024-04-25T00:26:34.124 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.119+0000 7f84e0ff9700 1 -- 172.21.15.121:0/883103502 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f84cc00cd70 con 0x7f84e4103520 2024-04-25T00:26:34.124 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.119+0000 7f84e0ff9700 1 -- 172.21.15.121:0/883103502 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f84cc0056f0 con 0x7f84e4103520 2024-04-25T00:26:34.124 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.120+0000 7f84e92c3700 1 -- 172.21.15.121:0/883103502 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f84c4005360 con 0x7f84e4103520 2024-04-25T00:26:34.124 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.121+0000 7f84e0ff9700 1 -- 172.21.15.121:0/883103502 <== mon.2 v2:172.21.15.121:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f84cc004750 con 0x7f84e4103520 2024-04-25T00:26:34.124 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.121+0000 7f84e0ff9700 1 --2- 172.21.15.121:0/883103502 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f84d00a2ca0 0x7f84d00a5160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:26:34.125 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.121+0000 7f84e0ff9700 1 -- 172.21.15.121:0/883103502 <== mon.2 v2:172.21.15.121:3300/0 5 ==== osd_map(27..27 src has 1..27) ==== 3698+0+0 (secure 0 0 0) 0x7f84cc0ce140 con 0x7f84e4103520 2024-04-25T00:26:34.125 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.122+0000 7f84e2ffd700 1 --2- 172.21.15.121:0/883103502 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f84d00a2ca0 0x7f84d00a5160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:26:34.125 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.122+0000 7f84e2ffd700 1 --2- 172.21.15.121:0/883103502 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f84d00a2ca0 0x7f84d00a5160 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f84e4069d10 tx=0x7f84d400b410 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:26:34.125 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.125+0000 7f84e0ff9700 1 -- 172.21.15.121:0/883103502 <== mon.2 v2:172.21.15.121:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f84cc053740 con 0x7f84e4103520 2024-04-25T00:26:34.331 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:34.331+0000 7f84e92c3700 1 -- 172.21.15.121:0/883103502 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_3", "target": ["mon-mgr", ""]}) -- 0x7f84c4000bf0 con 0x7f84d00a2ca0 2024-04-25T00:26:34.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:34 smithi121 ceph-mon[32231]: pgmap v101: 1 pgs: 1 remapped+peering; 577 KiB data, 513 MiB used, 357 GiB / 358 GiB avail 2024-04-25T00:26:34.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:34 smithi121 ceph-mon[32231]: Detected new or changed devices on smithi032 2024-04-25T00:26:34.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:34.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:34.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:26:34.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:26:34.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:34 smithi121 ceph-mon[32231]: Adjusting osd_memory_target on smithi032 to 8546M 2024-04-25T00:26:34.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:34.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:34.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:26:34.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:34.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:26:34.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:34.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:26:34.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:34.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:34 smithi121 ceph-mon[32231]: osdmap e27: 4 total, 4 up, 4 in 2024-04-25T00:26:34.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:26:34.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:26:34.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:34.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:34 smithi032 ceph-mon[29357]: pgmap v101: 1 pgs: 1 remapped+peering; 577 KiB data, 513 MiB used, 357 GiB / 358 GiB avail 2024-04-25T00:26:34.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:34 smithi032 ceph-mon[29357]: Detected new or changed devices on smithi032 2024-04-25T00:26:34.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:34.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:34.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:26:34.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:26:34.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:34 smithi032 ceph-mon[29357]: Adjusting osd_memory_target on smithi032 to 8546M 2024-04-25T00:26:34.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:34.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:34.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:26:34.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:34.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:26:34.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:34.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:26:34.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:34.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:34 smithi032 ceph-mon[29357]: osdmap e27: 4 total, 4 up, 4 in 2024-04-25T00:26:34.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:26:34.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:26:34.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:34 smithi130 ceph-mon[32525]: pgmap v101: 1 pgs: 1 remapped+peering; 577 KiB data, 513 MiB used, 357 GiB / 358 GiB avail 2024-04-25T00:26:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:34 smithi130 ceph-mon[32525]: Detected new or changed devices on smithi032 2024-04-25T00:26:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:26:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:26:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:34 smithi130 ceph-mon[32525]: Adjusting osd_memory_target on smithi032 to 8546M 2024-04-25T00:26:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:34.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:26:34.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:34.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:26:34.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:34.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:26:34.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:34.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:34 smithi130 ceph-mon[32525]: osdmap e27: 4 total, 4 up, 4 in 2024-04-25T00:26:34.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:26:34.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:26:34.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:35.830 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:35 smithi121 ceph-mon[32231]: from='client.24212 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:26:35.830 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:35 smithi121 ceph-mon[32231]: osdmap e28: 4 total, 4 up, 4 in 2024-04-25T00:26:35.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:35 smithi032 ceph-mon[29357]: from='client.24212 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:26:35.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:35 smithi032 ceph-mon[29357]: osdmap e28: 4 total, 4 up, 4 in 2024-04-25T00:26:36.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:35 smithi130 ceph-mon[32525]: from='client.24212 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:26:36.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:35 smithi130 ceph-mon[32525]: osdmap e28: 4 total, 4 up, 4 in 2024-04-25T00:26:36.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:36 smithi032 ceph-mon[29357]: pgmap v104: 1 pgs: 1 remapped+peering; 577 KiB data, 513 MiB used, 357 GiB / 358 GiB avail 2024-04-25T00:26:36.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:36 smithi032 ceph-mon[29357]: osdmap e29: 4 total, 4 up, 4 in 2024-04-25T00:26:37.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:36 smithi121 ceph-mon[32231]: pgmap v104: 1 pgs: 1 remapped+peering; 577 KiB data, 513 MiB used, 357 GiB / 358 GiB avail 2024-04-25T00:26:37.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:36 smithi121 ceph-mon[32231]: osdmap e29: 4 total, 4 up, 4 in 2024-04-25T00:26:37.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:36 smithi130 ceph-mon[32525]: pgmap v104: 1 pgs: 1 remapped+peering; 577 KiB data, 513 MiB used, 357 GiB / 358 GiB avail 2024-04-25T00:26:37.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:36 smithi130 ceph-mon[32525]: osdmap e29: 4 total, 4 up, 4 in 2024-04-25T00:26:37.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:37 smithi032 ceph-mon[29357]: from='client.? 172.21.15.121:0/3372127619' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5dede06a-2cc8-494b-a0ec-13551677b878"}]: dispatch 2024-04-25T00:26:37.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:37 smithi032 ceph-mon[29357]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5dede06a-2cc8-494b-a0ec-13551677b878"}]: dispatch 2024-04-25T00:26:37.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:37 smithi032 ceph-mon[29357]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5dede06a-2cc8-494b-a0ec-13551677b878"}]': finished 2024-04-25T00:26:37.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:37 smithi032 ceph-mon[29357]: osdmap e30: 5 total, 4 up, 5 in 2024-04-25T00:26:37.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:37 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T00:26:37.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:37 smithi032 ceph-mon[29357]: from='client.? 172.21.15.121:0/3909379540' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:26:38.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:37 smithi121 ceph-mon[32231]: from='client.? 172.21.15.121:0/3372127619' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5dede06a-2cc8-494b-a0ec-13551677b878"}]: dispatch 2024-04-25T00:26:38.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:37 smithi121 ceph-mon[32231]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5dede06a-2cc8-494b-a0ec-13551677b878"}]: dispatch 2024-04-25T00:26:38.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:37 smithi121 ceph-mon[32231]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5dede06a-2cc8-494b-a0ec-13551677b878"}]': finished 2024-04-25T00:26:38.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:37 smithi121 ceph-mon[32231]: osdmap e30: 5 total, 4 up, 5 in 2024-04-25T00:26:38.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T00:26:38.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:37 smithi121 ceph-mon[32231]: from='client.? 172.21.15.121:0/3909379540' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:26:38.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:37 smithi130 ceph-mon[32525]: from='client.? 172.21.15.121:0/3372127619' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5dede06a-2cc8-494b-a0ec-13551677b878"}]: dispatch 2024-04-25T00:26:38.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:37 smithi130 ceph-mon[32525]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5dede06a-2cc8-494b-a0ec-13551677b878"}]: dispatch 2024-04-25T00:26:38.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:37 smithi130 ceph-mon[32525]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5dede06a-2cc8-494b-a0ec-13551677b878"}]': finished 2024-04-25T00:26:38.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:37 smithi130 ceph-mon[32525]: osdmap e30: 5 total, 4 up, 5 in 2024-04-25T00:26:38.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:37 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T00:26:38.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:37 smithi130 ceph-mon[32525]: from='client.? 172.21.15.121:0/3909379540' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:26:38.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:38 smithi032 ceph-mon[29357]: pgmap v106: 1 pgs: 1 remapped+peering; 577 KiB data, 512 MiB used, 357 GiB / 358 GiB avail 2024-04-25T00:26:39.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:38 smithi121 ceph-mon[32231]: pgmap v106: 1 pgs: 1 remapped+peering; 577 KiB data, 512 MiB used, 357 GiB / 358 GiB avail 2024-04-25T00:26:39.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:38 smithi130 ceph-mon[32525]: pgmap v106: 1 pgs: 1 remapped+peering; 577 KiB data, 512 MiB used, 357 GiB / 358 GiB avail 2024-04-25T00:26:40.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:40 smithi032 ceph-mon[29357]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 117 MiB used, 357 GiB / 358 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-04-25T00:26:41.080 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:40 smithi121 ceph-mon[32231]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 117 MiB used, 357 GiB / 358 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-04-25T00:26:41.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:40 smithi130 ceph-mon[32525]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 117 MiB used, 357 GiB / 358 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-04-25T00:26:42.979 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:42 smithi121 ceph-mon[32231]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-04-25T00:26:42.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:42 smithi032 ceph-mon[29357]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-04-25T00:26:43.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:42 smithi130 ceph-mon[32525]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-04-25T00:26:44.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:44 smithi032 ceph-mon[29357]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-25T00:26:44.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-25T00:26:44.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:45.013 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:44 smithi121 ceph-mon[32231]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-25T00:26:45.013 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-25T00:26:45.013 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:45.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:44 smithi130 ceph-mon[32525]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-25T00:26:45.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-25T00:26:45.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:45.917 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:45 smithi121 ceph-mon[32231]: Deploying daemon osd.4 on smithi121 2024-04-25T00:26:45.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:45 smithi032 ceph-mon[29357]: Deploying daemon osd.4 on smithi121 2024-04-25T00:26:46.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:45 smithi130 ceph-mon[32525]: Deploying daemon osd.4 on smithi121 2024-04-25T00:26:46.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:46 smithi121 ceph-mon[32231]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-04-25T00:26:46.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:46 smithi032 ceph-mon[29357]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-04-25T00:26:47.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:46 smithi130 ceph-mon[32525]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-04-25T00:26:48.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:48 smithi032 ceph-mon[29357]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-25T00:26:49.031 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:48 smithi121 ceph-mon[32231]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-25T00:26:49.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:48 smithi130 ceph-mon[32525]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-25T00:26:50.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:50 smithi032 ceph-mon[29357]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-04-25T00:26:50.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:50 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:50.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:50 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:26:50.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:50 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:51.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:50 smithi121 ceph-mon[32231]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-04-25T00:26:51.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:50 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:51.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:50 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:26:51.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:50 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:51.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:50 smithi130 ceph-mon[32525]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-04-25T00:26:51.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:50 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:51.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:50 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:26:51.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:50 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:52.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:52 smithi121 ceph-mon[32231]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-25T00:26:52.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:52 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:52.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:52 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:52.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:52 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:52.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:52 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:26:52.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:52 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:52.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:52 smithi032 ceph-mon[29357]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-25T00:26:52.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:52 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:52.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:52 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:52.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:52 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:52.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:52 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:26:52.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:52 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:53.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:52 smithi130 ceph-mon[32525]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-25T00:26:53.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:52 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:53.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:52 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:53.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:52 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:53.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:52 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:26:53.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:52 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:53.551 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:53.549+0000 7f84e0ff9700 1 -- 172.21.15.121:0/883103502 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7f84c4000bf0 con 0x7f84d00a2ca0 2024-04-25T00:26:53.551 INFO:teuthology.orchestra.run.smithi121.stdout:Created osd(s) 4 on host 'smithi121' 2024-04-25T00:26:53.553 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:53.552+0000 7f84e92c3700 1 -- 172.21.15.121:0/883103502 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f84d00a2ca0 msgr2=0x7f84d00a5160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:26:53.553 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:53.552+0000 7f84e92c3700 1 --2- 172.21.15.121:0/883103502 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f84d00a2ca0 0x7f84d00a5160 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f84e4069d10 tx=0x7f84d400b410 comp rx=0 tx=0).stop 2024-04-25T00:26:53.553 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:53.552+0000 7f84e92c3700 1 -- 172.21.15.121:0/883103502 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f84e4103520 msgr2=0x7f84e4198920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:26:53.553 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:53.552+0000 7f84e92c3700 1 --2- 172.21.15.121:0/883103502 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f84e4103520 0x7f84e4198920 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f84cc00e9c0 tx=0x7f84cc00ecd0 comp rx=0 tx=0).stop 2024-04-25T00:26:53.553 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:53.552+0000 7f84e92c3700 1 -- 172.21.15.121:0/883103502 shutdown_connections 2024-04-25T00:26:53.553 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:53.552+0000 7f84e92c3700 1 --2- 172.21.15.121:0/883103502 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f84d00a2ca0 0x7f84d00a5160 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:53.553 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:53.552+0000 7f84e92c3700 1 --2- 172.21.15.121:0/883103502 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f84e4103ec0 0x7f84e419f9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:53.553 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:53.552+0000 7f84e92c3700 1 --2- 172.21.15.121:0/883103502 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f84e4100540 0x7f84e41983e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:53.554 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:53.552+0000 7f84e92c3700 1 --2- 172.21.15.121:0/883103502 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f84e4103520 0x7f84e4198920 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:53.554 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:53.552+0000 7f84e92c3700 1 -- 172.21.15.121:0/883103502 >> 172.21.15.121:0/883103502 conn(0x7f84e40fbec0 msgr2=0x7f84e41067c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:26:53.554 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:53.552+0000 7f84e92c3700 1 -- 172.21.15.121:0/883103502 shutdown_connections 2024-04-25T00:26:53.554 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:26:53.552+0000 7f84e92c3700 1 -- 172.21.15.121:0/883103502 wait complete. 2024-04-25T00:26:54.232 DEBUG:teuthology.orchestra.run.smithi121:osd.4> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.4.service 2024-04-25T00:26:54.234 INFO:tasks.cephadm:Deploying osd.5 on smithi130 with /dev/vg_nvme/lv_3... 2024-04-25T00:26:54.234 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-25T00:26:54.479 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.c/config 2024-04-25T00:26:54.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:54 smithi121 ceph-mon[32231]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-25T00:26:54.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:54 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:54.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:54 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:26:54.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:54 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:54.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:54 smithi121 ceph-mon[32231]: from='osd.4 [v2:172.21.15.121:6808/1286757826,v1:172.21.15.121:6809/1286757826]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-25T00:26:54.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:54 smithi121 ceph-mon[32231]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-25T00:26:54.849 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:54 smithi130 ceph-mon[32525]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-25T00:26:54.849 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:54 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:54.849 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:54 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:26:54.849 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:54 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:54.849 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:54 smithi130 ceph-mon[32525]: from='osd.4 [v2:172.21.15.121:6808/1286757826,v1:172.21.15.121:6809/1286757826]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-25T00:26:54.850 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:54 smithi130 ceph-mon[32525]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-25T00:26:54.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:54 smithi032 ceph-mon[29357]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-25T00:26:54.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:54 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:54.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:54 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:26:54.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:54 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:54.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:54 smithi032 ceph-mon[29357]: from='osd.4 [v2:172.21.15.121:6808/1286757826,v1:172.21.15.121:6809/1286757826]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-25T00:26:54.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:54 smithi032 ceph-mon[29357]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-25T00:26:55.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:55 smithi130 ceph-mon[32525]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-25T00:26:55.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:55 smithi130 ceph-mon[32525]: osdmap e31: 5 total, 4 up, 5 in 2024-04-25T00:26:55.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:55 smithi130 ceph-mon[32525]: from='osd.4 [v2:172.21.15.121:6808/1286757826,v1:172.21.15.121:6809/1286757826]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-25T00:26:55.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:55 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T00:26:55.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:55 smithi130 ceph-mon[32525]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-25T00:26:55.912 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:55 smithi121 ceph-mon[32231]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-25T00:26:55.913 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:55 smithi121 ceph-mon[32231]: osdmap e31: 5 total, 4 up, 5 in 2024-04-25T00:26:55.913 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:55 smithi121 ceph-mon[32231]: from='osd.4 [v2:172.21.15.121:6808/1286757826,v1:172.21.15.121:6809/1286757826]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-25T00:26:55.913 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:55 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T00:26:55.913 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:55 smithi121 ceph-mon[32231]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-25T00:26:55.913 INFO:journalctl@ceph.osd.4.smithi121.stdout:Apr 25 00:26:55 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-4[43300]: 2024-04-25T00:26:55.721+0000 7f797ab38700 -1 osd.4 0 waiting for initial osdmap 2024-04-25T00:26:55.913 INFO:journalctl@ceph.osd.4.smithi121.stdout:Apr 25 00:26:55 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-4[43300]: 2024-04-25T00:26:55.730+0000 7f797692b700 -1 osd.4 32 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T00:26:55.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:55 smithi032 ceph-mon[29357]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-25T00:26:55.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:55 smithi032 ceph-mon[29357]: osdmap e31: 5 total, 4 up, 5 in 2024-04-25T00:26:55.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:55 smithi032 ceph-mon[29357]: from='osd.4 [v2:172.21.15.121:6808/1286757826,v1:172.21.15.121:6809/1286757826]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-25T00:26:55.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:55 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T00:26:55.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:55 smithi032 ceph-mon[29357]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-25T00:26:56.370 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-25T00:26:56.370 INFO:teuthology.orchestra.run.smithi130.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-25T00:26:56.370 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10+0 records in 2024-04-25T00:26:56.370 INFO:teuthology.orchestra.run.smithi130.stderr:10+0 records out 2024-04-25T00:26:56.371 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0124671 s, 841 MB/s 2024-04-25T00:26:56.371 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping successful for: 2024-04-25T00:26:56.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:56 smithi121 ceph-mon[32231]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-25T00:26:56.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:56 smithi121 ceph-mon[32231]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-04-25T00:26:56.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:56 smithi121 ceph-mon[32231]: osdmap e32: 5 total, 4 up, 5 in 2024-04-25T00:26:56.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:56 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T00:26:56.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:56 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T00:26:56.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:56 smithi032 ceph-mon[29357]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-25T00:26:56.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:56 smithi032 ceph-mon[29357]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-04-25T00:26:56.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:56 smithi032 ceph-mon[29357]: osdmap e32: 5 total, 4 up, 5 in 2024-04-25T00:26:56.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:56 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T00:26:56.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:56 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T00:26:57.075 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:56 smithi130 ceph-mon[32525]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-25T00:26:57.075 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:56 smithi130 ceph-mon[32525]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-04-25T00:26:57.075 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:56 smithi130 ceph-mon[32525]: osdmap e32: 5 total, 4 up, 5 in 2024-04-25T00:26:57.075 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:56 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T00:26:57.075 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:56 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T00:26:57.108 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi130:vg_nvme/lv_3 2024-04-25T00:26:57.350 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.c/config 2024-04-25T00:26:57.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:57 smithi032 ceph-mon[29357]: purged_snaps scrub starts 2024-04-25T00:26:57.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:57 smithi032 ceph-mon[29357]: purged_snaps scrub ok 2024-04-25T00:26:57.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:57 smithi032 ceph-mon[29357]: osd.4 [v2:172.21.15.121:6808/1286757826,v1:172.21.15.121:6809/1286757826] boot 2024-04-25T00:26:57.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:57 smithi032 ceph-mon[29357]: osdmap e33: 5 total, 5 up, 5 in 2024-04-25T00:26:57.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:57 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T00:26:57.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:57 smithi032 ceph-mon[29357]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-25T00:26:57.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:57 smithi032 ceph-mon[29357]: Detected new or changed devices on smithi121 2024-04-25T00:26:57.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:57 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:57.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:57 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:57.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:57 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:26:57.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:57 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:26:57.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:57 smithi032 ceph-mon[29357]: Adjusting osd_memory_target on smithi121 to 8544M 2024-04-25T00:26:57.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:57 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:57.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:57 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:57.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:57 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:26:57.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:57 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:58.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:57 smithi121 ceph-mon[32231]: purged_snaps scrub starts 2024-04-25T00:26:58.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:57 smithi121 ceph-mon[32231]: purged_snaps scrub ok 2024-04-25T00:26:58.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:57 smithi121 ceph-mon[32231]: osd.4 [v2:172.21.15.121:6808/1286757826,v1:172.21.15.121:6809/1286757826] boot 2024-04-25T00:26:58.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:57 smithi121 ceph-mon[32231]: osdmap e33: 5 total, 5 up, 5 in 2024-04-25T00:26:58.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:57 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T00:26:58.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:57 smithi121 ceph-mon[32231]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-25T00:26:58.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:57 smithi121 ceph-mon[32231]: Detected new or changed devices on smithi121 2024-04-25T00:26:58.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:57 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:58.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:57 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:58.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:57 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:26:58.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:57 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:26:58.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:57 smithi121 ceph-mon[32231]: Adjusting osd_memory_target on smithi121 to 8544M 2024-04-25T00:26:58.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:57 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:58.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:57 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:58.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:57 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:26:58.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:57 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:58.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:57 smithi130 ceph-mon[32525]: purged_snaps scrub starts 2024-04-25T00:26:58.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:57 smithi130 ceph-mon[32525]: purged_snaps scrub ok 2024-04-25T00:26:58.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:57 smithi130 ceph-mon[32525]: osd.4 [v2:172.21.15.121:6808/1286757826,v1:172.21.15.121:6809/1286757826] boot 2024-04-25T00:26:58.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:57 smithi130 ceph-mon[32525]: osdmap e33: 5 total, 5 up, 5 in 2024-04-25T00:26:58.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:57 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T00:26:58.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:57 smithi130 ceph-mon[32525]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-25T00:26:58.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:57 smithi130 ceph-mon[32525]: Detected new or changed devices on smithi121 2024-04-25T00:26:58.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:57 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:58.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:57 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:58.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:57 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:26:58.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:57 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:26:58.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:57 smithi130 ceph-mon[32525]: Adjusting osd_memory_target on smithi121 to 8544M 2024-04-25T00:26:58.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:57 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:58.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:57 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:26:58.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:57 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:26:58.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:57 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:26:58.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:58 smithi032 ceph-mon[29357]: osdmap e34: 5 total, 5 up, 5 in 2024-04-25T00:26:59.017 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:58 smithi130 ceph-mon[32525]: osdmap e34: 5 total, 5 up, 5 in 2024-04-25T00:26:59.039 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.035+0000 7f53dc081700 1 -- 172.21.15.130:0/2943686153 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53d4102d30 msgr2=0x7f53d41095e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:26:59.039 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.035+0000 7f53dc081700 1 --2- 172.21.15.130:0/2943686153 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53d4102d30 0x7f53d41095e0 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f53cc00b0a0 tx=0x7f53cc00b3b0 comp rx=0 tx=0).stop 2024-04-25T00:26:59.039 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.035+0000 7f53dc081700 1 -- 172.21.15.130:0/2943686153 shutdown_connections 2024-04-25T00:26:59.039 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.035+0000 7f53dc081700 1 --2- 172.21.15.130:0/2943686153 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53d4102d30 0x7f53d41095e0 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:59.039 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.035+0000 7f53dc081700 1 --2- 172.21.15.130:0/2943686153 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f53d4101190 0x7f53d41015b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:59.039 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.035+0000 7f53dc081700 1 --2- 172.21.15.130:0/2943686153 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f53d4102390 0x7f53d41027f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:59.039 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.035+0000 7f53dc081700 1 -- 172.21.15.130:0/2943686153 >> 172.21.15.130:0/2943686153 conn(0x7f53d40fc730 msgr2=0x7f53d40feb70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:26:59.040 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.035+0000 7f53dc081700 1 -- 172.21.15.130:0/2943686153 shutdown_connections 2024-04-25T00:26:59.040 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.035+0000 7f53dc081700 1 -- 172.21.15.130:0/2943686153 wait complete. 2024-04-25T00:26:59.040 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.036+0000 7f53dc081700 1 Processor -- start 2024-04-25T00:26:59.040 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.036+0000 7f53dc081700 1 -- start start 2024-04-25T00:26:59.040 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.036+0000 7f53dc081700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f53d4101190 0x7f53d4105120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:26:59.040 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.036+0000 7f53dc081700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53d4102390 0x7f53d4103770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:26:59.040 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.036+0000 7f53dc081700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f53d4102d30 0x7f53d4103cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:26:59.040 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.036+0000 7f53dc081700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f53d410cd80 con 0x7f53d4101190 2024-04-25T00:26:59.040 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.036+0000 7f53dc081700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f53d410cc00 con 0x7f53d4102d30 2024-04-25T00:26:59.040 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.036+0000 7f53dc081700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f53d410cf00 con 0x7f53d4102390 2024-04-25T00:26:59.040 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.036+0000 7f53d961c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53d4102390 0x7f53d4103770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:26:59.040 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.036+0000 7f53d961c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53d4102390 0x7f53d4103770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.130:55624/0 (socket says 172.21.15.130:55624) 2024-04-25T00:26:59.041 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.036+0000 7f53d961c700 1 -- 172.21.15.130:0/3611694872 learned_addr learned my addr 172.21.15.130:0/3611694872 (peer_addr_for_me v2:172.21.15.130:0/0) 2024-04-25T00:26:59.041 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.036+0000 7f53d9e1d700 1 --2- 172.21.15.130:0/3611694872 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f53d4101190 0x7f53d4105120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:26:59.041 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.036+0000 7f53da61e700 1 --2- 172.21.15.130:0/3611694872 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f53d4102d30 0x7f53d4103cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:26:59.041 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.036+0000 7f53d961c700 1 -- 172.21.15.130:0/3611694872 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f53d4102d30 msgr2=0x7f53d4103cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:26:59.041 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.036+0000 7f53d961c700 1 --2- 172.21.15.130:0/3611694872 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f53d4102d30 0x7f53d4103cb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:59.041 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.036+0000 7f53d961c700 1 -- 172.21.15.130:0/3611694872 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f53d4101190 msgr2=0x7f53d4105120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:26:59.041 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.036+0000 7f53d961c700 1 --2- 172.21.15.130:0/3611694872 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f53d4101190 0x7f53d4105120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:26:59.041 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.036+0000 7f53d961c700 1 -- 172.21.15.130:0/3611694872 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f53d4104560 con 0x7f53d4102390 2024-04-25T00:26:59.041 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.037+0000 7f53d961c700 1 --2- 172.21.15.130:0/3611694872 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53d4102390 0x7f53d4103770 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f53d000d980 tx=0x7f53d000dc90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:26:59.041 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.037+0000 7f53c6ffd700 1 -- 172.21.15.130:0/3611694872 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f53d0017070 con 0x7f53d4102390 2024-04-25T00:26:59.041 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.037+0000 7f53c6ffd700 1 -- 172.21.15.130:0/3611694872 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f53d0005020 con 0x7f53d4102390 2024-04-25T00:26:59.042 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.037+0000 7f53dc081700 1 -- 172.21.15.130:0/3611694872 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f53d41a2610 con 0x7f53d4102390 2024-04-25T00:26:59.042 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.037+0000 7f53c6ffd700 1 -- 172.21.15.130:0/3611694872 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f53d0009bc0 con 0x7f53d4102390 2024-04-25T00:26:59.042 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.037+0000 7f53dc081700 1 -- 172.21.15.130:0/3611694872 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f53d41a2ad0 con 0x7f53d4102390 2024-04-25T00:26:59.043 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.038+0000 7f53dc081700 1 -- 172.21.15.130:0/3611694872 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f53d4068f20 con 0x7f53d4102390 2024-04-25T00:26:59.043 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.040+0000 7f53c6ffd700 1 -- 172.21.15.130:0/3611694872 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f53d00051d0 con 0x7f53d4102390 2024-04-25T00:26:59.043 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.040+0000 7f53c6ffd700 1 --2- 172.21.15.130:0/3611694872 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f53c00a2c20 0x7f53c00a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:26:59.043 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.040+0000 7f53c6ffd700 1 -- 172.21.15.130:0/3611694872 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(34..34 src has 1..34) ==== 4101+0+0 (secure 0 0 0) 0x7f53d00ce270 con 0x7f53d4102390 2024-04-25T00:26:59.043 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.040+0000 7f53d9e1d700 1 --2- 172.21.15.130:0/3611694872 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f53c00a2c20 0x7f53c00a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:26:59.043 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.041+0000 7f53d9e1d700 1 --2- 172.21.15.130:0/3611694872 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f53c00a2c20 0x7f53c00a50e0 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f53d41021f0 tx=0x7f53c8007400 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:26:59.045 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.045+0000 7f53c6ffd700 1 -- 172.21.15.130:0/3611694872 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f53d0020020 con 0x7f53d4102390 2024-04-25T00:26:59.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:58 smithi121 ceph-mon[32231]: osdmap e34: 5 total, 5 up, 5 in 2024-04-25T00:26:59.248 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:26:59.248+0000 7f53dc081700 1 -- 172.21.15.130:0/3611694872 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_3", "target": ["mon-mgr", ""]}) -- 0x7f53d40fe8e0 con 0x7f53c00a2c20 2024-04-25T00:26:59.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:59 smithi032 ceph-mon[29357]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:26:59.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:59 smithi032 ceph-mon[29357]: from='client.24232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:26:59.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:59 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:26:59.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:59 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:26:59.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:26:59 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:00.015 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:59 smithi130 ceph-mon[32525]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:00.016 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:59 smithi130 ceph-mon[32525]: from='client.24232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:27:00.016 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:59 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:27:00.016 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:59 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:27:00.016 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:26:59 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:00.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:59 smithi121 ceph-mon[32231]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:00.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:59 smithi121 ceph-mon[32231]: from='client.24232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:27:00.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:59 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:27:00.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:59 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:27:00.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:26:59 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:02.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:01 smithi130 ceph-mon[32525]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:02.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:01 smithi032 ceph-mon[29357]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:02.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:01 smithi121 ceph-mon[32231]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:03.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:02 smithi130 ceph-mon[32525]: from='client.? 172.21.15.130:0/3473449450' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3360c304-a7ae-4efc-a931-3b6261bbf24e"}]: dispatch 2024-04-25T00:27:03.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:02 smithi130 ceph-mon[32525]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3360c304-a7ae-4efc-a931-3b6261bbf24e"}]: dispatch 2024-04-25T00:27:03.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:02 smithi130 ceph-mon[32525]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3360c304-a7ae-4efc-a931-3b6261bbf24e"}]': finished 2024-04-25T00:27:03.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:02 smithi130 ceph-mon[32525]: osdmap e35: 6 total, 5 up, 6 in 2024-04-25T00:27:03.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:02 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T00:27:03.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:02 smithi130 ceph-mon[32525]: from='client.? 172.21.15.130:0/1158059914' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:27:03.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:02 smithi032 ceph-mon[29357]: from='client.? 172.21.15.130:0/3473449450' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3360c304-a7ae-4efc-a931-3b6261bbf24e"}]: dispatch 2024-04-25T00:27:03.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:02 smithi032 ceph-mon[29357]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3360c304-a7ae-4efc-a931-3b6261bbf24e"}]: dispatch 2024-04-25T00:27:03.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:02 smithi032 ceph-mon[29357]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3360c304-a7ae-4efc-a931-3b6261bbf24e"}]': finished 2024-04-25T00:27:03.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:02 smithi032 ceph-mon[29357]: osdmap e35: 6 total, 5 up, 6 in 2024-04-25T00:27:03.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:02 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T00:27:03.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:02 smithi032 ceph-mon[29357]: from='client.? 172.21.15.130:0/1158059914' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:27:03.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:02 smithi121 ceph-mon[32231]: from='client.? 172.21.15.130:0/3473449450' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3360c304-a7ae-4efc-a931-3b6261bbf24e"}]: dispatch 2024-04-25T00:27:03.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:02 smithi121 ceph-mon[32231]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3360c304-a7ae-4efc-a931-3b6261bbf24e"}]: dispatch 2024-04-25T00:27:03.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:02 smithi121 ceph-mon[32231]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3360c304-a7ae-4efc-a931-3b6261bbf24e"}]': finished 2024-04-25T00:27:03.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:02 smithi121 ceph-mon[32231]: osdmap e35: 6 total, 5 up, 6 in 2024-04-25T00:27:03.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:02 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T00:27:03.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:02 smithi121 ceph-mon[32231]: from='client.? 172.21.15.130:0/1158059914' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:27:04.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:03 smithi032 ceph-mon[29357]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:04.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:03 smithi121 ceph-mon[32231]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:04.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:03 smithi130 ceph-mon[32525]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:06.230 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:05 smithi130 ceph-mon[32525]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:06.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:05 smithi032 ceph-mon[29357]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:06.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:05 smithi121 ceph-mon[32231]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:08.004 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:07 smithi130 ceph-mon[32525]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:08.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:07 smithi032 ceph-mon[29357]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:08.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:07 smithi121 ceph-mon[32231]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:10.004 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:09 smithi130 ceph-mon[32525]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:10.004 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:09 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-25T00:27:10.004 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:09 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:10.004 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:09 smithi130 ceph-mon[32525]: Deploying daemon osd.5 on smithi130 2024-04-25T00:27:10.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:09 smithi032 ceph-mon[29357]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:10.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:09 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-25T00:27:10.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:09 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:10.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:09 smithi032 ceph-mon[29357]: Deploying daemon osd.5 on smithi130 2024-04-25T00:27:10.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:09 smithi121 ceph-mon[32231]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:10.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:09 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-25T00:27:10.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:09 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:10.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:09 smithi121 ceph-mon[32231]: Deploying daemon osd.5 on smithi130 2024-04-25T00:27:12.162 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:11 smithi130 ceph-mon[32525]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:12.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:11 smithi032 ceph-mon[29357]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:12.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:11 smithi121 ceph-mon[32231]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:14.071 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:13 smithi130 ceph-mon[32525]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:14.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:13 smithi032 ceph-mon[29357]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:14.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:13 smithi121 ceph-mon[32231]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:16.073 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:15 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:16.073 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:15 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:16.073 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:15 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:27:16.074 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:15 smithi130 ceph-mon[32525]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:16.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:15 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:16.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:15 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:16.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:15 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:27:16.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:15 smithi121 ceph-mon[32231]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:16.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:15 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:16.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:15 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:16.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:15 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:27:16.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:15 smithi032 ceph-mon[29357]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:18.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:17 smithi130 ceph-mon[32525]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:18.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:17 smithi032 ceph-mon[29357]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:18.320 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:27:18.319+0000 7f53c6ffd700 1 -- 172.21.15.130:0/3611694872 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7f53d40fe8e0 con 0x7f53c00a2c20 2024-04-25T00:27:18.320 INFO:teuthology.orchestra.run.smithi130.stdout:Created osd(s) 5 on host 'smithi130' 2024-04-25T00:27:18.323 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:27:18.322+0000 7f53dc081700 1 -- 172.21.15.130:0/3611694872 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f53c00a2c20 msgr2=0x7f53c00a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:27:18.323 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:27:18.322+0000 7f53dc081700 1 --2- 172.21.15.130:0/3611694872 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f53c00a2c20 0x7f53c00a50e0 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f53d41021f0 tx=0x7f53c8007400 comp rx=0 tx=0).stop 2024-04-25T00:27:18.323 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:27:18.322+0000 7f53dc081700 1 -- 172.21.15.130:0/3611694872 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53d4102390 msgr2=0x7f53d4103770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:27:18.323 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:27:18.322+0000 7f53dc081700 1 --2- 172.21.15.130:0/3611694872 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53d4102390 0x7f53d4103770 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f53d000d980 tx=0x7f53d000dc90 comp rx=0 tx=0).stop 2024-04-25T00:27:18.323 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:27:18.322+0000 7f53dc081700 1 -- 172.21.15.130:0/3611694872 shutdown_connections 2024-04-25T00:27:18.323 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:27:18.322+0000 7f53dc081700 1 --2- 172.21.15.130:0/3611694872 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f53c00a2c20 0x7f53c00a50e0 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:27:18.323 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:27:18.322+0000 7f53dc081700 1 --2- 172.21.15.130:0/3611694872 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f53d4102d30 0x7f53d4103cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:27:18.324 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:27:18.322+0000 7f53dc081700 1 --2- 172.21.15.130:0/3611694872 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f53d4101190 0x7f53d4105120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:27:18.324 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:27:18.322+0000 7f53dc081700 1 --2- 172.21.15.130:0/3611694872 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53d4102390 0x7f53d4103770 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:27:18.324 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:27:18.322+0000 7f53dc081700 1 -- 172.21.15.130:0/3611694872 >> 172.21.15.130:0/3611694872 conn(0x7f53d40fc730 msgr2=0x7f53d40fe1d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:27:18.324 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:27:18.322+0000 7f53dc081700 1 -- 172.21.15.130:0/3611694872 shutdown_connections 2024-04-25T00:27:18.324 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:27:18.322+0000 7f53dc081700 1 -- 172.21.15.130:0/3611694872 wait complete. 2024-04-25T00:27:18.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:17 smithi121 ceph-mon[32231]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:19.007 DEBUG:teuthology.orchestra.run.smithi130:osd.5> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.5.service 2024-04-25T00:27:19.010 INFO:tasks.cephadm:Deploying osd.6 on smithi032 with /dev/vg_nvme/lv_2... 2024-04-25T00:27:19.010 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-25T00:27:19.269 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:27:19.544 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:19 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:19.544 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:19 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:19.544 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:19 smithi032 ceph-mon[29357]: from='osd.5 [v2:172.21.15.130:6808/909656116,v1:172.21.15.130:6809/909656116]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-25T00:27:19.544 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:19 smithi032 ceph-mon[29357]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-25T00:27:19.544 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:19 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:19.544 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:19 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:19.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:19 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:19.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:19 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:19.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:19 smithi121 ceph-mon[32231]: from='osd.5 [v2:172.21.15.130:6808/909656116,v1:172.21.15.130:6809/909656116]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-25T00:27:19.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:19 smithi121 ceph-mon[32231]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-25T00:27:19.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:19 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:19.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:19 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:19.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:19 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:19.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:19 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:19.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:19 smithi130 ceph-mon[32525]: from='osd.5 [v2:172.21.15.130:6808/909656116,v1:172.21.15.130:6809/909656116]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-25T00:27:19.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:19 smithi130 ceph-mon[32525]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-25T00:27:19.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:19 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:19.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:19 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:20.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:20 smithi121 ceph-mon[32231]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:20.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:20 smithi121 ceph-mon[32231]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-25T00:27:20.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:20 smithi121 ceph-mon[32231]: osdmap e36: 6 total, 5 up, 6 in 2024-04-25T00:27:20.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:20 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T00:27:20.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:20 smithi121 ceph-mon[32231]: from='osd.5 [v2:172.21.15.130:6808/909656116,v1:172.21.15.130:6809/909656116]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-25T00:27:20.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:20 smithi121 ceph-mon[32231]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-25T00:27:20.603 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:20 smithi032 ceph-mon[29357]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:20.603 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:20 smithi032 ceph-mon[29357]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-25T00:27:20.603 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:20 smithi032 ceph-mon[29357]: osdmap e36: 6 total, 5 up, 6 in 2024-04-25T00:27:20.603 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:20 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T00:27:20.603 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:20 smithi032 ceph-mon[29357]: from='osd.5 [v2:172.21.15.130:6808/909656116,v1:172.21.15.130:6809/909656116]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-25T00:27:20.603 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:20 smithi032 ceph-mon[29357]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-25T00:27:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:20 smithi130 ceph-mon[32525]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:20 smithi130 ceph-mon[32525]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-25T00:27:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:20 smithi130 ceph-mon[32525]: osdmap e36: 6 total, 5 up, 6 in 2024-04-25T00:27:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:20 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T00:27:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:20 smithi130 ceph-mon[32525]: from='osd.5 [v2:172.21.15.130:6808/909656116,v1:172.21.15.130:6809/909656116]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-25T00:27:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:20 smithi130 ceph-mon[32525]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-25T00:27:20.642 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:27:20 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:27:20.309+0000 7fbd0f30c700 -1 osd.5 0 waiting for initial osdmap 2024-04-25T00:27:20.643 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:27:20 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:27:20.315+0000 7fbd0b0ff700 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T00:27:21.362 INFO:teuthology.orchestra.run.smithi032.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-25T00:27:21.363 INFO:teuthology.orchestra.run.smithi032.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-25T00:27:21.363 INFO:teuthology.orchestra.run.smithi032.stderr: stderr: 10+0 records in 2024-04-25T00:27:21.363 INFO:teuthology.orchestra.run.smithi032.stderr:10+0 records out 2024-04-25T00:27:21.363 INFO:teuthology.orchestra.run.smithi032.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00867964 s, 1.2 GB/s 2024-04-25T00:27:21.363 INFO:teuthology.orchestra.run.smithi032.stderr:--> Zapping successful for: 2024-04-25T00:27:21.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:21 smithi130 ceph-mon[32525]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-04-25T00:27:21.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:21 smithi130 ceph-mon[32525]: osdmap e37: 6 total, 5 up, 6 in 2024-04-25T00:27:21.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:21 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T00:27:21.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:21 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T00:27:21.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:21 smithi121 ceph-mon[32231]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-04-25T00:27:21.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:21 smithi121 ceph-mon[32231]: osdmap e37: 6 total, 5 up, 6 in 2024-04-25T00:27:21.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:21 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T00:27:21.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:21 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T00:27:21.670 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:21 smithi032 ceph-mon[29357]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-04-25T00:27:21.670 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:21 smithi032 ceph-mon[29357]: osdmap e37: 6 total, 5 up, 6 in 2024-04-25T00:27:21.670 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:21 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T00:27:21.670 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:21 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T00:27:22.061 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi032:vg_nvme/lv_2 2024-04-25T00:27:22.311 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:27:22.479 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:22 smithi130 ceph-mon[32525]: purged_snaps scrub starts 2024-04-25T00:27:22.479 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:22 smithi130 ceph-mon[32525]: purged_snaps scrub ok 2024-04-25T00:27:22.479 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:22 smithi130 ceph-mon[32525]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:22.479 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:22 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T00:27:22.479 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:22 smithi130 ceph-mon[32525]: osd.5 [v2:172.21.15.130:6808/909656116,v1:172.21.15.130:6809/909656116] boot 2024-04-25T00:27:22.479 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:22 smithi130 ceph-mon[32525]: osdmap e38: 6 total, 6 up, 6 in 2024-04-25T00:27:22.479 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:22 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T00:27:22.479 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:22 smithi130 ceph-mon[32525]: osdmap e39: 6 total, 6 up, 6 in 2024-04-25T00:27:22.569 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:22 smithi032 ceph-mon[29357]: purged_snaps scrub starts 2024-04-25T00:27:22.569 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:22 smithi032 ceph-mon[29357]: purged_snaps scrub ok 2024-04-25T00:27:22.569 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:22 smithi032 ceph-mon[29357]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:22.569 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:22 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T00:27:22.569 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:22 smithi032 ceph-mon[29357]: osd.5 [v2:172.21.15.130:6808/909656116,v1:172.21.15.130:6809/909656116] boot 2024-04-25T00:27:22.570 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:22 smithi032 ceph-mon[29357]: osdmap e38: 6 total, 6 up, 6 in 2024-04-25T00:27:22.570 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:22 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T00:27:22.570 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:22 smithi032 ceph-mon[29357]: osdmap e39: 6 total, 6 up, 6 in 2024-04-25T00:27:22.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:22 smithi121 ceph-mon[32231]: purged_snaps scrub starts 2024-04-25T00:27:22.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:22 smithi121 ceph-mon[32231]: purged_snaps scrub ok 2024-04-25T00:27:22.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:22 smithi121 ceph-mon[32231]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-25T00:27:22.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:22 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T00:27:22.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:22 smithi121 ceph-mon[32231]: osd.5 [v2:172.21.15.130:6808/909656116,v1:172.21.15.130:6809/909656116] boot 2024-04-25T00:27:22.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:22 smithi121 ceph-mon[32231]: osdmap e38: 6 total, 6 up, 6 in 2024-04-25T00:27:22.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:22 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T00:27:22.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:22 smithi121 ceph-mon[32231]: osdmap e39: 6 total, 6 up, 6 in 2024-04-25T00:27:23.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:23 smithi032 ceph-mon[29357]: Detected new or changed devices on smithi130 2024-04-25T00:27:23.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:23 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:23.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:23 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:23.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:23 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:27:23.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:23 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:27:23.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:23 smithi032 ceph-mon[29357]: Adjusting osd_memory_target on smithi130 to 8544M 2024-04-25T00:27:23.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:23 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:23.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:23 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:23.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:23 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:27:23.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:23 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:23.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:23 smithi032 ceph-mon[29357]: osdmap e40: 6 total, 6 up, 6 in 2024-04-25T00:27:23.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:23 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:27:23.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:23 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:23.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:23 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:27:23.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:23 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:23.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:23 smithi121 ceph-mon[32231]: Detected new or changed devices on smithi130 2024-04-25T00:27:23.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:23 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:23.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:23 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:23.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:23 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:27:23.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:23 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:27:23.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:23 smithi121 ceph-mon[32231]: Adjusting osd_memory_target on smithi130 to 8544M 2024-04-25T00:27:23.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:23 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:23.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:23 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:23.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:23 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:27:23.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:23 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:23.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:23 smithi121 ceph-mon[32231]: osdmap e40: 6 total, 6 up, 6 in 2024-04-25T00:27:23.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:23 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:27:23.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:23 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:23.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:23 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:27:23.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:23 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:23 smithi130 ceph-mon[32525]: Detected new or changed devices on smithi130 2024-04-25T00:27:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:23 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:23 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:23 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:27:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:23 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:27:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:23 smithi130 ceph-mon[32525]: Adjusting osd_memory_target on smithi130 to 8544M 2024-04-25T00:27:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:23 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:23 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:23 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:27:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:23 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:23 smithi130 ceph-mon[32525]: osdmap e40: 6 total, 6 up, 6 in 2024-04-25T00:27:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:23 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:27:23.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:23 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:23.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:23 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:27:23.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:23 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:24.038 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.037+0000 7fd0ec448700 1 -- 172.21.15.32:0/4209149077 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd0e4102d30 msgr2=0x7fd0e41095e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:27:24.038 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.037+0000 7fd0ec448700 1 --2- 172.21.15.32:0/4209149077 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd0e4102d30 0x7fd0e41095e0 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7fd0dc00b0d0 tx=0x7fd0dc00b3e0 comp rx=0 tx=0).stop 2024-04-25T00:27:24.038 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.037+0000 7fd0ec448700 1 -- 172.21.15.32:0/4209149077 shutdown_connections 2024-04-25T00:27:24.038 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.037+0000 7fd0ec448700 1 --2- 172.21.15.32:0/4209149077 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd0e4102d30 0x7fd0e41095e0 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:27:24.038 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.037+0000 7fd0ec448700 1 --2- 172.21.15.32:0/4209149077 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd0e4101190 0x7fd0e41015b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:27:24.038 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.037+0000 7fd0ec448700 1 --2- 172.21.15.32:0/4209149077 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd0e4102390 0x7fd0e41027f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:27:24.038 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.037+0000 7fd0ec448700 1 -- 172.21.15.32:0/4209149077 >> 172.21.15.32:0/4209149077 conn(0x7fd0e40fc730 msgr2=0x7fd0e40feb70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:27:24.038 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.037+0000 7fd0ec448700 1 -- 172.21.15.32:0/4209149077 shutdown_connections 2024-04-25T00:27:24.039 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.037+0000 7fd0ec448700 1 -- 172.21.15.32:0/4209149077 wait complete. 2024-04-25T00:27:24.041 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.038+0000 7fd0ec448700 1 Processor -- start 2024-04-25T00:27:24.041 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.038+0000 7fd0ec448700 1 -- start start 2024-04-25T00:27:24.041 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.038+0000 7fd0ec448700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd0e4101190 0x7fd0e4198460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:27:24.041 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.038+0000 7fd0ec448700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd0e4102390 0x7fd0e41989a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:27:24.041 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.038+0000 7fd0ec448700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd0e4102d30 0x7fd0e419fa20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:27:24.041 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.038+0000 7fd0ec448700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fd0e410cdc0 con 0x7fd0e4101190 2024-04-25T00:27:24.041 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.038+0000 7fd0ec448700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fd0e410cc40 con 0x7fd0e4102d30 2024-04-25T00:27:24.041 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.038+0000 7fd0ec448700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fd0e410cf40 con 0x7fd0e4102390 2024-04-25T00:27:24.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.038+0000 7fd0ea1e4700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd0e4101190 0x7fd0e4198460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:27:24.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.038+0000 7fd0ea1e4700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd0e4101190 0x7fd0e4198460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:48900/0 (socket says 172.21.15.32:48900) 2024-04-25T00:27:24.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.038+0000 7fd0ea1e4700 1 -- 172.21.15.32:0/453879844 learned_addr learned my addr 172.21.15.32:0/453879844 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:27:24.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.038+0000 7fd0e99e3700 1 --2- 172.21.15.32:0/453879844 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd0e4102390 0x7fd0e41989a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:27:24.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.038+0000 7fd0ea9e5700 1 --2- 172.21.15.32:0/453879844 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd0e4102d30 0x7fd0e419fa20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:27:24.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.038+0000 7fd0ea1e4700 1 -- 172.21.15.32:0/453879844 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd0e4102390 msgr2=0x7fd0e41989a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:27:24.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.038+0000 7fd0ea1e4700 1 --2- 172.21.15.32:0/453879844 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd0e4102390 0x7fd0e41989a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:27:24.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.038+0000 7fd0ea1e4700 1 -- 172.21.15.32:0/453879844 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd0e4102d30 msgr2=0x7fd0e419fa20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:27:24.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.038+0000 7fd0ea1e4700 1 --2- 172.21.15.32:0/453879844 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd0e4102d30 0x7fd0e419fa20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:27:24.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.038+0000 7fd0ea1e4700 1 -- 172.21.15.32:0/453879844 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd0e41a00c0 con 0x7fd0e4101190 2024-04-25T00:27:24.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.038+0000 7fd0e99e3700 1 --2- 172.21.15.32:0/453879844 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd0e4102390 0x7fd0e41989a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:27:24.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.039+0000 7fd0ea1e4700 1 --2- 172.21.15.32:0/453879844 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd0e4101190 0x7fd0e4198460 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7fd0e000d9b0 tx=0x7fd0e000dd70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:27:24.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.039+0000 7fd0db7fe700 1 -- 172.21.15.32:0/453879844 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd0e0017030 con 0x7fd0e4101190 2024-04-25T00:27:24.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.039+0000 7fd0ec448700 1 -- 172.21.15.32:0/453879844 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd0e41a03b0 con 0x7fd0e4101190 2024-04-25T00:27:24.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.039+0000 7fd0db7fe700 1 -- 172.21.15.32:0/453879844 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd0e0004500 con 0x7fd0e4101190 2024-04-25T00:27:24.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.039+0000 7fd0db7fe700 1 -- 172.21.15.32:0/453879844 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd0e0003aa0 con 0x7fd0e4101190 2024-04-25T00:27:24.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.039+0000 7fd0ec448700 1 -- 172.21.15.32:0/453879844 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd0e41a0940 con 0x7fd0e4101190 2024-04-25T00:27:24.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.040+0000 7fd0ec448700 1 -- 172.21.15.32:0/453879844 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd0e4068f20 con 0x7fd0e4101190 2024-04-25T00:27:24.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.041+0000 7fd0db7fe700 1 -- 172.21.15.32:0/453879844 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7fd0e001a070 con 0x7fd0e4101190 2024-04-25T00:27:24.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.042+0000 7fd0db7fe700 1 --2- 172.21.15.32:0/453879844 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fd0d00a2c20 0x7fd0d00a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:27:24.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.042+0000 7fd0e99e3700 1 --2- 172.21.15.32:0/453879844 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fd0d00a2c20 0x7fd0d00a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:27:24.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.042+0000 7fd0db7fe700 1 -- 172.21.15.32:0/453879844 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(41..41 src has 1..41) ==== 4533+0+0 (secure 0 0 0) 0x7fd0e00cdc00 con 0x7fd0e4101190 2024-04-25T00:27:24.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.042+0000 7fd0e99e3700 1 --2- 172.21.15.32:0/453879844 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fd0d00a2c20 0x7fd0d00a50e0 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7fd0d4005950 tx=0x7fd0d40058e0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:27:24.047 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.046+0000 7fd0db7fe700 1 -- 172.21.15.32:0/453879844 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fd0e009b980 con 0x7fd0e4101190 2024-04-25T00:27:24.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:24.265+0000 7fd0ec448700 1 -- 172.21.15.32:0/453879844 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_2", "target": ["mon-mgr", ""]}) -- 0x7fd0e40fe840 con 0x7fd0d00a2c20 2024-04-25T00:27:24.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:24 smithi121 ceph-mon[32231]: pgmap v140: 1 pgs: 1 remapped+peering; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-25T00:27:24.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:24 smithi121 ceph-mon[32231]: osdmap e41: 6 total, 6 up, 6 in 2024-04-25T00:27:24.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:24 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:27:24.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:24 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:27:24.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:24 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:24.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:24 smithi130 ceph-mon[32525]: pgmap v140: 1 pgs: 1 remapped+peering; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-25T00:27:24.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:24 smithi130 ceph-mon[32525]: osdmap e41: 6 total, 6 up, 6 in 2024-04-25T00:27:24.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:24 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:27:24.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:24 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:27:24.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:24 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:24.907 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:24 smithi032 ceph-mon[29357]: pgmap v140: 1 pgs: 1 remapped+peering; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-25T00:27:24.907 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:24 smithi032 ceph-mon[29357]: osdmap e41: 6 total, 6 up, 6 in 2024-04-25T00:27:24.908 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:24 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:27:24.908 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:24 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:27:24.908 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:24 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:25.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:25 smithi121 ceph-mon[32231]: from='client.14367 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:27:25.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:25 smithi121 ceph-mon[32231]: osdmap e42: 6 total, 6 up, 6 in 2024-04-25T00:27:25.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:25 smithi130 ceph-mon[32525]: from='client.14367 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:27:25.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:25 smithi130 ceph-mon[32525]: osdmap e42: 6 total, 6 up, 6 in 2024-04-25T00:27:25.931 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:25 smithi032 ceph-mon[29357]: from='client.14367 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:27:25.932 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:25 smithi032 ceph-mon[29357]: osdmap e42: 6 total, 6 up, 6 in 2024-04-25T00:27:26.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:26 smithi121 ceph-mon[32231]: pgmap v143: 1 pgs: 1 remapped+peering; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-25T00:27:26.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:26 smithi130 ceph-mon[32525]: pgmap v143: 1 pgs: 1 remapped+peering; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-25T00:27:26.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:26 smithi032 ceph-mon[29357]: pgmap v143: 1 pgs: 1 remapped+peering; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-25T00:27:27.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:27 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1049183537' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f60af4f3-4dd1-4021-a778-d0d97e9a674e"}]: dispatch 2024-04-25T00:27:27.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:27 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1049183537' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f60af4f3-4dd1-4021-a778-d0d97e9a674e"}]': finished 2024-04-25T00:27:27.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:27 smithi121 ceph-mon[32231]: osdmap e43: 7 total, 6 up, 7 in 2024-04-25T00:27:27.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:27 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T00:27:27.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:27 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1049183537' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f60af4f3-4dd1-4021-a778-d0d97e9a674e"}]: dispatch 2024-04-25T00:27:27.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:27 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1049183537' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f60af4f3-4dd1-4021-a778-d0d97e9a674e"}]': finished 2024-04-25T00:27:27.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:27 smithi130 ceph-mon[32525]: osdmap e43: 7 total, 6 up, 7 in 2024-04-25T00:27:27.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:27 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T00:27:27.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:27 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1049183537' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f60af4f3-4dd1-4021-a778-d0d97e9a674e"}]: dispatch 2024-04-25T00:27:27.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:27 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1049183537' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f60af4f3-4dd1-4021-a778-d0d97e9a674e"}]': finished 2024-04-25T00:27:27.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:27 smithi032 ceph-mon[29357]: osdmap e43: 7 total, 6 up, 7 in 2024-04-25T00:27:27.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:27 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T00:27:28.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:28 smithi121 ceph-mon[32231]: pgmap v144: 1 pgs: 1 remapped+peering; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-25T00:27:28.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:28 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3397791522' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:27:28.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:28 smithi130 ceph-mon[32525]: pgmap v144: 1 pgs: 1 remapped+peering; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-25T00:27:28.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:28 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3397791522' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:27:28.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:28 smithi032 ceph-mon[29357]: pgmap v144: 1 pgs: 1 remapped+peering; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-25T00:27:28.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:28 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3397791522' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:27:30.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:30 smithi121 ceph-mon[32231]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-25T00:27:30.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:30 smithi130 ceph-mon[32525]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-25T00:27:30.972 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:30 smithi032 ceph-mon[29357]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-25T00:27:32.657 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:32 smithi032 ceph-mon[29357]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-04-25T00:27:32.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:32 smithi121 ceph-mon[32231]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-04-25T00:27:32.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:32 smithi130 ceph-mon[32525]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-04-25T00:27:34.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:34 smithi121 ceph-mon[32231]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-25T00:27:34.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:34 smithi130 ceph-mon[32525]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-25T00:27:34.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:34 smithi032 ceph-mon[29357]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-25T00:27:35.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:35 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-25T00:27:35.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:35 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:35.841 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:35 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-25T00:27:35.842 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:35 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:35.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:35 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-25T00:27:35.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:35 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:36.831 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:36 smithi032 ceph-mon[29357]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-25T00:27:36.831 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:36 smithi032 ceph-mon[29357]: Deploying daemon osd.6 on smithi032 2024-04-25T00:27:36.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:36 smithi121 ceph-mon[32231]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-25T00:27:36.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:36 smithi121 ceph-mon[32231]: Deploying daemon osd.6 on smithi032 2024-04-25T00:27:36.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:36 smithi130 ceph-mon[32525]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-25T00:27:36.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:36 smithi130 ceph-mon[32525]: Deploying daemon osd.6 on smithi032 2024-04-25T00:27:38.725 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:38 smithi032 ceph-mon[29357]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-25T00:27:38.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:38 smithi121 ceph-mon[32231]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-25T00:27:38.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:38 smithi130 ceph-mon[32525]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-25T00:27:40.713 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:40 smithi032 ceph-mon[29357]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-04-25T00:27:40.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:40 smithi121 ceph-mon[32231]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-04-25T00:27:40.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:40 smithi130 ceph-mon[32525]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-04-25T00:27:42.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:42 smithi032 ceph-mon[29357]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-25T00:27:42.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:42.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:42.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:27:42.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:42 smithi121 ceph-mon[32231]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-25T00:27:42.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:42 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:42.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:42 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:42.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:42 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:27:42.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:42 smithi130 ceph-mon[32525]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-25T00:27:42.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:42 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:42.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:42 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:42.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:42 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:27:44.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:44 smithi032 ceph-mon[29357]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-25T00:27:44.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:44 smithi121 ceph-mon[32231]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-25T00:27:44.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:44 smithi130 ceph-mon[32525]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-25T00:27:45.730 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:45.729+0000 7fd0db7fe700 1 -- 172.21.15.32:0/453879844 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7fd0e40fe840 con 0x7fd0d00a2c20 2024-04-25T00:27:45.730 INFO:teuthology.orchestra.run.smithi032.stdout:Created osd(s) 6 on host 'smithi032' 2024-04-25T00:27:45.732 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:45.732+0000 7fd0ec448700 1 -- 172.21.15.32:0/453879844 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fd0d00a2c20 msgr2=0x7fd0d00a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:27:45.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:45.732+0000 7fd0ec448700 1 --2- 172.21.15.32:0/453879844 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fd0d00a2c20 0x7fd0d00a50e0 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7fd0d4005950 tx=0x7fd0d40058e0 comp rx=0 tx=0).stop 2024-04-25T00:27:45.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:45.732+0000 7fd0ec448700 1 -- 172.21.15.32:0/453879844 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd0e4101190 msgr2=0x7fd0e4198460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:27:45.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:45.732+0000 7fd0ec448700 1 --2- 172.21.15.32:0/453879844 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd0e4101190 0x7fd0e4198460 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7fd0e000d9b0 tx=0x7fd0e000dd70 comp rx=0 tx=0).stop 2024-04-25T00:27:45.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:45.732+0000 7fd0ec448700 1 -- 172.21.15.32:0/453879844 shutdown_connections 2024-04-25T00:27:45.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:45.732+0000 7fd0ec448700 1 --2- 172.21.15.32:0/453879844 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fd0d00a2c20 0x7fd0d00a50e0 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:27:45.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:45.732+0000 7fd0ec448700 1 --2- 172.21.15.32:0/453879844 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd0e4102d30 0x7fd0e419fa20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:27:45.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:45.732+0000 7fd0ec448700 1 --2- 172.21.15.32:0/453879844 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd0e4101190 0x7fd0e4198460 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:27:45.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:45.732+0000 7fd0ec448700 1 --2- 172.21.15.32:0/453879844 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd0e4102390 0x7fd0e41989a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:27:45.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:45.732+0000 7fd0ec448700 1 -- 172.21.15.32:0/453879844 >> 172.21.15.32:0/453879844 conn(0x7fd0e40fc730 msgr2=0x7fd0e40fe130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:27:45.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:45.732+0000 7fd0ec448700 1 -- 172.21.15.32:0/453879844 shutdown_connections 2024-04-25T00:27:45.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:27:45.732+0000 7fd0ec448700 1 -- 172.21.15.32:0/453879844 wait complete. 2024-04-25T00:27:45.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:45.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:45.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:45.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:27:45.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:45 smithi032 ceph-mon[29357]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-25T00:27:45.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:45.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:45 smithi032 ceph-mon[29357]: from='osd.6 [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-25T00:27:45.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:45.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:45.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:27:46.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:46.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:46.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:46.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:27:46.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:45 smithi121 ceph-mon[32231]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-25T00:27:46.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:46.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:45 smithi121 ceph-mon[32231]: from='osd.6 [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-25T00:27:46.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:46.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:46.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:27:46.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:46.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:46.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:46.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:27:46.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:45 smithi130 ceph-mon[32525]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-25T00:27:46.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:46.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:45 smithi130 ceph-mon[32525]: from='osd.6 [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-25T00:27:46.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:46.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:46.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:27:46.538 DEBUG:teuthology.orchestra.run.smithi032:osd.6> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.6.service 2024-04-25T00:27:46.541 INFO:tasks.cephadm:Deploying osd.7 on smithi121 with /dev/vg_nvme/lv_2... 2024-04-25T00:27:46.541 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-25T00:27:46.786 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.b/config 2024-04-25T00:27:47.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:46 smithi121 ceph-mon[32231]: from='osd.6 [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-25T00:27:47.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:46 smithi121 ceph-mon[32231]: osdmap e44: 7 total, 6 up, 7 in 2024-04-25T00:27:47.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:46 smithi121 ceph-mon[32231]: from='osd.6 [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-04-25T00:27:47.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:46 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T00:27:47.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:46 smithi130 ceph-mon[32525]: from='osd.6 [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-25T00:27:47.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:46 smithi130 ceph-mon[32525]: osdmap e44: 7 total, 6 up, 7 in 2024-04-25T00:27:47.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:46 smithi130 ceph-mon[32525]: from='osd.6 [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-04-25T00:27:47.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:46 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T00:27:47.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:46 smithi032 ceph-mon[29357]: from='osd.6 [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-25T00:27:47.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:46 smithi032 ceph-mon[29357]: osdmap e44: 7 total, 6 up, 7 in 2024-04-25T00:27:47.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:46 smithi032 ceph-mon[29357]: from='osd.6 [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-04-25T00:27:47.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:46 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T00:27:47.737 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:27:47 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:27:47.393+0000 7efd3c792700 -1 osd.6 0 waiting for initial osdmap 2024-04-25T00:27:47.737 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:27:47 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:27:47.402+0000 7efd3457d700 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T00:27:47.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:47 smithi032 ceph-mon[29357]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-25T00:27:47.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:47 smithi032 ceph-mon[29357]: from='osd.6 [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-04-25T00:27:47.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:47 smithi032 ceph-mon[29357]: osdmap e45: 7 total, 6 up, 7 in 2024-04-25T00:27:47.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:47 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T00:27:47.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:47 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T00:27:47.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:47 smithi032 ceph-mon[29357]: from='osd.6 [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027]' entity='osd.6' 2024-04-25T00:27:47.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:47 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T00:27:48.054 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:47 smithi121 ceph-mon[32231]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-25T00:27:48.055 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:47 smithi121 ceph-mon[32231]: from='osd.6 [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-04-25T00:27:48.055 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:47 smithi121 ceph-mon[32231]: osdmap e45: 7 total, 6 up, 7 in 2024-04-25T00:27:48.055 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:47 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T00:27:48.055 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:47 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T00:27:48.055 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:47 smithi121 ceph-mon[32231]: from='osd.6 [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027]' entity='osd.6' 2024-04-25T00:27:48.055 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:47 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T00:27:48.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:47 smithi130 ceph-mon[32525]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-25T00:27:48.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:47 smithi130 ceph-mon[32525]: from='osd.6 [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-04-25T00:27:48.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:47 smithi130 ceph-mon[32525]: osdmap e45: 7 total, 6 up, 7 in 2024-04-25T00:27:48.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:47 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T00:27:48.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:47 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T00:27:48.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:47 smithi130 ceph-mon[32525]: from='osd.6 [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027]' entity='osd.6' 2024-04-25T00:27:48.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:47 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T00:27:48.962 INFO:teuthology.orchestra.run.smithi121.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-25T00:27:48.962 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-25T00:27:48.962 INFO:teuthology.orchestra.run.smithi121.stderr: stderr: 10+0 records in 2024-04-25T00:27:48.962 INFO:teuthology.orchestra.run.smithi121.stderr:10+0 records out 2024-04-25T00:27:48.962 INFO:teuthology.orchestra.run.smithi121.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0115345 s, 909 MB/s 2024-04-25T00:27:48.962 INFO:teuthology.orchestra.run.smithi121.stderr: stderr: 2024-04-25T00:27:48.963 INFO:teuthology.orchestra.run.smithi121.stderr:--> Zapping successful for: 2024-04-25T00:27:49.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:48 smithi130 ceph-mon[32525]: purged_snaps scrub starts 2024-04-25T00:27:49.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:48 smithi130 ceph-mon[32525]: purged_snaps scrub ok 2024-04-25T00:27:49.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:48 smithi130 ceph-mon[32525]: osd.6 [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027] boot 2024-04-25T00:27:49.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:48 smithi130 ceph-mon[32525]: osdmap e46: 7 total, 7 up, 7 in 2024-04-25T00:27:49.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:48 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T00:27:49.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:48 smithi032 ceph-mon[29357]: purged_snaps scrub starts 2024-04-25T00:27:49.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:48 smithi032 ceph-mon[29357]: purged_snaps scrub ok 2024-04-25T00:27:49.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:48 smithi032 ceph-mon[29357]: osd.6 [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027] boot 2024-04-25T00:27:49.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:48 smithi032 ceph-mon[29357]: osdmap e46: 7 total, 7 up, 7 in 2024-04-25T00:27:49.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:48 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T00:27:49.266 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:48 smithi121 ceph-mon[32231]: purged_snaps scrub starts 2024-04-25T00:27:49.266 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:48 smithi121 ceph-mon[32231]: purged_snaps scrub ok 2024-04-25T00:27:49.266 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:48 smithi121 ceph-mon[32231]: osd.6 [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027] boot 2024-04-25T00:27:49.266 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:48 smithi121 ceph-mon[32231]: osdmap e46: 7 total, 7 up, 7 in 2024-04-25T00:27:49.267 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:48 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T00:27:49.561 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi121:vg_nvme/lv_2 2024-04-25T00:27:49.802 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.b/config 2024-04-25T00:27:50.549 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:50 smithi121 ceph-mon[32231]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-25T00:27:50.549 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:50 smithi121 ceph-mon[32231]: osdmap e47: 7 total, 7 up, 7 in 2024-04-25T00:27:50.549 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:50 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:50.549 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:50 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:50.549 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:50 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:27:50.549 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:50 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:27:50.549 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:50 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:27:50.549 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:50 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:50.549 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:50 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:50.549 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:50 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:27:50.550 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:50 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:50.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:50 smithi032 ceph-mon[29357]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-25T00:27:50.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:50 smithi032 ceph-mon[29357]: osdmap e47: 7 total, 7 up, 7 in 2024-04-25T00:27:50.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:50 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:50.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:50 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:50.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:50 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:27:50.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:50 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:27:50.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:50 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:27:50.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:50 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:50.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:50 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:50.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:50 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:27:50.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:50 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:50.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:50 smithi130 ceph-mon[32525]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-25T00:27:50.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:50 smithi130 ceph-mon[32525]: osdmap e47: 7 total, 7 up, 7 in 2024-04-25T00:27:50.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:50 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:50.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:50 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:50.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:50 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:27:50.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:50 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:27:50.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:50 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:27:50.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:50 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:50.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:50 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:50.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:50 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:27:50.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:50 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:27:51.391 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.390+0000 7f4591990700 1 -- 172.21.15.121:0/1789772841 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f458c103e20 msgr2=0x7f458c10a6d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:27:51.391 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.390+0000 7f4591990700 1 --2- 172.21.15.121:0/1789772841 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f458c103e20 0x7f458c10a6d0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f458400b0d0 tx=0x7f458400b3e0 comp rx=0 tx=0).stop 2024-04-25T00:27:51.391 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.390+0000 7f4591990700 1 -- 172.21.15.121:0/1789772841 shutdown_connections 2024-04-25T00:27:51.391 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.390+0000 7f4591990700 1 --2- 172.21.15.121:0/1789772841 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f458c103e20 0x7f458c10a6d0 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:27:51.391 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.390+0000 7f4591990700 1 --2- 172.21.15.121:0/1789772841 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f458c102280 0x7f458c1026a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:27:51.391 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.390+0000 7f4591990700 1 --2- 172.21.15.121:0/1789772841 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f458c103480 0x7f458c1038e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:27:51.391 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.390+0000 7f4591990700 1 -- 172.21.15.121:0/1789772841 >> 172.21.15.121:0/1789772841 conn(0x7f458c0fd820 msgr2=0x7f458c0ffc60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:27:51.392 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.390+0000 7f4591990700 1 -- 172.21.15.121:0/1789772841 shutdown_connections 2024-04-25T00:27:51.392 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.390+0000 7f4591990700 1 -- 172.21.15.121:0/1789772841 wait complete. 2024-04-25T00:27:51.393 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.391+0000 7f4591990700 1 Processor -- start 2024-04-25T00:27:51.393 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.391+0000 7f4591990700 1 -- start start 2024-04-25T00:27:51.393 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.391+0000 7f4591990700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f458c102280 0x7f458c198410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:27:51.393 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.391+0000 7f4591990700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f458c103480 0x7f458c198950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:27:51.394 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.391+0000 7f4591990700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f458c103e20 0x7f458c19f9d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:27:51.394 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.391+0000 7f4591990700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f458c10de60 con 0x7f458c103e20 2024-04-25T00:27:51.394 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.391+0000 7f4591990700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f458c10dce0 con 0x7f458c102280 2024-04-25T00:27:51.394 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.391+0000 7f4591990700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f458c10dfe0 con 0x7f458c103480 2024-04-25T00:27:51.394 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.391+0000 7f458affd700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f458c102280 0x7f458c198410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:27:51.394 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.391+0000 7f458affd700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f458c102280 0x7f458c198410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.121:57710/0 (socket says 172.21.15.121:57710) 2024-04-25T00:27:51.394 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.391+0000 7f458affd700 1 -- 172.21.15.121:0/4059791478 learned_addr learned my addr 172.21.15.121:0/4059791478 (peer_addr_for_me v2:172.21.15.121:0/0) 2024-04-25T00:27:51.394 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.392+0000 7f458b7fe700 1 --2- 172.21.15.121:0/4059791478 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f458c103e20 0x7f458c19f9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:27:51.394 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.392+0000 7f458a7fc700 1 --2- 172.21.15.121:0/4059791478 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f458c103480 0x7f458c198950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:27:51.394 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.392+0000 7f458affd700 1 -- 172.21.15.121:0/4059791478 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f458c103480 msgr2=0x7f458c198950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:27:51.394 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.392+0000 7f458affd700 1 --2- 172.21.15.121:0/4059791478 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f458c103480 0x7f458c198950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:27:51.394 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.392+0000 7f458affd700 1 -- 172.21.15.121:0/4059791478 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f458c103e20 msgr2=0x7f458c19f9d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:27:51.394 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.392+0000 7f458affd700 1 --2- 172.21.15.121:0/4059791478 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f458c103e20 0x7f458c19f9d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:27:51.395 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.392+0000 7f458affd700 1 -- 172.21.15.121:0/4059791478 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f458c1a0070 con 0x7f458c102280 2024-04-25T00:27:51.395 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.392+0000 7f458affd700 1 --2- 172.21.15.121:0/4059791478 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f458c102280 0x7f458c198410 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f458000d980 tx=0x7f458000dc90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:27:51.395 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.392+0000 7f459098e700 1 -- 172.21.15.121:0/4059791478 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4580017070 con 0x7f458c102280 2024-04-25T00:27:51.395 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.392+0000 7f4591990700 1 -- 172.21.15.121:0/4059791478 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f458c0744c0 con 0x7f458c102280 2024-04-25T00:27:51.395 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.392+0000 7f459098e700 1 -- 172.21.15.121:0/4059791478 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4580009e10 con 0x7f458c102280 2024-04-25T00:27:51.395 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.392+0000 7f459098e700 1 -- 172.21.15.121:0/4059791478 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f458000f620 con 0x7f458c102280 2024-04-25T00:27:51.395 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.392+0000 7f4591990700 1 -- 172.21.15.121:0/4059791478 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f458c074a00 con 0x7f458c102280 2024-04-25T00:27:51.395 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.393+0000 7f4591990700 1 -- 172.21.15.121:0/4059791478 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f458c068f20 con 0x7f458c102280 2024-04-25T00:27:51.396 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.394+0000 7f459098e700 1 -- 172.21.15.121:0/4059791478 <== mon.2 v2:172.21.15.121:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f4580010710 con 0x7f458c102280 2024-04-25T00:27:51.396 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.395+0000 7f459098e700 1 --2- 172.21.15.121:0/4059791478 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f45740a2c20 0x7f45740a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:27:51.396 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.395+0000 7f459098e700 1 -- 172.21.15.121:0/4059791478 <== mon.2 v2:172.21.15.121:3300/0 5 ==== osd_map(47..47 src has 1..47) ==== 4965+0+0 (secure 0 0 0) 0x7f45800cde70 con 0x7f458c102280 2024-04-25T00:27:51.396 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.395+0000 7f458a7fc700 1 --2- 172.21.15.121:0/4059791478 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f45740a2c20 0x7f45740a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:27:51.396 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.395+0000 7f458a7fc700 1 --2- 172.21.15.121:0/4059791478 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f45740a2c20 0x7f45740a50e0 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f457c005950 tx=0x7f457c0058e0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:27:51.399 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.398+0000 7f459098e700 1 -- 172.21.15.121:0/4059791478 <== mon.2 v2:172.21.15.121:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f458009bae0 con 0x7f458c102280 2024-04-25T00:27:51.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:51 smithi121 ceph-mon[32231]: Detected new or changed devices on smithi032 2024-04-25T00:27:51.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:51 smithi121 ceph-mon[32231]: Adjusting osd_memory_target on smithi032 to 5697M 2024-04-25T00:27:51.606 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:27:51.606+0000 7f4591990700 1 -- 172.21.15.121:0/4059791478 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_2", "target": ["mon-mgr", ""]}) -- 0x7f458c075e50 con 0x7f45740a2c20 2024-04-25T00:27:51.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:51 smithi032 ceph-mon[29357]: Detected new or changed devices on smithi032 2024-04-25T00:27:51.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:51 smithi032 ceph-mon[29357]: Adjusting osd_memory_target on smithi032 to 5697M 2024-04-25T00:27:51.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:51 smithi130 ceph-mon[32525]: Detected new or changed devices on smithi032 2024-04-25T00:27:51.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:51 smithi130 ceph-mon[32525]: Adjusting osd_memory_target on smithi032 to 5697M 2024-04-25T00:27:52.498 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:52 smithi121 ceph-mon[32231]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 205 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:27:52.498 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:52 smithi121 ceph-mon[32231]: from='client.24269 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:27:52.498 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:52 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:27:52.498 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:52 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:27:52.498 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:52 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:52.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:52 smithi032 ceph-mon[29357]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 205 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:27:52.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:52 smithi032 ceph-mon[29357]: from='client.24269 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:27:52.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:52 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:27:52.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:52 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:27:52.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:52 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:52.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:52 smithi130 ceph-mon[32525]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 205 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:27:52.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:52 smithi130 ceph-mon[32525]: from='client.24269 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:27:52.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:52 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:27:52.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:52 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:27:52.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:52 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:27:54.678 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:54 smithi121 ceph-mon[32231]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:27:54.678 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:54 smithi121 ceph-mon[32231]: from='client.? 172.21.15.121:0/2992283460' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7181e550-d24b-40ee-8d52-9b650a041739"}]: dispatch 2024-04-25T00:27:54.678 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:54 smithi121 ceph-mon[32231]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7181e550-d24b-40ee-8d52-9b650a041739"}]: dispatch 2024-04-25T00:27:54.678 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:54 smithi121 ceph-mon[32231]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7181e550-d24b-40ee-8d52-9b650a041739"}]': finished 2024-04-25T00:27:54.678 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:54 smithi121 ceph-mon[32231]: osdmap e48: 8 total, 7 up, 8 in 2024-04-25T00:27:54.678 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:54 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T00:27:54.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:54 smithi032 ceph-mon[29357]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:27:54.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:54 smithi032 ceph-mon[29357]: from='client.? 172.21.15.121:0/2992283460' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7181e550-d24b-40ee-8d52-9b650a041739"}]: dispatch 2024-04-25T00:27:54.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:54 smithi032 ceph-mon[29357]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7181e550-d24b-40ee-8d52-9b650a041739"}]: dispatch 2024-04-25T00:27:54.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:54 smithi032 ceph-mon[29357]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7181e550-d24b-40ee-8d52-9b650a041739"}]': finished 2024-04-25T00:27:54.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:54 smithi032 ceph-mon[29357]: osdmap e48: 8 total, 7 up, 8 in 2024-04-25T00:27:54.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:54 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T00:27:54.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:54 smithi130 ceph-mon[32525]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:27:54.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:54 smithi130 ceph-mon[32525]: from='client.? 172.21.15.121:0/2992283460' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7181e550-d24b-40ee-8d52-9b650a041739"}]: dispatch 2024-04-25T00:27:54.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:54 smithi130 ceph-mon[32525]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7181e550-d24b-40ee-8d52-9b650a041739"}]: dispatch 2024-04-25T00:27:54.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:54 smithi130 ceph-mon[32525]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7181e550-d24b-40ee-8d52-9b650a041739"}]': finished 2024-04-25T00:27:54.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:54 smithi130 ceph-mon[32525]: osdmap e48: 8 total, 7 up, 8 in 2024-04-25T00:27:54.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:54 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T00:27:55.688 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:55 smithi121 ceph-mon[32231]: from='client.? 172.21.15.121:0/3480202597' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:27:55.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:55 smithi032 ceph-mon[29357]: from='client.? 172.21.15.121:0/3480202597' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:27:55.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:55 smithi130 ceph-mon[32525]: from='client.? 172.21.15.121:0/3480202597' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:27:56.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:56 smithi032 ceph-mon[29357]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:27:56.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:56 smithi121 ceph-mon[32231]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:27:56.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:56 smithi130 ceph-mon[32525]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:27:58.560 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:27:58 smithi121 ceph-mon[32231]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:27:58.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:27:58 smithi032 ceph-mon[29357]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:27:58.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:27:58 smithi130 ceph-mon[32525]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:00.491 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:00 smithi121 ceph-mon[32231]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:00.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:00 smithi032 ceph-mon[29357]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:00.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:00 smithi130 ceph-mon[32525]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:02.504 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:02 smithi121 ceph-mon[32231]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:02.504 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:02 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-25T00:28:02.504 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:02 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:02.504 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:02 smithi121 ceph-mon[32231]: Deploying daemon osd.7 on smithi121 2024-04-25T00:28:02.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:02 smithi032 ceph-mon[29357]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:02.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:02 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-25T00:28:02.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:02 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:02.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:02 smithi032 ceph-mon[29357]: Deploying daemon osd.7 on smithi121 2024-04-25T00:28:02.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:02 smithi130 ceph-mon[32525]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:02.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:02 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-25T00:28:02.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:02 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:02.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:02 smithi130 ceph-mon[32525]: Deploying daemon osd.7 on smithi121 2024-04-25T00:28:04.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:04 smithi032 ceph-mon[29357]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:04.828 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:04 smithi121 ceph-mon[32231]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:04.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:04 smithi130 ceph-mon[32525]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:06.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:06 smithi121 ceph-mon[32231]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:06.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:06 smithi032 ceph-mon[29357]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:06.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:06 smithi130 ceph-mon[32525]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:08.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:08 smithi032 ceph-mon[29357]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:08.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:08 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:08.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:08 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:08.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:08 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:28:08.525 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:08 smithi121 ceph-mon[32231]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:08.526 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:08 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:08.526 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:08 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:08.526 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:08 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:28:08.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:08 smithi130 ceph-mon[32525]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:08.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:08 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:08.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:08 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:08.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:08 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:28:10.260 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:10 smithi121 ceph-mon[32231]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:10.322 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:28:10.322+0000 7f459098e700 1 -- 172.21.15.121:0/4059791478 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7f458c075e50 con 0x7f45740a2c20 2024-04-25T00:28:10.323 INFO:teuthology.orchestra.run.smithi121.stdout:Created osd(s) 7 on host 'smithi121' 2024-04-25T00:28:10.325 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:28:10.324+0000 7f4591990700 1 -- 172.21.15.121:0/4059791478 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f45740a2c20 msgr2=0x7f45740a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:28:10.325 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:28:10.324+0000 7f4591990700 1 --2- 172.21.15.121:0/4059791478 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f45740a2c20 0x7f45740a50e0 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f457c005950 tx=0x7f457c0058e0 comp rx=0 tx=0).stop 2024-04-25T00:28:10.325 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:28:10.324+0000 7f4591990700 1 -- 172.21.15.121:0/4059791478 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f458c102280 msgr2=0x7f458c198410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:28:10.325 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:28:10.324+0000 7f4591990700 1 --2- 172.21.15.121:0/4059791478 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f458c102280 0x7f458c198410 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f458000d980 tx=0x7f458000dc90 comp rx=0 tx=0).stop 2024-04-25T00:28:10.325 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:28:10.324+0000 7f4591990700 1 -- 172.21.15.121:0/4059791478 shutdown_connections 2024-04-25T00:28:10.325 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:28:10.324+0000 7f4591990700 1 --2- 172.21.15.121:0/4059791478 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f45740a2c20 0x7f45740a50e0 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:28:10.325 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:28:10.324+0000 7f4591990700 1 --2- 172.21.15.121:0/4059791478 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f458c103e20 0x7f458c19f9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:28:10.325 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:28:10.324+0000 7f4591990700 1 --2- 172.21.15.121:0/4059791478 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f458c102280 0x7f458c198410 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:28:10.325 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:28:10.324+0000 7f4591990700 1 --2- 172.21.15.121:0/4059791478 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f458c103480 0x7f458c198950 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:28:10.325 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:28:10.324+0000 7f4591990700 1 -- 172.21.15.121:0/4059791478 >> 172.21.15.121:0/4059791478 conn(0x7f458c0fd820 msgr2=0x7f458c1067a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:28:10.325 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:28:10.324+0000 7f4591990700 1 -- 172.21.15.121:0/4059791478 shutdown_connections 2024-04-25T00:28:10.325 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:28:10.324+0000 7f4591990700 1 -- 172.21.15.121:0/4059791478 wait complete. 2024-04-25T00:28:10.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:10 smithi032 ceph-mon[29357]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:10.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:10 smithi130 ceph-mon[32525]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:11.251 DEBUG:teuthology.orchestra.run.smithi121:osd.7> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.7.service 2024-04-25T00:28:11.253 INFO:tasks.cephadm:Deploying osd.8 on smithi130 with /dev/vg_nvme/lv_2... 2024-04-25T00:28:11.253 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-25T00:28:11.499 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.c/config 2024-04-25T00:28:11.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:11 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:11.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:11 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:11.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:11 smithi121 ceph-mon[32231]: from='osd.7 [v2:172.21.15.121:6816/3677614004,v1:172.21.15.121:6817/3677614004]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-25T00:28:11.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:11 smithi121 ceph-mon[32231]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-25T00:28:11.621 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:11 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:11.621 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:11 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:11.621 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:11 smithi130 ceph-mon[32525]: from='osd.7 [v2:172.21.15.121:6816/3677614004,v1:172.21.15.121:6817/3677614004]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-25T00:28:11.622 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:11 smithi130 ceph-mon[32525]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-25T00:28:11.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:11 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:11.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:11 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:11.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:11 smithi032 ceph-mon[29357]: from='osd.7 [v2:172.21.15.121:6816/3677614004,v1:172.21.15.121:6817/3677614004]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-25T00:28:11.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:11 smithi032 ceph-mon[29357]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-25T00:28:12.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:12 smithi121 ceph-mon[32231]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:12.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:12 smithi121 ceph-mon[32231]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-25T00:28:12.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:12 smithi121 ceph-mon[32231]: osdmap e49: 8 total, 7 up, 8 in 2024-04-25T00:28:12.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:12 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T00:28:12.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:12 smithi121 ceph-mon[32231]: from='osd.7 [v2:172.21.15.121:6816/3677614004,v1:172.21.15.121:6817/3677614004]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-25T00:28:12.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:12 smithi121 ceph-mon[32231]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-25T00:28:12.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:12 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:12.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:12 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:12.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:12 smithi130 ceph-mon[32525]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:12.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:12 smithi130 ceph-mon[32525]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-25T00:28:12.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:12 smithi130 ceph-mon[32525]: osdmap e49: 8 total, 7 up, 8 in 2024-04-25T00:28:12.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:12 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T00:28:12.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:12 smithi130 ceph-mon[32525]: from='osd.7 [v2:172.21.15.121:6816/3677614004,v1:172.21.15.121:6817/3677614004]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-25T00:28:12.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:12 smithi130 ceph-mon[32525]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-25T00:28:12.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:12 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:12.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:12 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:12.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:12 smithi032 ceph-mon[29357]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:12.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:12 smithi032 ceph-mon[29357]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-25T00:28:12.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:12 smithi032 ceph-mon[29357]: osdmap e49: 8 total, 7 up, 8 in 2024-04-25T00:28:12.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:12 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T00:28:12.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:12 smithi032 ceph-mon[29357]: from='osd.7 [v2:172.21.15.121:6816/3677614004,v1:172.21.15.121:6817/3677614004]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-25T00:28:12.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:12 smithi032 ceph-mon[29357]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-25T00:28:12.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:12 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:12.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:12 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:13.317 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:28:12 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:28:12.980+0000 7f0c380aa700 -1 osd.7 0 waiting for initial osdmap 2024-04-25T00:28:13.317 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:28:12 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:28:12.990+0000 7f0c2fe95700 -1 osd.7 50 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T00:28:13.573 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:13 smithi130 ceph-mon[32525]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-04-25T00:28:13.573 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:13 smithi130 ceph-mon[32525]: osdmap e50: 8 total, 7 up, 8 in 2024-04-25T00:28:13.573 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:13 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T00:28:13.574 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:13 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T00:28:13.574 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:13 smithi130 ceph-mon[32525]: from='osd.7 ' entity='osd.7' 2024-04-25T00:28:13.574 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:13 smithi130 ceph-mon[32525]: osd.7 [v2:172.21.15.121:6816/3677614004,v1:172.21.15.121:6817/3677614004] boot 2024-04-25T00:28:13.574 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:13 smithi130 ceph-mon[32525]: osdmap e51: 8 total, 8 up, 8 in 2024-04-25T00:28:13.574 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:13 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T00:28:13.574 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-25T00:28:13.574 INFO:teuthology.orchestra.run.smithi130.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-25T00:28:13.574 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10+0 records in 2024-04-25T00:28:13.574 INFO:teuthology.orchestra.run.smithi130.stderr:10+0 records out 2024-04-25T00:28:13.574 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0112062 s, 936 MB/s 2024-04-25T00:28:13.574 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping successful for: 2024-04-25T00:28:13.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:13 smithi121 ceph-mon[32231]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-04-25T00:28:13.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:13 smithi121 ceph-mon[32231]: osdmap e50: 8 total, 7 up, 8 in 2024-04-25T00:28:13.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:13 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T00:28:13.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:13 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T00:28:13.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:13 smithi121 ceph-mon[32231]: from='osd.7 ' entity='osd.7' 2024-04-25T00:28:13.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:13 smithi121 ceph-mon[32231]: osd.7 [v2:172.21.15.121:6816/3677614004,v1:172.21.15.121:6817/3677614004] boot 2024-04-25T00:28:13.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:13 smithi121 ceph-mon[32231]: osdmap e51: 8 total, 8 up, 8 in 2024-04-25T00:28:13.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:13 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T00:28:13.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:13 smithi032 ceph-mon[29357]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-04-25T00:28:13.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:13 smithi032 ceph-mon[29357]: osdmap e50: 8 total, 7 up, 8 in 2024-04-25T00:28:13.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:13 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T00:28:13.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:13 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T00:28:13.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:13 smithi032 ceph-mon[29357]: from='osd.7 ' entity='osd.7' 2024-04-25T00:28:13.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:13 smithi032 ceph-mon[29357]: osd.7 [v2:172.21.15.121:6816/3677614004,v1:172.21.15.121:6817/3677614004] boot 2024-04-25T00:28:13.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:13 smithi032 ceph-mon[29357]: osdmap e51: 8 total, 8 up, 8 in 2024-04-25T00:28:13.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:13 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T00:28:14.269 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi130:vg_nvme/lv_2 2024-04-25T00:28:14.513 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.c/config 2024-04-25T00:28:14.580 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:14 smithi121 ceph-mon[32231]: purged_snaps scrub starts 2024-04-25T00:28:14.580 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:14 smithi121 ceph-mon[32231]: purged_snaps scrub ok 2024-04-25T00:28:14.580 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:14 smithi121 ceph-mon[32231]: pgmap v175: 1 pgs: 1 unknown; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:14.580 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:14 smithi121 ceph-mon[32231]: osdmap e52: 8 total, 8 up, 8 in 2024-04-25T00:28:14.635 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:14 smithi130 ceph-mon[32525]: purged_snaps scrub starts 2024-04-25T00:28:14.635 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:14 smithi130 ceph-mon[32525]: purged_snaps scrub ok 2024-04-25T00:28:14.635 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:14 smithi130 ceph-mon[32525]: pgmap v175: 1 pgs: 1 unknown; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:14.636 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:14 smithi130 ceph-mon[32525]: osdmap e52: 8 total, 8 up, 8 in 2024-04-25T00:28:14.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:14 smithi032 ceph-mon[29357]: purged_snaps scrub starts 2024-04-25T00:28:14.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:14 smithi032 ceph-mon[29357]: purged_snaps scrub ok 2024-04-25T00:28:14.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:14 smithi032 ceph-mon[29357]: pgmap v175: 1 pgs: 1 unknown; 577 KiB data, 206 MiB used, 626 GiB / 626 GiB avail 2024-04-25T00:28:14.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:14 smithi032 ceph-mon[29357]: osdmap e52: 8 total, 8 up, 8 in 2024-04-25T00:28:16.034 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:15 smithi130 ceph-mon[32525]: pgmap v178: 1 pgs: 1 unknown; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:16.034 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:15 smithi130 ceph-mon[32525]: Detected new or changed devices on smithi121 2024-04-25T00:28:16.034 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:15 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:16.034 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:15 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:16.034 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:15 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:28:16.034 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:15 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:28:16.034 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:15 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:28:16.035 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:15 smithi130 ceph-mon[32525]: Adjusting osd_memory_target on smithi121 to 5696M 2024-04-25T00:28:16.035 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:15 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:16.035 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:15 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:16.035 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:15 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:28:16.035 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:15 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:16.035 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:15 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:28:16.035 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:15 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:16.035 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:15 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:28:16.035 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:15 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:16.035 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:15 smithi130 ceph-mon[32525]: osdmap e53: 8 total, 8 up, 8 in 2024-04-25T00:28:16.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:15 smithi032 ceph-mon[29357]: pgmap v178: 1 pgs: 1 unknown; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:16.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:15 smithi032 ceph-mon[29357]: Detected new or changed devices on smithi121 2024-04-25T00:28:16.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:15 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:16.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:15 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:16.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:15 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:28:16.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:15 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:28:16.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:15 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:28:16.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:15 smithi032 ceph-mon[29357]: Adjusting osd_memory_target on smithi121 to 5696M 2024-04-25T00:28:16.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:15 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:16.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:15 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:16.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:15 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:28:16.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:15 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:16.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:15 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:28:16.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:15 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:16.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:15 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:28:16.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:15 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:16.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:15 smithi032 ceph-mon[29357]: osdmap e53: 8 total, 8 up, 8 in 2024-04-25T00:28:16.315 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.312+0000 7fab8e623700 1 -- 172.21.15.130:0/2624066316 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fab88102290 msgr2=0x7fab881026b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:28:16.315 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.312+0000 7fab8e623700 1 --2- 172.21.15.130:0/2624066316 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fab88102290 0x7fab881026b0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7fab78009990 tx=0x7fab78009ca0 comp rx=0 tx=0).stop 2024-04-25T00:28:16.315 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.312+0000 7fab8e623700 1 -- 172.21.15.130:0/2624066316 shutdown_connections 2024-04-25T00:28:16.315 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.312+0000 7fab8e623700 1 --2- 172.21.15.130:0/2624066316 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fab88103e30 0x7fab8810a6e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:28:16.315 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.312+0000 7fab8e623700 1 --2- 172.21.15.130:0/2624066316 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fab88102290 0x7fab881026b0 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:28:16.315 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.312+0000 7fab8e623700 1 --2- 172.21.15.130:0/2624066316 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fab88103490 0x7fab881038f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:28:16.315 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.312+0000 7fab8e623700 1 -- 172.21.15.130:0/2624066316 >> 172.21.15.130:0/2624066316 conn(0x7fab880fd830 msgr2=0x7fab880ffc70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:28:16.315 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.312+0000 7fab8e623700 1 -- 172.21.15.130:0/2624066316 shutdown_connections 2024-04-25T00:28:16.315 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.312+0000 7fab8e623700 1 -- 172.21.15.130:0/2624066316 wait complete. 2024-04-25T00:28:16.315 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.312+0000 7fab8e623700 1 Processor -- start 2024-04-25T00:28:16.315 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.313+0000 7fab8e623700 1 -- start start 2024-04-25T00:28:16.316 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.313+0000 7fab8e623700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fab88102290 0x7fab88198450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:28:16.316 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.313+0000 7fab8e623700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fab88103490 0x7fab88198990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:28:16.316 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.313+0000 7fab8e623700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fab88103e30 0x7fab8819fa10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:28:16.316 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.313+0000 7fab8e623700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fab8810de70 con 0x7fab88103e30 2024-04-25T00:28:16.316 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.313+0000 7fab8e623700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fab8810dcf0 con 0x7fab88103490 2024-04-25T00:28:16.316 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.313+0000 7fab8e623700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fab8810dff0 con 0x7fab88102290 2024-04-25T00:28:16.316 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.313+0000 7fab87fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fab88102290 0x7fab88198450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:28:16.316 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.313+0000 7fab87fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fab88102290 0x7fab88198450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.130:54468/0 (socket says 172.21.15.130:54468) 2024-04-25T00:28:16.316 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.313+0000 7fab87fff700 1 -- 172.21.15.130:0/1361452851 learned_addr learned my addr 172.21.15.130:0/1361452851 (peer_addr_for_me v2:172.21.15.130:0/0) 2024-04-25T00:28:16.316 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.313+0000 7fab877fe700 1 --2- 172.21.15.130:0/1361452851 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fab88103490 0x7fab88198990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:28:16.316 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.313+0000 7fab8cbc0700 1 --2- 172.21.15.130:0/1361452851 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fab88103e30 0x7fab8819fa10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:28:16.316 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.313+0000 7fab87fff700 1 -- 172.21.15.130:0/1361452851 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fab88103490 msgr2=0x7fab88198990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:28:16.316 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.313+0000 7fab87fff700 1 --2- 172.21.15.130:0/1361452851 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fab88103490 0x7fab88198990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:28:16.317 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.313+0000 7fab87fff700 1 -- 172.21.15.130:0/1361452851 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fab88103e30 msgr2=0x7fab8819fa10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:28:16.317 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.313+0000 7fab87fff700 1 --2- 172.21.15.130:0/1361452851 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fab88103e30 0x7fab8819fa10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:28:16.317 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.313+0000 7fab87fff700 1 -- 172.21.15.130:0/1361452851 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fab881a00b0 con 0x7fab88102290 2024-04-25T00:28:16.317 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.313+0000 7fab87fff700 1 --2- 172.21.15.130:0/1361452851 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fab88102290 0x7fab88198450 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7fab78005b40 tx=0x7fab7800f740 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:28:16.317 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.314+0000 7fab857fa700 1 -- 172.21.15.130:0/1361452851 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fab78026070 con 0x7fab88102290 2024-04-25T00:28:16.317 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.314+0000 7fab857fa700 1 -- 172.21.15.130:0/1361452851 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fab780218f0 con 0x7fab88102290 2024-04-25T00:28:16.317 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.314+0000 7fab8e623700 1 -- 172.21.15.130:0/1361452851 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fab881a0340 con 0x7fab88102290 2024-04-25T00:28:16.317 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.314+0000 7fab857fa700 1 -- 172.21.15.130:0/1361452851 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fab78017750 con 0x7fab88102290 2024-04-25T00:28:16.317 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.314+0000 7fab8e623700 1 -- 172.21.15.130:0/1361452851 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fab881a0790 con 0x7fab88102290 2024-04-25T00:28:16.318 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.315+0000 7fab8e623700 1 -- 172.21.15.130:0/1361452851 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fab88068f20 con 0x7fab88102290 2024-04-25T00:28:16.318 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.316+0000 7fab857fa700 1 -- 172.21.15.130:0/1361452851 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7fab78029070 con 0x7fab88102290 2024-04-25T00:28:16.318 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.316+0000 7fab857fa700 1 --2- 172.21.15.130:0/1361452851 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fab700a2c20 0x7fab700a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:28:16.318 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.316+0000 7fab857fa700 1 -- 172.21.15.130:0/1361452851 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(53..53 src has 1..53) ==== 5426+0+0 (secure 0 0 0) 0x7fab780cf8d0 con 0x7fab88102290 2024-04-25T00:28:16.318 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.317+0000 7fab877fe700 1 --2- 172.21.15.130:0/1361452851 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fab700a2c20 0x7fab700a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:28:16.318 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.317+0000 7fab877fe700 1 --2- 172.21.15.130:0/1361452851 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fab700a2c20 0x7fab700a50e0 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7fab7c009bf0 tx=0x7fab7c009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:28:16.320 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.320+0000 7fab857fa700 1 -- 172.21.15.130:0/1361452851 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fab78013070 con 0x7fab88102290 2024-04-25T00:28:16.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:15 smithi121 ceph-mon[32231]: pgmap v178: 1 pgs: 1 unknown; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:16.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:15 smithi121 ceph-mon[32231]: Detected new or changed devices on smithi121 2024-04-25T00:28:16.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:15 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:16.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:15 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:16.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:15 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:28:16.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:15 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:28:16.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:15 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:28:16.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:15 smithi121 ceph-mon[32231]: Adjusting osd_memory_target on smithi121 to 5696M 2024-04-25T00:28:16.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:15 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:16.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:15 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:16.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:15 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:28:16.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:15 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:16.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:15 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:28:16.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:15 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:16.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:15 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:28:16.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:15 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:16.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:15 smithi121 ceph-mon[32231]: osdmap e53: 8 total, 8 up, 8 in 2024-04-25T00:28:16.523 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:16.523+0000 7fab8e623700 1 -- 172.21.15.130:0/1361452851 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_2", "target": ["mon-mgr", ""]}) -- 0x7fab88062670 con 0x7fab700a2c20 2024-04-25T00:28:17.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:17 smithi130 ceph-mon[32525]: osdmap e54: 8 total, 8 up, 8 in 2024-04-25T00:28:17.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:17 smithi130 ceph-mon[32525]: from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:28:17.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:17 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:28:17.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:17 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:28:17.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:17 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:17.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:17 smithi121 ceph-mon[32231]: osdmap e54: 8 total, 8 up, 8 in 2024-04-25T00:28:17.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:17 smithi121 ceph-mon[32231]: from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:28:17.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:17 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:28:17.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:17 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:28:17.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:17 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:17.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:17 smithi032 ceph-mon[29357]: osdmap e54: 8 total, 8 up, 8 in 2024-04-25T00:28:17.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:17 smithi032 ceph-mon[29357]: from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:28:17.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:17 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:28:17.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:17 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:28:17.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:17 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:18.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:18 smithi130 ceph-mon[32525]: pgmap v181: 1 pgs: 1 unknown; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:18.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:18 smithi130 ceph-mon[32525]: osdmap e55: 8 total, 8 up, 8 in 2024-04-25T00:28:18.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:18 smithi121 ceph-mon[32231]: pgmap v181: 1 pgs: 1 unknown; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:18.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:18 smithi121 ceph-mon[32231]: osdmap e55: 8 total, 8 up, 8 in 2024-04-25T00:28:18.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:18 smithi032 ceph-mon[29357]: pgmap v181: 1 pgs: 1 unknown; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:18.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:18 smithi032 ceph-mon[29357]: osdmap e55: 8 total, 8 up, 8 in 2024-04-25T00:28:19.639 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:19 smithi130 ceph-mon[32525]: from='client.? 172.21.15.130:0/1786887063' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "700b2360-e592-406b-a97e-cf9387feae70"}]: dispatch 2024-04-25T00:28:19.639 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:19 smithi130 ceph-mon[32525]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "700b2360-e592-406b-a97e-cf9387feae70"}]: dispatch 2024-04-25T00:28:19.639 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:19 smithi130 ceph-mon[32525]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "700b2360-e592-406b-a97e-cf9387feae70"}]': finished 2024-04-25T00:28:19.639 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:19 smithi130 ceph-mon[32525]: osdmap e56: 9 total, 8 up, 9 in 2024-04-25T00:28:19.639 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:19 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-25T00:28:19.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:19 smithi032 ceph-mon[29357]: from='client.? 172.21.15.130:0/1786887063' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "700b2360-e592-406b-a97e-cf9387feae70"}]: dispatch 2024-04-25T00:28:19.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:19 smithi032 ceph-mon[29357]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "700b2360-e592-406b-a97e-cf9387feae70"}]: dispatch 2024-04-25T00:28:19.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:19 smithi032 ceph-mon[29357]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "700b2360-e592-406b-a97e-cf9387feae70"}]': finished 2024-04-25T00:28:19.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:19 smithi032 ceph-mon[29357]: osdmap e56: 9 total, 8 up, 9 in 2024-04-25T00:28:19.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:19 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-25T00:28:19.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:19 smithi121 ceph-mon[32231]: from='client.? 172.21.15.130:0/1786887063' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "700b2360-e592-406b-a97e-cf9387feae70"}]: dispatch 2024-04-25T00:28:19.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:19 smithi121 ceph-mon[32231]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "700b2360-e592-406b-a97e-cf9387feae70"}]: dispatch 2024-04-25T00:28:19.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:19 smithi121 ceph-mon[32231]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "700b2360-e592-406b-a97e-cf9387feae70"}]': finished 2024-04-25T00:28:19.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:19 smithi121 ceph-mon[32231]: osdmap e56: 9 total, 8 up, 9 in 2024-04-25T00:28:19.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:19 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-25T00:28:20.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:20 smithi130 ceph-mon[32525]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:20 smithi130 ceph-mon[32525]: from='client.? 172.21.15.130:0/768068922' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:28:20.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:20 smithi032 ceph-mon[29357]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:20.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:20 smithi032 ceph-mon[29357]: from='client.? 172.21.15.130:0/768068922' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:28:20.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:20 smithi121 ceph-mon[32231]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:20.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:20 smithi121 ceph-mon[32231]: from='client.? 172.21.15.130:0/768068922' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:28:22.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:22 smithi130 ceph-mon[32525]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:22.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:22 smithi032 ceph-mon[29357]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:22.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:22 smithi121 ceph-mon[32231]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:24.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:24 smithi130 ceph-mon[32525]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:24.640 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:24 smithi121 ceph-mon[32231]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:24.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:24 smithi032 ceph-mon[29357]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:26.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:26 smithi130 ceph-mon[32525]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:26.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:26 smithi032 ceph-mon[29357]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:26.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:26 smithi121 ceph-mon[32231]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:27.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:27 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.8"}]: dispatch 2024-04-25T00:28:27.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:27 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:27.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:27 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.8"}]: dispatch 2024-04-25T00:28:27.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:27 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:27.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:27 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.8"}]: dispatch 2024-04-25T00:28:27.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:27 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:28.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:28 smithi130 ceph-mon[32525]: Deploying daemon osd.8 on smithi130 2024-04-25T00:28:28.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:28 smithi130 ceph-mon[32525]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:28.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:28 smithi032 ceph-mon[29357]: Deploying daemon osd.8 on smithi130 2024-04-25T00:28:28.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:28 smithi032 ceph-mon[29357]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:28.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:28 smithi121 ceph-mon[32231]: Deploying daemon osd.8 on smithi130 2024-04-25T00:28:28.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:28 smithi121 ceph-mon[32231]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:30.614 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:30 smithi130 ceph-mon[32525]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:30.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:30 smithi032 ceph-mon[29357]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:30.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:30 smithi121 ceph-mon[32231]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:32.495 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:32 smithi130 ceph-mon[32525]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:32.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:32 smithi032 ceph-mon[29357]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:32.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:32 smithi121 ceph-mon[32231]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:33.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:33 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:33.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:33 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:33.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:33 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:28:33.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:33 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:33.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:33 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:33.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:33 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:28:33.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:33 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:33.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:33 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:33.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:33 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:28:34.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:34 smithi130 ceph-mon[32525]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:34.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:34 smithi032 ceph-mon[29357]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:34.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:34 smithi121 ceph-mon[32231]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:35.847 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:35.846+0000 7fab857fa700 1 -- 172.21.15.130:0/1361452851 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7fab88062670 con 0x7fab700a2c20 2024-04-25T00:28:35.847 INFO:teuthology.orchestra.run.smithi130.stdout:Created osd(s) 8 on host 'smithi130' 2024-04-25T00:28:35.849 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:35.848+0000 7fab8e623700 1 -- 172.21.15.130:0/1361452851 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fab700a2c20 msgr2=0x7fab700a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:28:35.849 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:35.848+0000 7fab8e623700 1 --2- 172.21.15.130:0/1361452851 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fab700a2c20 0x7fab700a50e0 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7fab7c009bf0 tx=0x7fab7c009260 comp rx=0 tx=0).stop 2024-04-25T00:28:35.849 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:35.848+0000 7fab8e623700 1 -- 172.21.15.130:0/1361452851 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fab88102290 msgr2=0x7fab88198450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:28:35.849 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:35.848+0000 7fab8e623700 1 --2- 172.21.15.130:0/1361452851 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fab88102290 0x7fab88198450 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7fab78005b40 tx=0x7fab7800f740 comp rx=0 tx=0).stop 2024-04-25T00:28:35.849 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:35.848+0000 7fab8e623700 1 -- 172.21.15.130:0/1361452851 shutdown_connections 2024-04-25T00:28:35.849 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:35.848+0000 7fab8e623700 1 --2- 172.21.15.130:0/1361452851 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fab700a2c20 0x7fab700a50e0 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:28:35.849 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:35.848+0000 7fab8e623700 1 --2- 172.21.15.130:0/1361452851 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fab88103e30 0x7fab8819fa10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:28:35.849 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:35.848+0000 7fab8e623700 1 --2- 172.21.15.130:0/1361452851 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fab88102290 0x7fab88198450 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:28:35.849 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:35.848+0000 7fab8e623700 1 --2- 172.21.15.130:0/1361452851 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fab88103490 0x7fab88198990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:28:35.850 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:35.848+0000 7fab8e623700 1 -- 172.21.15.130:0/1361452851 >> 172.21.15.130:0/1361452851 conn(0x7fab880fd830 msgr2=0x7fab881067e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:28:35.850 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:35.848+0000 7fab8e623700 1 -- 172.21.15.130:0/1361452851 shutdown_connections 2024-04-25T00:28:35.850 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:28:35.848+0000 7fab8e623700 1 -- 172.21.15.130:0/1361452851 wait complete. 2024-04-25T00:28:36.507 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:36 smithi130 ceph-mon[32525]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:36.507 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:36 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:36.507 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:36 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:36.507 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:36 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:36.507 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:36 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:36.508 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:36 smithi130 ceph-mon[32525]: from='osd.8 [v2:172.21.15.130:6816/285467211,v1:172.21.15.130:6817/285467211]' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2024-04-25T00:28:36.508 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:36 smithi130 ceph-mon[32525]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2024-04-25T00:28:36.537 DEBUG:teuthology.orchestra.run.smithi130:osd.8> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.8.service 2024-04-25T00:28:36.540 INFO:tasks.cephadm:Deploying osd.9 on smithi032 with /dev/vg_nvme/lv_1... 2024-04-25T00:28:36.540 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-25T00:28:36.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:36 smithi032 ceph-mon[29357]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:36.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:36 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:36.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:36 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:36.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:36 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:36.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:36 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:36.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:36 smithi032 ceph-mon[29357]: from='osd.8 [v2:172.21.15.130:6816/285467211,v1:172.21.15.130:6817/285467211]' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2024-04-25T00:28:36.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:36 smithi032 ceph-mon[29357]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2024-04-25T00:28:36.800 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:28:36.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:36 smithi121 ceph-mon[32231]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:36.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:36 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:36.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:36 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:36.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:36 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:36.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:36 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:36.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:36 smithi121 ceph-mon[32231]: from='osd.8 [v2:172.21.15.130:6816/285467211,v1:172.21.15.130:6817/285467211]' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2024-04-25T00:28:36.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:36 smithi121 ceph-mon[32231]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2024-04-25T00:28:38.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:37 smithi130 ceph-mon[32525]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:38.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:37 smithi130 ceph-mon[32525]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]': finished 2024-04-25T00:28:38.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:37 smithi130 ceph-mon[32525]: osdmap e57: 9 total, 8 up, 9 in 2024-04-25T00:28:38.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:37 smithi130 ceph-mon[32525]: from='osd.8 [v2:172.21.15.130:6816/285467211,v1:172.21.15.130:6817/285467211]' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-25T00:28:38.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:37 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-25T00:28:38.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:37 smithi130 ceph-mon[32525]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-25T00:28:38.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:37 smithi032 ceph-mon[29357]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:38.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:37 smithi032 ceph-mon[29357]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]': finished 2024-04-25T00:28:38.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:37 smithi032 ceph-mon[29357]: osdmap e57: 9 total, 8 up, 9 in 2024-04-25T00:28:38.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:37 smithi032 ceph-mon[29357]: from='osd.8 [v2:172.21.15.130:6816/285467211,v1:172.21.15.130:6817/285467211]' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-25T00:28:38.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:37 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-25T00:28:38.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:37 smithi032 ceph-mon[29357]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-25T00:28:38.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:37 smithi121 ceph-mon[32231]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:38.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:37 smithi121 ceph-mon[32231]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]': finished 2024-04-25T00:28:38.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:37 smithi121 ceph-mon[32231]: osdmap e57: 9 total, 8 up, 9 in 2024-04-25T00:28:38.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:37 smithi121 ceph-mon[32231]: from='osd.8 [v2:172.21.15.130:6816/285467211,v1:172.21.15.130:6817/285467211]' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-25T00:28:38.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-25T00:28:38.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:37 smithi121 ceph-mon[32231]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-25T00:28:38.906 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:28:38 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:28:38.643+0000 7f487bd79700 -1 osd.8 0 waiting for initial osdmap 2024-04-25T00:28:38.906 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:28:38 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:28:38.653+0000 7f4873b64700 -1 osd.8 58 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T00:28:39.232 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:38 smithi032 ceph-mon[29357]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-04-25T00:28:39.232 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:38 smithi032 ceph-mon[29357]: osdmap e58: 9 total, 8 up, 9 in 2024-04-25T00:28:39.232 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:38 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-25T00:28:39.232 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:38 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-25T00:28:39.232 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:38 smithi032 ceph-mon[29357]: from='osd.8 ' entity='osd.8' 2024-04-25T00:28:39.232 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:38 smithi032 ceph-mon[29357]: osd.8 [v2:172.21.15.130:6816/285467211,v1:172.21.15.130:6817/285467211] boot 2024-04-25T00:28:39.232 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:38 smithi032 ceph-mon[29357]: osdmap e59: 9 total, 9 up, 9 in 2024-04-25T00:28:39.232 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:38 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-25T00:28:39.233 INFO:teuthology.orchestra.run.smithi032.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-25T00:28:39.233 INFO:teuthology.orchestra.run.smithi032.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-25T00:28:39.233 INFO:teuthology.orchestra.run.smithi032.stderr: stderr: 10+0 records in 2024-04-25T00:28:39.233 INFO:teuthology.orchestra.run.smithi032.stderr:10+0 records out 2024-04-25T00:28:39.233 INFO:teuthology.orchestra.run.smithi032.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00840722 s, 1.2 GB/s 2024-04-25T00:28:39.233 INFO:teuthology.orchestra.run.smithi032.stderr:--> Zapping successful for: 2024-04-25T00:28:39.264 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:38 smithi130 ceph-mon[32525]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-04-25T00:28:39.264 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:38 smithi130 ceph-mon[32525]: osdmap e58: 9 total, 8 up, 9 in 2024-04-25T00:28:39.264 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:38 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-25T00:28:39.264 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:38 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-25T00:28:39.264 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:38 smithi130 ceph-mon[32525]: from='osd.8 ' entity='osd.8' 2024-04-25T00:28:39.264 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:38 smithi130 ceph-mon[32525]: osd.8 [v2:172.21.15.130:6816/285467211,v1:172.21.15.130:6817/285467211] boot 2024-04-25T00:28:39.264 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:38 smithi130 ceph-mon[32525]: osdmap e59: 9 total, 9 up, 9 in 2024-04-25T00:28:39.264 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:38 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-25T00:28:39.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:38 smithi121 ceph-mon[32231]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-04-25T00:28:39.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:38 smithi121 ceph-mon[32231]: osdmap e58: 9 total, 8 up, 9 in 2024-04-25T00:28:39.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:38 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-25T00:28:39.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:38 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-25T00:28:39.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:38 smithi121 ceph-mon[32231]: from='osd.8 ' entity='osd.8' 2024-04-25T00:28:39.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:38 smithi121 ceph-mon[32231]: osd.8 [v2:172.21.15.130:6816/285467211,v1:172.21.15.130:6817/285467211] boot 2024-04-25T00:28:39.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:38 smithi121 ceph-mon[32231]: osdmap e59: 9 total, 9 up, 9 in 2024-04-25T00:28:39.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:38 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-25T00:28:39.922 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi032:vg_nvme/lv_1 2024-04-25T00:28:40.195 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:28:40.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:39 smithi032 ceph-mon[29357]: purged_snaps scrub starts 2024-04-25T00:28:40.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:39 smithi032 ceph-mon[29357]: purged_snaps scrub ok 2024-04-25T00:28:40.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:39 smithi032 ceph-mon[29357]: pgmap v196: 1 pgs: 1 peering; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:40.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:39 smithi032 ceph-mon[29357]: Detected new or changed devices on smithi130 2024-04-25T00:28:40.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:39 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:40.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:39 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:40.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:39 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:28:40.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:39 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:28:40.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:39 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:28:40.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:39 smithi032 ceph-mon[29357]: Adjusting osd_memory_target on smithi130 to 5696M 2024-04-25T00:28:40.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:39 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:40.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:39 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:40.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:39 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:28:40.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:39 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:40.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:39 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:28:40.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:39 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:40.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:39 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:28:40.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:39 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:40.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:39 smithi032 ceph-mon[29357]: osdmap e60: 9 total, 9 up, 9 in 2024-04-25T00:28:40.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:39 smithi121 ceph-mon[32231]: purged_snaps scrub starts 2024-04-25T00:28:40.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:39 smithi121 ceph-mon[32231]: purged_snaps scrub ok 2024-04-25T00:28:40.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:39 smithi121 ceph-mon[32231]: pgmap v196: 1 pgs: 1 peering; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:40.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:39 smithi121 ceph-mon[32231]: Detected new or changed devices on smithi130 2024-04-25T00:28:40.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:39 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:40.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:39 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:40.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:39 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:28:40.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:39 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:28:40.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:39 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:28:40.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:39 smithi121 ceph-mon[32231]: Adjusting osd_memory_target on smithi130 to 5696M 2024-04-25T00:28:40.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:39 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:40.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:39 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:40.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:39 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:28:40.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:39 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:40.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:39 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:28:40.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:39 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:40.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:39 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:28:40.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:39 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:40.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:39 smithi121 ceph-mon[32231]: osdmap e60: 9 total, 9 up, 9 in 2024-04-25T00:28:40.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:39 smithi130 ceph-mon[32525]: purged_snaps scrub starts 2024-04-25T00:28:40.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:39 smithi130 ceph-mon[32525]: purged_snaps scrub ok 2024-04-25T00:28:40.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:39 smithi130 ceph-mon[32525]: pgmap v196: 1 pgs: 1 peering; 577 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-04-25T00:28:40.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:39 smithi130 ceph-mon[32525]: Detected new or changed devices on smithi130 2024-04-25T00:28:40.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:39 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:40.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:39 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:40.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:39 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:28:40.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:39 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:28:40.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:39 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:28:40.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:39 smithi130 ceph-mon[32525]: Adjusting osd_memory_target on smithi130 to 5696M 2024-04-25T00:28:40.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:39 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:40.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:39 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:40.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:39 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:28:40.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:39 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:40.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:39 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:28:40.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:39 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:40.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:39 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:28:40.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:39 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:28:40.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:39 smithi130 ceph-mon[32525]: osdmap e60: 9 total, 9 up, 9 in 2024-04-25T00:28:41.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.893+0000 7fb5f668b700 1 -- 172.21.15.32:0/2700731157 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb5f0102d30 msgr2=0x7fb5f01095e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:28:41.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.893+0000 7fb5f668b700 1 --2- 172.21.15.32:0/2700731157 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb5f0102d30 0x7fb5f01095e0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7fb5e800b0d0 tx=0x7fb5e800b3e0 comp rx=0 tx=0).stop 2024-04-25T00:28:41.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.893+0000 7fb5f668b700 1 -- 172.21.15.32:0/2700731157 shutdown_connections 2024-04-25T00:28:41.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.893+0000 7fb5f668b700 1 --2- 172.21.15.32:0/2700731157 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb5f0102d30 0x7fb5f01095e0 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:28:41.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.893+0000 7fb5f668b700 1 --2- 172.21.15.32:0/2700731157 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb5f0101190 0x7fb5f01015b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:28:41.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.893+0000 7fb5f668b700 1 --2- 172.21.15.32:0/2700731157 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb5f0102390 0x7fb5f01027f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:28:41.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.893+0000 7fb5f668b700 1 -- 172.21.15.32:0/2700731157 >> 172.21.15.32:0/2700731157 conn(0x7fb5f00fc730 msgr2=0x7fb5f00feb70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:28:41.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.893+0000 7fb5f668b700 1 -- 172.21.15.32:0/2700731157 shutdown_connections 2024-04-25T00:28:41.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.893+0000 7fb5f668b700 1 -- 172.21.15.32:0/2700731157 wait complete. 2024-04-25T00:28:41.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.894+0000 7fb5f668b700 1 Processor -- start 2024-04-25T00:28:41.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.894+0000 7fb5f668b700 1 -- start start 2024-04-25T00:28:41.897 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.894+0000 7fb5f668b700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb5f0101190 0x7fb5f0198630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:28:41.897 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.894+0000 7fb5f668b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb5f0102390 0x7fb5f0198b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:28:41.897 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.894+0000 7fb5f668b700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb5f0102d30 0x7fb5f019fba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:28:41.897 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.894+0000 7fb5f668b700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fb5f010ce70 con 0x7fb5f0101190 2024-04-25T00:28:41.897 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.894+0000 7fb5f668b700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fb5f010ccf0 con 0x7fb5f0102d30 2024-04-25T00:28:41.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.894+0000 7fb5f668b700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fb5f010cff0 con 0x7fb5f0102390 2024-04-25T00:28:41.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.895+0000 7fb5effff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb5f0101190 0x7fb5f0198630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:28:41.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.895+0000 7fb5effff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb5f0101190 0x7fb5f0198630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:37646/0 (socket says 172.21.15.32:37646) 2024-04-25T00:28:41.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.895+0000 7fb5effff700 1 -- 172.21.15.32:0/3250255016 learned_addr learned my addr 172.21.15.32:0/3250255016 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:28:41.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.895+0000 7fb5ef7fe700 1 --2- 172.21.15.32:0/3250255016 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb5f0102390 0x7fb5f0198b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:28:41.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.895+0000 7fb5f4c28700 1 --2- 172.21.15.32:0/3250255016 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb5f0102d30 0x7fb5f019fba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:28:41.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.895+0000 7fb5effff700 1 -- 172.21.15.32:0/3250255016 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb5f0102390 msgr2=0x7fb5f0198b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:28:41.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.895+0000 7fb5effff700 1 --2- 172.21.15.32:0/3250255016 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb5f0102390 0x7fb5f0198b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:28:41.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.895+0000 7fb5effff700 1 -- 172.21.15.32:0/3250255016 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb5f0102d30 msgr2=0x7fb5f019fba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:28:41.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.895+0000 7fb5effff700 1 --2- 172.21.15.32:0/3250255016 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb5f0102d30 0x7fb5f019fba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:28:41.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.895+0000 7fb5effff700 1 -- 172.21.15.32:0/3250255016 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb5f01a0240 con 0x7fb5f0101190 2024-04-25T00:28:41.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.895+0000 7fb5ef7fe700 1 --2- 172.21.15.32:0/3250255016 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb5f0102390 0x7fb5f0198b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:28:41.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.895+0000 7fb5effff700 1 --2- 172.21.15.32:0/3250255016 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb5f0101190 0x7fb5f0198630 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7fb5e000b530 tx=0x7fb5e000b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:28:41.899 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.895+0000 7fb5ed7fa700 1 -- 172.21.15.32:0/3250255016 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb5e0015020 con 0x7fb5f0101190 2024-04-25T00:28:41.899 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.895+0000 7fb5f668b700 1 -- 172.21.15.32:0/3250255016 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb5f01a0530 con 0x7fb5f0101190 2024-04-25T00:28:41.899 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.895+0000 7fb5ed7fa700 1 -- 172.21.15.32:0/3250255016 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb5e0011d10 con 0x7fb5f0101190 2024-04-25T00:28:41.899 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.895+0000 7fb5ed7fa700 1 -- 172.21.15.32:0/3250255016 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb5e000f2e0 con 0x7fb5f0101190 2024-04-25T00:28:41.899 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.895+0000 7fb5f668b700 1 -- 172.21.15.32:0/3250255016 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb5f01a0b40 con 0x7fb5f0101190 2024-04-25T00:28:41.899 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.896+0000 7fb5f668b700 1 -- 172.21.15.32:0/3250255016 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb5f0068f20 con 0x7fb5f0101190 2024-04-25T00:28:41.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.897+0000 7fb5ed7fa700 1 -- 172.21.15.32:0/3250255016 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7fb5e0018020 con 0x7fb5f0101190 2024-04-25T00:28:41.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.898+0000 7fb5ed7fa700 1 --2- 172.21.15.32:0/3250255016 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb5d80a2c20 0x7fb5d80a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:28:41.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.898+0000 7fb5ef7fe700 1 --2- 172.21.15.32:0/3250255016 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb5d80a2c20 0x7fb5d80a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:28:41.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.898+0000 7fb5ed7fa700 1 -- 172.21.15.32:0/3250255016 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(62..62 src has 1..62) ==== 5829+0+0 (secure 0 0 0) 0x7fb5e00ce400 con 0x7fb5f0101190 2024-04-25T00:28:41.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.898+0000 7fb5ef7fe700 1 --2- 172.21.15.32:0/3250255016 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb5d80a2c20 0x7fb5d80a50e0 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7fb5e40099d0 tx=0x7fb5e4008040 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:28:41.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:41.902+0000 7fb5ed7fa700 1 -- 172.21.15.32:0/3250255016 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fb5e001f020 con 0x7fb5f0101190 2024-04-25T00:28:42.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:28:42.123+0000 7fb5f668b700 1 -- 172.21.15.32:0/3250255016 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_1", "target": ["mon-mgr", ""]}) -- 0x7fb5f01a0e00 con 0x7fb5d80a2c20 2024-04-25T00:28:42.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:41 smithi032 ceph-mon[29357]: pgmap v199: 1 pgs: 1 peering; 577 KiB data, 664 MiB used, 804 GiB / 805 GiB avail 2024-04-25T00:28:42.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:41 smithi032 ceph-mon[29357]: osdmap e61: 9 total, 9 up, 9 in 2024-04-25T00:28:42.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:41 smithi032 ceph-mon[29357]: osdmap e62: 9 total, 9 up, 9 in 2024-04-25T00:28:42.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:41 smithi121 ceph-mon[32231]: pgmap v199: 1 pgs: 1 peering; 577 KiB data, 664 MiB used, 804 GiB / 805 GiB avail 2024-04-25T00:28:42.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:41 smithi121 ceph-mon[32231]: osdmap e61: 9 total, 9 up, 9 in 2024-04-25T00:28:42.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:41 smithi121 ceph-mon[32231]: osdmap e62: 9 total, 9 up, 9 in 2024-04-25T00:28:42.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:41 smithi130 ceph-mon[32525]: pgmap v199: 1 pgs: 1 peering; 577 KiB data, 664 MiB used, 804 GiB / 805 GiB avail 2024-04-25T00:28:42.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:41 smithi130 ceph-mon[32525]: osdmap e61: 9 total, 9 up, 9 in 2024-04-25T00:28:42.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:41 smithi130 ceph-mon[32525]: osdmap e62: 9 total, 9 up, 9 in 2024-04-25T00:28:43.012 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:42 smithi032 ceph-mon[29357]: from='client.14439 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:28:43.012 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:28:43.012 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:28:43.012 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:43.012 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:42 smithi032 ceph-mon[29357]: osdmap e63: 9 total, 9 up, 9 in 2024-04-25T00:28:43.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:42 smithi121 ceph-mon[32231]: from='client.14439 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:28:43.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:42 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:28:43.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:42 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:28:43.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:42 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:43.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:42 smithi121 ceph-mon[32231]: osdmap e63: 9 total, 9 up, 9 in 2024-04-25T00:28:43.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:42 smithi130 ceph-mon[32525]: from='client.14439 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:28:43.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:42 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:28:43.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:42 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:28:43.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:42 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:43.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:42 smithi130 ceph-mon[32525]: osdmap e63: 9 total, 9 up, 9 in 2024-04-25T00:28:44.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:44 smithi032 ceph-mon[29357]: pgmap v203: 1 pgs: 1 peering; 577 KiB data, 664 MiB used, 804 GiB / 805 GiB avail 2024-04-25T00:28:45.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:44 smithi121 ceph-mon[32231]: pgmap v203: 1 pgs: 1 peering; 577 KiB data, 664 MiB used, 804 GiB / 805 GiB avail 2024-04-25T00:28:45.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:44 smithi130 ceph-mon[32525]: pgmap v203: 1 pgs: 1 peering; 577 KiB data, 664 MiB used, 804 GiB / 805 GiB avail 2024-04-25T00:28:45.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:45 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2539323915' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4457f5a8-6991-48c5-a382-dbca97cb206e"}]: dispatch 2024-04-25T00:28:45.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:45 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2539323915' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4457f5a8-6991-48c5-a382-dbca97cb206e"}]': finished 2024-04-25T00:28:45.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:45 smithi032 ceph-mon[29357]: osdmap e64: 10 total, 9 up, 10 in 2024-04-25T00:28:45.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-25T00:28:46.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:45 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2539323915' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4457f5a8-6991-48c5-a382-dbca97cb206e"}]: dispatch 2024-04-25T00:28:46.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:45 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2539323915' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4457f5a8-6991-48c5-a382-dbca97cb206e"}]': finished 2024-04-25T00:28:46.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:45 smithi121 ceph-mon[32231]: osdmap e64: 10 total, 9 up, 10 in 2024-04-25T00:28:46.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-25T00:28:46.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:45 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2539323915' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4457f5a8-6991-48c5-a382-dbca97cb206e"}]: dispatch 2024-04-25T00:28:46.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:45 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2539323915' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4457f5a8-6991-48c5-a382-dbca97cb206e"}]': finished 2024-04-25T00:28:46.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:45 smithi130 ceph-mon[32525]: osdmap e64: 10 total, 9 up, 10 in 2024-04-25T00:28:46.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-25T00:28:46.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:46 smithi032 ceph-mon[29357]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 270 MiB used, 804 GiB / 805 GiB avail; 118 KiB/s, 0 objects/s recovering 2024-04-25T00:28:46.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:46 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2480308241' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:28:47.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:46 smithi121 ceph-mon[32231]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 270 MiB used, 804 GiB / 805 GiB avail; 118 KiB/s, 0 objects/s recovering 2024-04-25T00:28:47.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:46 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2480308241' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:28:47.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:46 smithi130 ceph-mon[32525]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 270 MiB used, 804 GiB / 805 GiB avail; 118 KiB/s, 0 objects/s recovering 2024-04-25T00:28:47.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:46 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2480308241' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:28:48.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:48 smithi032 ceph-mon[29357]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 270 MiB used, 804 GiB / 805 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-04-25T00:28:49.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:48 smithi121 ceph-mon[32231]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 270 MiB used, 804 GiB / 805 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-04-25T00:28:49.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:48 smithi130 ceph-mon[32525]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 270 MiB used, 804 GiB / 805 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-04-25T00:28:50.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:50 smithi032 ceph-mon[29357]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-25T00:28:51.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:50 smithi121 ceph-mon[32231]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-25T00:28:51.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:50 smithi130 ceph-mon[32525]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-25T00:28:52.991 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:52 smithi032 ceph-mon[29357]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-25T00:28:53.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:52 smithi121 ceph-mon[32231]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-25T00:28:53.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:52 smithi130 ceph-mon[32525]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-25T00:28:53.922 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:53 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.9"}]: dispatch 2024-04-25T00:28:53.922 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:53 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:54.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:53 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.9"}]: dispatch 2024-04-25T00:28:54.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:53 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:54.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:53 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.9"}]: dispatch 2024-04-25T00:28:54.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:53 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:28:54.988 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:54 smithi121 ceph-mon[32231]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-25T00:28:54.989 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:54 smithi121 ceph-mon[32231]: Deploying daemon osd.9 on smithi032 2024-04-25T00:28:54.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:54 smithi032 ceph-mon[29357]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-25T00:28:54.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:54 smithi032 ceph-mon[29357]: Deploying daemon osd.9 on smithi032 2024-04-25T00:28:55.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:54 smithi130 ceph-mon[32525]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-25T00:28:55.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:54 smithi130 ceph-mon[32525]: Deploying daemon osd.9 on smithi032 2024-04-25T00:28:56.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:56 smithi032 ceph-mon[29357]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-04-25T00:28:57.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:56 smithi121 ceph-mon[32231]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-04-25T00:28:57.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:56 smithi130 ceph-mon[32525]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-04-25T00:28:58.951 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:28:58 smithi032 ceph-mon[29357]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-04-25T00:28:59.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:28:58 smithi121 ceph-mon[32231]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-04-25T00:28:59.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:28:58 smithi130 ceph-mon[32525]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-04-25T00:29:00.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:00 smithi121 ceph-mon[32231]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-04-25T00:29:00.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:00 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:00.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:00 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:29:00.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:00 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:00.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:00 smithi130 ceph-mon[32525]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-04-25T00:29:00.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:00 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:00.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:00 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:29:00.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:00 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:00.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:00 smithi032 ceph-mon[29357]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-04-25T00:29:00.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:00 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:00.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:00 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:29:00.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:00 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:02.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:02 smithi032 ceph-mon[29357]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-04-25T00:29:02.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:02 smithi121 ceph-mon[32231]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-04-25T00:29:02.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:02 smithi130 ceph-mon[32525]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-04-25T00:29:03.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:03.439+0000 7fb5ed7fa700 1 -- 172.21.15.32:0/3250255016 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7fb5f01a0e00 con 0x7fb5d80a2c20 2024-04-25T00:29:03.439 INFO:teuthology.orchestra.run.smithi032.stdout:Created osd(s) 9 on host 'smithi032' 2024-04-25T00:29:03.441 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:03.441+0000 7fb5f668b700 1 -- 172.21.15.32:0/3250255016 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb5d80a2c20 msgr2=0x7fb5d80a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:29:03.441 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:03.441+0000 7fb5f668b700 1 --2- 172.21.15.32:0/3250255016 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb5d80a2c20 0x7fb5d80a50e0 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7fb5e40099d0 tx=0x7fb5e4008040 comp rx=0 tx=0).stop 2024-04-25T00:29:03.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:03.441+0000 7fb5f668b700 1 -- 172.21.15.32:0/3250255016 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb5f0101190 msgr2=0x7fb5f0198630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:29:03.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:03.441+0000 7fb5f668b700 1 --2- 172.21.15.32:0/3250255016 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb5f0101190 0x7fb5f0198630 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7fb5e000b530 tx=0x7fb5e000b8f0 comp rx=0 tx=0).stop 2024-04-25T00:29:03.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:03.441+0000 7fb5f668b700 1 -- 172.21.15.32:0/3250255016 shutdown_connections 2024-04-25T00:29:03.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:03.441+0000 7fb5f668b700 1 --2- 172.21.15.32:0/3250255016 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb5d80a2c20 0x7fb5d80a50e0 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:03.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:03.441+0000 7fb5f668b700 1 --2- 172.21.15.32:0/3250255016 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb5f0102d30 0x7fb5f019fba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:03.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:03.441+0000 7fb5f668b700 1 --2- 172.21.15.32:0/3250255016 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb5f0101190 0x7fb5f0198630 unknown :-1 s=CLOSED pgs=135 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:03.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:03.441+0000 7fb5f668b700 1 --2- 172.21.15.32:0/3250255016 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb5f0102390 0x7fb5f0198b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:03.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:03.441+0000 7fb5f668b700 1 -- 172.21.15.32:0/3250255016 >> 172.21.15.32:0/3250255016 conn(0x7fb5f00fc730 msgr2=0x7fb5f0069f10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:29:03.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:03.441+0000 7fb5f668b700 1 -- 172.21.15.32:0/3250255016 shutdown_connections 2024-04-25T00:29:03.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:03.441+0000 7fb5f668b700 1 -- 172.21.15.32:0/3250255016 wait complete. 2024-04-25T00:29:03.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:03 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:03.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:03 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:03.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:03 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:03.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:03 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:29:03.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:03 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:03.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:03 smithi032 ceph-mon[29357]: from='osd.9 [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050]' entity='osd.9' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]: dispatch 2024-04-25T00:29:03.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:03 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:03.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:03 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:03.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:03 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:29:03.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:03 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:03.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:03 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:03.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:03 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:03.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:03 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:29:03.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:03 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:03.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:03 smithi130 ceph-mon[32525]: from='osd.9 [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050]' entity='osd.9' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]: dispatch 2024-04-25T00:29:03.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:03 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:03.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:03 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:03.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:03 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:29:04.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:03 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:04.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:03 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:04.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:03 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:04.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:03 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:29:04.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:03 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:04.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:03 smithi121 ceph-mon[32231]: from='osd.9 [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050]' entity='osd.9' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]: dispatch 2024-04-25T00:29:04.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:03 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:04.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:03 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:04.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:03 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:29:04.133 DEBUG:teuthology.orchestra.run.smithi032:osd.9> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.9.service 2024-04-25T00:29:04.136 INFO:tasks.cephadm:Deploying osd.10 on smithi121 with /dev/vg_nvme/lv_1... 2024-04-25T00:29:04.136 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-25T00:29:04.378 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.b/config 2024-04-25T00:29:04.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:04 smithi032 ceph-mon[29357]: pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-04-25T00:29:04.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:04 smithi032 ceph-mon[29357]: from='osd.9 [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050]' entity='osd.9' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]': finished 2024-04-25T00:29:04.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:04 smithi032 ceph-mon[29357]: osdmap e65: 10 total, 9 up, 10 in 2024-04-25T00:29:04.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:04 smithi032 ceph-mon[29357]: from='osd.9 [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050]' entity='osd.9' cmd=[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-04-25T00:29:04.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:04 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-25T00:29:04.752 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:04 smithi121 ceph-mon[32231]: pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-04-25T00:29:04.752 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:04 smithi121 ceph-mon[32231]: from='osd.9 [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050]' entity='osd.9' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]': finished 2024-04-25T00:29:04.752 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:04 smithi121 ceph-mon[32231]: osdmap e65: 10 total, 9 up, 10 in 2024-04-25T00:29:04.752 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:04 smithi121 ceph-mon[32231]: from='osd.9 [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050]' entity='osd.9' cmd=[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-04-25T00:29:04.752 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:04 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-25T00:29:05.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:04 smithi130 ceph-mon[32525]: pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-04-25T00:29:05.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:04 smithi130 ceph-mon[32525]: from='osd.9 [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050]' entity='osd.9' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]': finished 2024-04-25T00:29:05.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:04 smithi130 ceph-mon[32525]: osdmap e65: 10 total, 9 up, 10 in 2024-04-25T00:29:05.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:04 smithi130 ceph-mon[32525]: from='osd.9 [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050]' entity='osd.9' cmd=[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-04-25T00:29:05.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:04 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-25T00:29:05.498 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:29:05 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:29:05.195+0000 7f3a448da700 -1 osd.9 0 waiting for initial osdmap 2024-04-25T00:29:05.499 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:29:05 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:29:05.207+0000 7f3a3bec4700 -1 osd.9 66 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T00:29:05.771 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:05 smithi121 ceph-mon[32231]: from='osd.9 [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050]' entity='osd.9' cmd='[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-04-25T00:29:05.771 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:05 smithi121 ceph-mon[32231]: osdmap e66: 10 total, 9 up, 10 in 2024-04-25T00:29:05.771 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:05 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-25T00:29:05.771 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:05 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-25T00:29:05.771 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:05 smithi121 ceph-mon[32231]: from='osd.9 [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050]' entity='osd.9' 2024-04-25T00:29:05.771 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:05 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-25T00:29:05.935 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:05 smithi032 ceph-mon[29357]: from='osd.9 [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050]' entity='osd.9' cmd='[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-04-25T00:29:05.936 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:05 smithi032 ceph-mon[29357]: osdmap e66: 10 total, 9 up, 10 in 2024-04-25T00:29:05.936 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:05 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-25T00:29:05.936 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:05 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-25T00:29:05.936 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:05 smithi032 ceph-mon[29357]: from='osd.9 [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050]' entity='osd.9' 2024-04-25T00:29:05.936 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:05 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-25T00:29:06.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:05 smithi130 ceph-mon[32525]: from='osd.9 [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050]' entity='osd.9' cmd='[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-04-25T00:29:06.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:05 smithi130 ceph-mon[32525]: osdmap e66: 10 total, 9 up, 10 in 2024-04-25T00:29:06.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:05 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-25T00:29:06.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:05 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-25T00:29:06.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:05 smithi130 ceph-mon[32525]: from='osd.9 [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050]' entity='osd.9' 2024-04-25T00:29:06.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:05 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-25T00:29:06.348 INFO:teuthology.orchestra.run.smithi121.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-25T00:29:06.348 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-25T00:29:06.348 INFO:teuthology.orchestra.run.smithi121.stderr: stderr: 10+0 records in 2024-04-25T00:29:06.348 INFO:teuthology.orchestra.run.smithi121.stderr:10+0 records out 2024-04-25T00:29:06.348 INFO:teuthology.orchestra.run.smithi121.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0113908 s, 921 MB/s 2024-04-25T00:29:06.348 INFO:teuthology.orchestra.run.smithi121.stderr:--> Zapping successful for: 2024-04-25T00:29:06.751 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:06 smithi032 ceph-mon[29357]: purged_snaps scrub starts 2024-04-25T00:29:06.752 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:06 smithi032 ceph-mon[29357]: purged_snaps scrub ok 2024-04-25T00:29:06.752 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:06 smithi032 ceph-mon[29357]: pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-04-25T00:29:06.752 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:06 smithi032 ceph-mon[29357]: osd.9 [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050] boot 2024-04-25T00:29:06.752 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:06 smithi032 ceph-mon[29357]: osdmap e67: 10 total, 10 up, 10 in 2024-04-25T00:29:06.752 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:06 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-25T00:29:06.899 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:06 smithi121 ceph-mon[32231]: purged_snaps scrub starts 2024-04-25T00:29:06.899 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:06 smithi121 ceph-mon[32231]: purged_snaps scrub ok 2024-04-25T00:29:06.900 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:06 smithi121 ceph-mon[32231]: pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-04-25T00:29:06.900 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:06 smithi121 ceph-mon[32231]: osd.9 [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050] boot 2024-04-25T00:29:06.900 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:06 smithi121 ceph-mon[32231]: osdmap e67: 10 total, 10 up, 10 in 2024-04-25T00:29:06.900 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:06 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-25T00:29:06.975 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi121:vg_nvme/lv_1 2024-04-25T00:29:07.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:06 smithi130 ceph-mon[32525]: purged_snaps scrub starts 2024-04-25T00:29:07.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:06 smithi130 ceph-mon[32525]: purged_snaps scrub ok 2024-04-25T00:29:07.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:06 smithi130 ceph-mon[32525]: pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-04-25T00:29:07.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:06 smithi130 ceph-mon[32525]: osd.9 [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050] boot 2024-04-25T00:29:07.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:06 smithi130 ceph-mon[32525]: osdmap e67: 10 total, 10 up, 10 in 2024-04-25T00:29:07.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:06 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-25T00:29:07.208 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.b/config 2024-04-25T00:29:08.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:08 smithi032 ceph-mon[29357]: pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 304 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:08.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:08 smithi032 ceph-mon[29357]: osdmap e68: 10 total, 10 up, 10 in 2024-04-25T00:29:08.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:08 smithi032 ceph-mon[29357]: Detected new or changed devices on smithi032 2024-04-25T00:29:08.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:08 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:08.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:08 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:08.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:08 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:08.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:08 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:08.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:08 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:08.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:08 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:08.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:08 smithi032 ceph-mon[29357]: Adjusting osd_memory_target on smithi032 to 4273M 2024-04-25T00:29:08.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:08 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:08.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:08 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:08.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:08 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:29:08.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:08 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:08.551 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:08 smithi121 ceph-mon[32231]: pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 304 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:08.551 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:08 smithi121 ceph-mon[32231]: osdmap e68: 10 total, 10 up, 10 in 2024-04-25T00:29:08.551 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:08 smithi121 ceph-mon[32231]: Detected new or changed devices on smithi032 2024-04-25T00:29:08.551 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:08 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:08.551 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:08 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:08.551 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:08 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:08.551 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:08 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:08.552 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:08 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:08.552 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:08 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:08.552 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:08 smithi121 ceph-mon[32231]: Adjusting osd_memory_target on smithi032 to 4273M 2024-04-25T00:29:08.552 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:08 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:08.552 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:08 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:08.552 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:08 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:29:08.552 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:08 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:08.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:08 smithi130 ceph-mon[32525]: pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 304 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:08.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:08 smithi130 ceph-mon[32525]: osdmap e68: 10 total, 10 up, 10 in 2024-04-25T00:29:08.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:08 smithi130 ceph-mon[32525]: Detected new or changed devices on smithi032 2024-04-25T00:29:08.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:08 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:08.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:08 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:08.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:08 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:08.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:08 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:08.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:08 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:08.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:08 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:08.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:08 smithi130 ceph-mon[32525]: Adjusting osd_memory_target on smithi032 to 4273M 2024-04-25T00:29:08.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:08 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:08.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:08 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:08.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:08 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:29:08.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:08 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:08.766 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.763+0000 7fb72b5f8700 1 -- 172.21.15.121:0/1408302283 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb724101e60 msgr2=0x7fb724109700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:29:08.766 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.763+0000 7fb72b5f8700 1 --2- 172.21.15.121:0/1408302283 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb724101e60 0x7fb724109700 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fb71c00b0a0 tx=0x7fb71c00b3b0 comp rx=0 tx=0).stop 2024-04-25T00:29:08.766 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.764+0000 7fb72b5f8700 1 -- 172.21.15.121:0/1408302283 shutdown_connections 2024-04-25T00:29:08.766 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.764+0000 7fb72b5f8700 1 --2- 172.21.15.121:0/1408302283 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb724101e60 0x7fb724109700 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:08.766 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.764+0000 7fb72b5f8700 1 --2- 172.21.15.121:0/1408302283 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb7241028d0 0x7fb724102cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:08.766 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.764+0000 7fb72b5f8700 1 --2- 172.21.15.121:0/1408302283 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb7241014a0 0x7fb724101920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:08.767 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.764+0000 7fb72b5f8700 1 -- 172.21.15.121:0/1408302283 >> 172.21.15.121:0/1408302283 conn(0x7fb7240fbf70 msgr2=0x7fb7240fe3b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:29:08.767 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.764+0000 7fb72b5f8700 1 -- 172.21.15.121:0/1408302283 shutdown_connections 2024-04-25T00:29:08.767 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.764+0000 7fb72b5f8700 1 -- 172.21.15.121:0/1408302283 wait complete. 2024-04-25T00:29:08.767 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.764+0000 7fb72b5f8700 1 Processor -- start 2024-04-25T00:29:08.767 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.764+0000 7fb72b5f8700 1 -- start start 2024-04-25T00:29:08.767 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.764+0000 7fb72b5f8700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb7241014a0 0x7fb724100ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:29:08.767 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.764+0000 7fb72b5f8700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb724101e60 0x7fb72419d1f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:29:08.767 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.764+0000 7fb72b5f8700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb7241028d0 0x7fb72419f830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:29:08.767 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.764+0000 7fb72b5f8700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fb72410ce70 con 0x7fb7241014a0 2024-04-25T00:29:08.767 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.764+0000 7fb72b5f8700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fb72410ccf0 con 0x7fb724101e60 2024-04-25T00:29:08.767 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.764+0000 7fb72b5f8700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fb72410cff0 con 0x7fb7241028d0 2024-04-25T00:29:08.767 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.765+0000 7fb728b93700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb724101e60 0x7fb72419d1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:29:08.767 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.765+0000 7fb728b93700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb724101e60 0x7fb72419d1f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.121:55404/0 (socket says 172.21.15.121:55404) 2024-04-25T00:29:08.767 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.765+0000 7fb728b93700 1 -- 172.21.15.121:0/2061876434 learned_addr learned my addr 172.21.15.121:0/2061876434 (peer_addr_for_me v2:172.21.15.121:0/0) 2024-04-25T00:29:08.768 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.765+0000 7fb729b95700 1 --2- 172.21.15.121:0/2061876434 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb7241028d0 0x7fb72419f830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:29:08.768 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.765+0000 7fb729394700 1 --2- 172.21.15.121:0/2061876434 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb7241014a0 0x7fb724100ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:29:08.768 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.765+0000 7fb728b93700 1 -- 172.21.15.121:0/2061876434 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb7241028d0 msgr2=0x7fb72419f830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:29:08.768 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.765+0000 7fb728b93700 1 --2- 172.21.15.121:0/2061876434 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb7241028d0 0x7fb72419f830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:08.768 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.765+0000 7fb728b93700 1 -- 172.21.15.121:0/2061876434 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb7241014a0 msgr2=0x7fb724100ef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:29:08.768 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.765+0000 7fb728b93700 1 --2- 172.21.15.121:0/2061876434 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb7241014a0 0x7fb724100ef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:08.768 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.765+0000 7fb728b93700 1 -- 172.21.15.121:0/2061876434 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb7241a0100 con 0x7fb724101e60 2024-04-25T00:29:08.768 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.765+0000 7fb728b93700 1 --2- 172.21.15.121:0/2061876434 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb724101e60 0x7fb72419d1f0 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7fb71400c920 tx=0x7fb71400cc30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:29:08.768 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.765+0000 7fb71a7fc700 1 -- 172.21.15.121:0/2061876434 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb714004a30 con 0x7fb724101e60 2024-04-25T00:29:08.768 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.765+0000 7fb72b5f8700 1 -- 172.21.15.121:0/2061876434 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb7241a03f0 con 0x7fb724101e60 2024-04-25T00:29:08.768 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.765+0000 7fb71a7fc700 1 -- 172.21.15.121:0/2061876434 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb714005500 con 0x7fb724101e60 2024-04-25T00:29:08.768 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.765+0000 7fb71a7fc700 1 -- 172.21.15.121:0/2061876434 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb714007d50 con 0x7fb724101e60 2024-04-25T00:29:08.768 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.765+0000 7fb72b5f8700 1 -- 172.21.15.121:0/2061876434 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb7241a0930 con 0x7fb724101e60 2024-04-25T00:29:08.769 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.766+0000 7fb72b5f8700 1 -- 172.21.15.121:0/2061876434 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb724068f20 con 0x7fb724101e60 2024-04-25T00:29:08.769 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.767+0000 7fb71a7fc700 1 -- 172.21.15.121:0/2061876434 <== mon.2 v2:172.21.15.121:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7fb714004d60 con 0x7fb724101e60 2024-04-25T00:29:08.769 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.768+0000 7fb71a7fc700 1 --2- 172.21.15.121:0/2061876434 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb7100a2c20 0x7fb7100a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:29:08.769 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.768+0000 7fb71a7fc700 1 -- 172.21.15.121:0/2061876434 <== mon.2 v2:172.21.15.121:3300/0 5 ==== osd_map(68..68 src has 1..68) ==== 6261+0+0 (secure 0 0 0) 0x7fb7140ce720 con 0x7fb724101e60 2024-04-25T00:29:08.769 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.768+0000 7fb729394700 1 --2- 172.21.15.121:0/2061876434 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb7100a2c20 0x7fb7100a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:29:08.769 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.769+0000 7fb729394700 1 --2- 172.21.15.121:0/2061876434 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb7100a2c20 0x7fb7100a50e0 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7fb720005950 tx=0x7fb72000a300 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:29:08.772 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.772+0000 7fb71a7fc700 1 -- 172.21.15.121:0/2061876434 <== mon.2 v2:172.21.15.121:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fb714014030 con 0x7fb724101e60 2024-04-25T00:29:08.976 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:08.975+0000 7fb72b5f8700 1 -- 172.21.15.121:0/2061876434 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_1", "target": ["mon-mgr", ""]}) -- 0x7fb724062670 con 0x7fb7100a2c20 2024-04-25T00:29:09.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:09 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:29:09.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:09 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:29:09.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:09 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:09.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:09 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:29:09.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:09 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:29:09.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:09 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:09.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:09 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:29:09.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:09 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:29:09.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:09 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:10.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:10 smithi032 ceph-mon[29357]: pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:10.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:10 smithi032 ceph-mon[29357]: from='client.24335 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:29:10.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:10 smithi121 ceph-mon[32231]: pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:10.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:10 smithi121 ceph-mon[32231]: from='client.24335 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:29:10.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:10 smithi130 ceph-mon[32525]: pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:10.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:10 smithi130 ceph-mon[32525]: from='client.24335 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:29:12.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:12 smithi032 ceph-mon[29357]: pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:12.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:12 smithi032 ceph-mon[29357]: from='client.? 172.21.15.121:0/3059798074' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "16161ce2-7971-4403-95d4-1c6b2977ce2f"}]: dispatch 2024-04-25T00:29:12.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:12 smithi032 ceph-mon[29357]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "16161ce2-7971-4403-95d4-1c6b2977ce2f"}]: dispatch 2024-04-25T00:29:12.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:12 smithi032 ceph-mon[29357]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "16161ce2-7971-4403-95d4-1c6b2977ce2f"}]': finished 2024-04-25T00:29:12.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:12 smithi032 ceph-mon[29357]: osdmap e69: 11 total, 10 up, 11 in 2024-04-25T00:29:12.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:12 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-25T00:29:12.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:12 smithi121 ceph-mon[32231]: pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:12.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:12 smithi121 ceph-mon[32231]: from='client.? 172.21.15.121:0/3059798074' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "16161ce2-7971-4403-95d4-1c6b2977ce2f"}]: dispatch 2024-04-25T00:29:12.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:12 smithi121 ceph-mon[32231]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "16161ce2-7971-4403-95d4-1c6b2977ce2f"}]: dispatch 2024-04-25T00:29:12.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:12 smithi121 ceph-mon[32231]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "16161ce2-7971-4403-95d4-1c6b2977ce2f"}]': finished 2024-04-25T00:29:12.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:12 smithi121 ceph-mon[32231]: osdmap e69: 11 total, 10 up, 11 in 2024-04-25T00:29:12.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:12 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-25T00:29:12.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:12 smithi130 ceph-mon[32525]: pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:12.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:12 smithi130 ceph-mon[32525]: from='client.? 172.21.15.121:0/3059798074' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "16161ce2-7971-4403-95d4-1c6b2977ce2f"}]: dispatch 2024-04-25T00:29:12.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:12 smithi130 ceph-mon[32525]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "16161ce2-7971-4403-95d4-1c6b2977ce2f"}]: dispatch 2024-04-25T00:29:12.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:12 smithi130 ceph-mon[32525]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "16161ce2-7971-4403-95d4-1c6b2977ce2f"}]': finished 2024-04-25T00:29:12.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:12 smithi130 ceph-mon[32525]: osdmap e69: 11 total, 10 up, 11 in 2024-04-25T00:29:12.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:12 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-25T00:29:13.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:13 smithi032 ceph-mon[29357]: from='client.? 172.21.15.121:0/4161866331' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:29:13.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:13 smithi121 ceph-mon[32231]: from='client.? 172.21.15.121:0/4161866331' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:29:13.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:13 smithi130 ceph-mon[32525]: from='client.? 172.21.15.121:0/4161866331' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:29:14.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:14 smithi032 ceph-mon[29357]: pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:14.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:14 smithi121 ceph-mon[32231]: pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:14.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:14 smithi130 ceph-mon[32525]: pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:16.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:16 smithi121 ceph-mon[32231]: pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:16.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:16 smithi032 ceph-mon[29357]: pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:16.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:16 smithi130 ceph-mon[32525]: pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:18.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:18 smithi032 ceph-mon[29357]: pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:18.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:18 smithi121 ceph-mon[32231]: pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:18.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:18 smithi130 ceph-mon[32525]: pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:19.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:19 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.10"}]: dispatch 2024-04-25T00:29:19.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:19 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:19.540 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:19 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.10"}]: dispatch 2024-04-25T00:29:19.540 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:19 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:19.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:19 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.10"}]: dispatch 2024-04-25T00:29:19.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:19 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:20.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:20 smithi032 ceph-mon[29357]: pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:20.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:20 smithi032 ceph-mon[29357]: Deploying daemon osd.10 on smithi121 2024-04-25T00:29:20.543 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:20 smithi121 ceph-mon[32231]: pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:20.544 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:20 smithi121 ceph-mon[32231]: Deploying daemon osd.10 on smithi121 2024-04-25T00:29:20.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:20 smithi130 ceph-mon[32525]: pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:20 smithi130 ceph-mon[32525]: Deploying daemon osd.10 on smithi121 2024-04-25T00:29:22.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:22 smithi032 ceph-mon[29357]: pgmap v228: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:22.574 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:22 smithi121 ceph-mon[32231]: pgmap v228: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:22.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:22 smithi130 ceph-mon[32525]: pgmap v228: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:24.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:24 smithi032 ceph-mon[29357]: pgmap v229: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:24.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:24 smithi121 ceph-mon[32231]: pgmap v229: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:24.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:24 smithi130 ceph-mon[32525]: pgmap v229: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:25.529 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:25 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:25.529 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:25 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:25.530 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:25 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:29:25.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:25 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:25.751 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:25 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:25.751 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:25 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:29:25.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:25 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:25.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:25 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:25.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:25 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:29:26.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:26 smithi032 ceph-mon[29357]: pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:26.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:26 smithi121 ceph-mon[32231]: pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:26.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:26 smithi130 ceph-mon[32525]: pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:28.045 INFO:teuthology.orchestra.run.smithi121.stdout:Created osd(s) 10 on host 'smithi121' 2024-04-25T00:29:28.046 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:28.044+0000 7fb71a7fc700 1 -- 172.21.15.121:0/2061876434 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+38 (secure 0 0 0) 0x7fb724062670 con 0x7fb7100a2c20 2024-04-25T00:29:28.047 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:28.046+0000 7fb72b5f8700 1 -- 172.21.15.121:0/2061876434 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb7100a2c20 msgr2=0x7fb7100a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:29:28.047 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:28.046+0000 7fb72b5f8700 1 --2- 172.21.15.121:0/2061876434 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb7100a2c20 0x7fb7100a50e0 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7fb720005950 tx=0x7fb72000a300 comp rx=0 tx=0).stop 2024-04-25T00:29:28.047 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:28.046+0000 7fb72b5f8700 1 -- 172.21.15.121:0/2061876434 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb724101e60 msgr2=0x7fb72419d1f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:29:28.047 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:28.046+0000 7fb72b5f8700 1 --2- 172.21.15.121:0/2061876434 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb724101e60 0x7fb72419d1f0 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7fb71400c920 tx=0x7fb71400cc30 comp rx=0 tx=0).stop 2024-04-25T00:29:28.047 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:28.046+0000 7fb72b5f8700 1 -- 172.21.15.121:0/2061876434 shutdown_connections 2024-04-25T00:29:28.047 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:28.046+0000 7fb72b5f8700 1 --2- 172.21.15.121:0/2061876434 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb7100a2c20 0x7fb7100a50e0 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:28.047 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:28.046+0000 7fb72b5f8700 1 --2- 172.21.15.121:0/2061876434 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb7241028d0 0x7fb72419f830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:28.048 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:28.046+0000 7fb72b5f8700 1 --2- 172.21.15.121:0/2061876434 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb7241014a0 0x7fb724100ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:28.048 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:28.046+0000 7fb72b5f8700 1 --2- 172.21.15.121:0/2061876434 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb724101e60 0x7fb72419d1f0 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:28.048 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:28.046+0000 7fb72b5f8700 1 -- 172.21.15.121:0/2061876434 >> 172.21.15.121:0/2061876434 conn(0x7fb7240fbf70 msgr2=0x7fb7241056e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:29:28.048 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:28.046+0000 7fb72b5f8700 1 -- 172.21.15.121:0/2061876434 shutdown_connections 2024-04-25T00:29:28.048 INFO:teuthology.orchestra.run.smithi121.stderr:2024-04-25T00:29:28.046+0000 7fb72b5f8700 1 -- 172.21.15.121:0/2061876434 wait complete. 2024-04-25T00:29:28.655 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:28 smithi121 ceph-mon[32231]: pgmap v231: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:28.655 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:28 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:28.655 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:28 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:28.655 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:28 smithi121 ceph-mon[32231]: from='osd.10 [v2:172.21.15.121:6824/3217408278,v1:172.21.15.121:6825/3217408278]' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2024-04-25T00:29:28.655 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:28 smithi121 ceph-mon[32231]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2024-04-25T00:29:28.680 DEBUG:teuthology.orchestra.run.smithi121:osd.10> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.10.service 2024-04-25T00:29:28.683 INFO:tasks.cephadm:Deploying osd.11 on smithi130 with /dev/vg_nvme/lv_1... 2024-04-25T00:29:28.683 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-25T00:29:28.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:28 smithi032 ceph-mon[29357]: pgmap v231: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:28.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:28 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:28.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:28 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:28.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:28 smithi032 ceph-mon[29357]: from='osd.10 [v2:172.21.15.121:6824/3217408278,v1:172.21.15.121:6825/3217408278]' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2024-04-25T00:29:28.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:28 smithi032 ceph-mon[29357]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2024-04-25T00:29:28.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:28 smithi130 ceph-mon[32525]: pgmap v231: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:28.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:28.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:28.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:28 smithi130 ceph-mon[32525]: from='osd.10 [v2:172.21.15.121:6824/3217408278,v1:172.21.15.121:6825/3217408278]' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2024-04-25T00:29:28.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:28 smithi130 ceph-mon[32525]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2024-04-25T00:29:28.930 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.c/config 2024-04-25T00:29:30.032 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:29 smithi121 ceph-mon[32231]: pgmap v232: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:30.032 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:29 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:30.032 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:29 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:30.032 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:29 smithi121 ceph-mon[32231]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]': finished 2024-04-25T00:29:30.032 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:29 smithi121 ceph-mon[32231]: from='osd.10 [v2:172.21.15.121:6824/3217408278,v1:172.21.15.121:6825/3217408278]' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-25T00:29:30.032 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:29 smithi121 ceph-mon[32231]: osdmap e70: 11 total, 10 up, 11 in 2024-04-25T00:29:30.032 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:29 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-25T00:29:30.032 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:29 smithi121 ceph-mon[32231]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-25T00:29:30.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:29 smithi130 ceph-mon[32525]: pgmap v232: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:30.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:29 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:30.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:29 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:30.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:29 smithi130 ceph-mon[32525]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]': finished 2024-04-25T00:29:30.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:29 smithi130 ceph-mon[32525]: from='osd.10 [v2:172.21.15.121:6824/3217408278,v1:172.21.15.121:6825/3217408278]' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-25T00:29:30.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:29 smithi130 ceph-mon[32525]: osdmap e70: 11 total, 10 up, 11 in 2024-04-25T00:29:30.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:29 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-25T00:29:30.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:29 smithi130 ceph-mon[32525]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-25T00:29:30.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:29 smithi032 ceph-mon[29357]: pgmap v232: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:30.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:29 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:30.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:29 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:30.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:29 smithi032 ceph-mon[29357]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]': finished 2024-04-25T00:29:30.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:29 smithi032 ceph-mon[29357]: from='osd.10 [v2:172.21.15.121:6824/3217408278,v1:172.21.15.121:6825/3217408278]' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-25T00:29:30.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:29 smithi032 ceph-mon[29357]: osdmap e70: 11 total, 10 up, 11 in 2024-04-25T00:29:30.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:29 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-25T00:29:30.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:29 smithi032 ceph-mon[29357]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-25T00:29:30.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:29:30 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:29:30.030+0000 7fe96c47f700 -1 osd.10 0 waiting for initial osdmap 2024-04-25T00:29:30.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:29:30 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:29:30.041+0000 7fe968272700 -1 osd.10 71 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T00:29:31.120 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-25T00:29:31.120 INFO:teuthology.orchestra.run.smithi130.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-25T00:29:31.120 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10+0 records in 2024-04-25T00:29:31.120 INFO:teuthology.orchestra.run.smithi130.stderr:10+0 records out 2024-04-25T00:29:31.121 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0128369 s, 817 MB/s 2024-04-25T00:29:31.121 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping successful for: 2024-04-25T00:29:31.147 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:31 smithi130 ceph-mon[32525]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-04-25T00:29:31.147 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:31 smithi130 ceph-mon[32525]: osdmap e71: 11 total, 10 up, 11 in 2024-04-25T00:29:31.148 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:31 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-25T00:29:31.148 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:31 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-25T00:29:31.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:31 smithi121 ceph-mon[32231]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-04-25T00:29:31.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:31 smithi121 ceph-mon[32231]: osdmap e71: 11 total, 10 up, 11 in 2024-04-25T00:29:31.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:31 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-25T00:29:31.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:31 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-25T00:29:31.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:31 smithi032 ceph-mon[29357]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-04-25T00:29:31.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:31 smithi032 ceph-mon[29357]: osdmap e71: 11 total, 10 up, 11 in 2024-04-25T00:29:31.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:31 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-25T00:29:31.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:31 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-25T00:29:31.897 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi130:vg_nvme/lv_1 2024-04-25T00:29:32.137 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.c/config 2024-04-25T00:29:32.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:32 smithi130 ceph-mon[32525]: purged_snaps scrub starts 2024-04-25T00:29:32.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:32 smithi130 ceph-mon[32525]: purged_snaps scrub ok 2024-04-25T00:29:32.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:32 smithi130 ceph-mon[32525]: pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:32.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:32 smithi130 ceph-mon[32525]: osd.10 [v2:172.21.15.121:6824/3217408278,v1:172.21.15.121:6825/3217408278] boot 2024-04-25T00:29:32.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:32 smithi130 ceph-mon[32525]: osdmap e72: 11 total, 11 up, 11 in 2024-04-25T00:29:32.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:32 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-25T00:29:32.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:32 smithi130 ceph-mon[32525]: osdmap e73: 11 total, 11 up, 11 in 2024-04-25T00:29:32.205 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:32 smithi121 ceph-mon[32231]: purged_snaps scrub starts 2024-04-25T00:29:32.205 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:32 smithi121 ceph-mon[32231]: purged_snaps scrub ok 2024-04-25T00:29:32.205 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:32 smithi121 ceph-mon[32231]: pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:32.205 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:32 smithi121 ceph-mon[32231]: osd.10 [v2:172.21.15.121:6824/3217408278,v1:172.21.15.121:6825/3217408278] boot 2024-04-25T00:29:32.205 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:32 smithi121 ceph-mon[32231]: osdmap e72: 11 total, 11 up, 11 in 2024-04-25T00:29:32.205 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:32 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-25T00:29:32.205 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:32 smithi121 ceph-mon[32231]: osdmap e73: 11 total, 11 up, 11 in 2024-04-25T00:29:32.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:32 smithi032 ceph-mon[29357]: purged_snaps scrub starts 2024-04-25T00:29:32.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:32 smithi032 ceph-mon[29357]: purged_snaps scrub ok 2024-04-25T00:29:32.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:32 smithi032 ceph-mon[29357]: pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 305 MiB used, 894 GiB / 894 GiB avail 2024-04-25T00:29:32.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:32 smithi032 ceph-mon[29357]: osd.10 [v2:172.21.15.121:6824/3217408278,v1:172.21.15.121:6825/3217408278] boot 2024-04-25T00:29:32.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:32 smithi032 ceph-mon[29357]: osdmap e72: 11 total, 11 up, 11 in 2024-04-25T00:29:32.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:32 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-25T00:29:32.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:32 smithi032 ceph-mon[29357]: osdmap e73: 11 total, 11 up, 11 in 2024-04-25T00:29:33.345 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:33 smithi130 ceph-mon[32525]: Detected new or changed devices on smithi121 2024-04-25T00:29:33.345 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:33 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:33.345 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:33 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:33.345 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:33 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:33.346 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:33 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:33.346 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:33 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:33.346 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:33 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:33.346 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:33 smithi130 ceph-mon[32525]: Adjusting osd_memory_target on smithi121 to 4272M 2024-04-25T00:29:33.346 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:33 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:33.346 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:33 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:33.346 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:33 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:29:33.346 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:33 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:33.346 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:33 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:29:33.346 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:33 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:33.346 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:33 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:29:33.346 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:33 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:33.346 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:33 smithi130 ceph-mon[32525]: osdmap e74: 11 total, 11 up, 11 in 2024-04-25T00:29:33.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:33 smithi121 ceph-mon[32231]: Detected new or changed devices on smithi121 2024-04-25T00:29:33.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:33 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:33.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:33 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:33.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:33 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:33.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:33 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:33.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:33 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:33.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:33 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:33.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:33 smithi121 ceph-mon[32231]: Adjusting osd_memory_target on smithi121 to 4272M 2024-04-25T00:29:33.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:33 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:33.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:33 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:33.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:33 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:29:33.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:33 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:33.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:33 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:29:33.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:33 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:33.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:33 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:29:33.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:33 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:33.589 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:33 smithi121 ceph-mon[32231]: osdmap e74: 11 total, 11 up, 11 in 2024-04-25T00:29:33.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:33 smithi032 ceph-mon[29357]: Detected new or changed devices on smithi121 2024-04-25T00:29:33.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:33 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:33.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:33 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:33.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:33 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:33.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:33 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:33.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:33 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:33.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:33 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:33.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:33 smithi032 ceph-mon[29357]: Adjusting osd_memory_target on smithi121 to 4272M 2024-04-25T00:29:33.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:33 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:33.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:33 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:33.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:33 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:29:33.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:33 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:33.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:33 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:29:33.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:33 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:33.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:33 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:29:33.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:33 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:33.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:33 smithi032 ceph-mon[29357]: osdmap e74: 11 total, 11 up, 11 in 2024-04-25T00:29:33.854 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.852+0000 7f6daeb14700 1 -- 172.21.15.130:0/1012851611 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6da8103ed0 msgr2=0x7f6da810a780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:29:33.854 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.852+0000 7f6daeb14700 1 --2- 172.21.15.130:0/1012851611 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6da8103ed0 0x7f6da810a780 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7f6da000b0a0 tx=0x7f6da000b3b0 comp rx=0 tx=0).stop 2024-04-25T00:29:33.854 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.852+0000 7f6daeb14700 1 -- 172.21.15.130:0/1012851611 shutdown_connections 2024-04-25T00:29:33.854 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.852+0000 7f6daeb14700 1 --2- 172.21.15.130:0/1012851611 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6da8103ed0 0x7f6da810a780 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:33.854 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.852+0000 7f6daeb14700 1 --2- 172.21.15.130:0/1012851611 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6da8100550 0x7f6da8100970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:33.854 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.852+0000 7f6daeb14700 1 --2- 172.21.15.130:0/1012851611 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6da8103530 0x7f6da8103990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:33.854 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.852+0000 7f6daeb14700 1 -- 172.21.15.130:0/1012851611 >> 172.21.15.130:0/1012851611 conn(0x7f6da80fbe90 msgr2=0x7f6da80fe2f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:29:33.854 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.852+0000 7f6daeb14700 1 -- 172.21.15.130:0/1012851611 shutdown_connections 2024-04-25T00:29:33.855 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.852+0000 7f6daeb14700 1 -- 172.21.15.130:0/1012851611 wait complete. 2024-04-25T00:29:33.855 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.852+0000 7f6daeb14700 1 Processor -- start 2024-04-25T00:29:33.855 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.852+0000 7f6daeb14700 1 -- start start 2024-04-25T00:29:33.855 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.852+0000 7f6daeb14700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6da8100550 0x7f6da8198490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:29:33.855 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.852+0000 7f6daeb14700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6da8103530 0x7f6da81989d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:29:33.855 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.852+0000 7f6daeb14700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6da8103ed0 0x7f6da819fa50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:29:33.855 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.852+0000 7f6daeb14700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f6da810dee0 con 0x7f6da8100550 2024-04-25T00:29:33.855 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.852+0000 7f6daeb14700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f6da810dd60 con 0x7f6da8103530 2024-04-25T00:29:33.855 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.852+0000 7f6daeb14700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f6da810e060 con 0x7f6da8103ed0 2024-04-25T00:29:33.855 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.853+0000 7f6dad0b1700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6da8103ed0 0x7f6da819fa50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:29:33.855 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.853+0000 7f6dad0b1700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6da8103ed0 0x7f6da819fa50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.130:45434/0 (socket says 172.21.15.130:45434) 2024-04-25T00:29:33.855 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.853+0000 7f6dad0b1700 1 -- 172.21.15.130:0/2086418676 learned_addr learned my addr 172.21.15.130:0/2086418676 (peer_addr_for_me v2:172.21.15.130:0/0) 2024-04-25T00:29:33.855 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.853+0000 7f6da7fff700 1 --2- 172.21.15.130:0/2086418676 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6da8103530 0x7f6da81989d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:29:33.856 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.853+0000 7f6dac8b0700 1 --2- 172.21.15.130:0/2086418676 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6da8100550 0x7f6da8198490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:29:33.856 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.853+0000 7f6dad0b1700 1 -- 172.21.15.130:0/2086418676 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6da8103530 msgr2=0x7f6da81989d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:29:33.856 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.853+0000 7f6dad0b1700 1 --2- 172.21.15.130:0/2086418676 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6da8103530 0x7f6da81989d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:33.856 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.853+0000 7f6dad0b1700 1 -- 172.21.15.130:0/2086418676 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6da8100550 msgr2=0x7f6da8198490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:29:33.856 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.853+0000 7f6dad0b1700 1 --2- 172.21.15.130:0/2086418676 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6da8100550 0x7f6da8198490 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:33.856 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.853+0000 7f6dad0b1700 1 -- 172.21.15.130:0/2086418676 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6da81a00f0 con 0x7f6da8103ed0 2024-04-25T00:29:33.856 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.853+0000 7f6da7fff700 1 --2- 172.21.15.130:0/2086418676 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6da8103530 0x7f6da81989d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:29:33.856 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.853+0000 7f6dad0b1700 1 --2- 172.21.15.130:0/2086418676 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6da8103ed0 0x7f6da819fa50 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f6da000d010 tx=0x7f6da000bf10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:29:33.856 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.853+0000 7f6da5ffb700 1 -- 172.21.15.130:0/2086418676 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6da0021070 con 0x7f6da8103ed0 2024-04-25T00:29:33.856 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.853+0000 7f6da5ffb700 1 -- 172.21.15.130:0/2086418676 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6da0004020 con 0x7f6da8103ed0 2024-04-25T00:29:33.856 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.853+0000 7f6da5ffb700 1 -- 172.21.15.130:0/2086418676 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6da001d6b0 con 0x7f6da8103ed0 2024-04-25T00:29:33.856 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.853+0000 7f6daeb14700 1 -- 172.21.15.130:0/2086418676 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6da81a0380 con 0x7f6da8103ed0 2024-04-25T00:29:33.856 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.853+0000 7f6daeb14700 1 -- 172.21.15.130:0/2086418676 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f6da81a07e0 con 0x7f6da8103ed0 2024-04-25T00:29:33.857 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.854+0000 7f6daeb14700 1 -- 172.21.15.130:0/2086418676 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6da8068f20 con 0x7f6da8103ed0 2024-04-25T00:29:33.857 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.855+0000 7f6da5ffb700 1 -- 172.21.15.130:0/2086418676 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f6da00041d0 con 0x7f6da8103ed0 2024-04-25T00:29:33.857 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.856+0000 7f6da5ffb700 1 --2- 172.21.15.130:0/2086418676 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f6d980a2c20 0x7f6d980a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:29:33.858 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.856+0000 7f6da5ffb700 1 -- 172.21.15.130:0/2086418676 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(75..75 src has 1..75) ==== 6694+0+0 (secure 0 0 0) 0x7f6da00cfdd0 con 0x7f6da8103ed0 2024-04-25T00:29:33.858 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.856+0000 7f6dac8b0700 1 --2- 172.21.15.130:0/2086418676 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f6d980a2c20 0x7f6d980a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:29:33.858 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.857+0000 7f6dac8b0700 1 --2- 172.21.15.130:0/2086418676 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f6d980a2c20 0x7f6d980a50e0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f6d9c005fd0 tx=0x7f6d9c00a560 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:29:33.861 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:33.861+0000 7f6da5ffb700 1 -- 172.21.15.130:0/2086418676 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f6da009d410 con 0x7f6da8103ed0 2024-04-25T00:29:34.067 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:34.066+0000 7f6daeb14700 1 -- 172.21.15.130:0/2086418676 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_1", "target": ["mon-mgr", ""]}) -- 0x7f6da8062670 con 0x7f6d980a2c20 2024-04-25T00:29:34.994 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:34 smithi130 ceph-mon[32525]: pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 332 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:34.994 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:34 smithi130 ceph-mon[32525]: osdmap e75: 11 total, 11 up, 11 in 2024-04-25T00:29:34.994 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:29:34.994 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:29:34.994 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:34.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:34 smithi032 ceph-mon[29357]: pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 332 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:34.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:34 smithi032 ceph-mon[29357]: osdmap e75: 11 total, 11 up, 11 in 2024-04-25T00:29:34.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:29:34.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:29:34.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:35.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:34 smithi121 ceph-mon[32231]: pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 332 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:35.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:34 smithi121 ceph-mon[32231]: osdmap e75: 11 total, 11 up, 11 in 2024-04-25T00:29:35.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T00:29:35.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T00:29:35.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:35.931 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:35 smithi130 ceph-mon[32525]: from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:29:35.931 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:35 smithi130 ceph-mon[32525]: osdmap e76: 11 total, 11 up, 11 in 2024-04-25T00:29:35.975 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:35 smithi121 ceph-mon[32231]: from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:29:35.975 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:35 smithi121 ceph-mon[32231]: osdmap e76: 11 total, 11 up, 11 in 2024-04-25T00:29:35.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:35 smithi032 ceph-mon[29357]: from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:29:35.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:35 smithi032 ceph-mon[29357]: osdmap e76: 11 total, 11 up, 11 in 2024-04-25T00:29:36.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:36 smithi121 ceph-mon[32231]: pgmap v242: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 332 MiB used, 983 GiB / 983 GiB avail; 151 KiB/s, 0 objects/s recovering 2024-04-25T00:29:36.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:36 smithi032 ceph-mon[29357]: pgmap v242: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 332 MiB used, 983 GiB / 983 GiB avail; 151 KiB/s, 0 objects/s recovering 2024-04-25T00:29:37.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:36 smithi130 ceph-mon[32525]: pgmap v242: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 332 MiB used, 983 GiB / 983 GiB avail; 151 KiB/s, 0 objects/s recovering 2024-04-25T00:29:37.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:37 smithi032 ceph-mon[29357]: from='client.? 172.21.15.130:0/548922778' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fdf0b14b-2f62-4c0b-aec5-a8304d03042b"}]: dispatch 2024-04-25T00:29:37.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:37 smithi032 ceph-mon[29357]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fdf0b14b-2f62-4c0b-aec5-a8304d03042b"}]: dispatch 2024-04-25T00:29:37.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:37 smithi032 ceph-mon[29357]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fdf0b14b-2f62-4c0b-aec5-a8304d03042b"}]': finished 2024-04-25T00:29:37.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:37 smithi032 ceph-mon[29357]: osdmap e77: 12 total, 11 up, 12 in 2024-04-25T00:29:37.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:37 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-25T00:29:37.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:37 smithi032 ceph-mon[29357]: from='client.? 172.21.15.130:0/2614734597' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:29:38.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:37 smithi121 ceph-mon[32231]: from='client.? 172.21.15.130:0/548922778' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fdf0b14b-2f62-4c0b-aec5-a8304d03042b"}]: dispatch 2024-04-25T00:29:38.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:37 smithi121 ceph-mon[32231]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fdf0b14b-2f62-4c0b-aec5-a8304d03042b"}]: dispatch 2024-04-25T00:29:38.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:37 smithi121 ceph-mon[32231]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fdf0b14b-2f62-4c0b-aec5-a8304d03042b"}]': finished 2024-04-25T00:29:38.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:37 smithi121 ceph-mon[32231]: osdmap e77: 12 total, 11 up, 12 in 2024-04-25T00:29:38.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-25T00:29:38.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:37 smithi121 ceph-mon[32231]: from='client.? 172.21.15.130:0/2614734597' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:29:38.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:37 smithi130 ceph-mon[32525]: from='client.? 172.21.15.130:0/548922778' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fdf0b14b-2f62-4c0b-aec5-a8304d03042b"}]: dispatch 2024-04-25T00:29:38.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:37 smithi130 ceph-mon[32525]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fdf0b14b-2f62-4c0b-aec5-a8304d03042b"}]: dispatch 2024-04-25T00:29:38.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:37 smithi130 ceph-mon[32525]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fdf0b14b-2f62-4c0b-aec5-a8304d03042b"}]': finished 2024-04-25T00:29:38.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:37 smithi130 ceph-mon[32525]: osdmap e77: 12 total, 11 up, 12 in 2024-04-25T00:29:38.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:37 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-25T00:29:38.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:37 smithi130 ceph-mon[32525]: from='client.? 172.21.15.130:0/2614734597' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T00:29:38.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:38 smithi032 ceph-mon[29357]: pgmap v243: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-04-25T00:29:39.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:38 smithi121 ceph-mon[32231]: pgmap v243: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-04-25T00:29:39.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:38 smithi130 ceph-mon[32525]: pgmap v243: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-04-25T00:29:40.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:40 smithi032 ceph-mon[29357]: pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-25T00:29:40.999 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:40 smithi130 ceph-mon[32525]: pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-25T00:29:41.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:40 smithi121 ceph-mon[32231]: pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-25T00:29:42.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:42 smithi032 ceph-mon[29357]: pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:43.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:42 smithi121 ceph-mon[32231]: pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:43.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:42 smithi130 ceph-mon[32525]: pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:44.861 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:44 smithi130 ceph-mon[32525]: pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:44.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:44 smithi032 ceph-mon[29357]: pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:45.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:44 smithi121 ceph-mon[32231]: pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:45.773 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.11"}]: dispatch 2024-04-25T00:29:45.773 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:45.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.11"}]: dispatch 2024-04-25T00:29:45.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:46.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.11"}]: dispatch 2024-04-25T00:29:46.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:46.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:46 smithi130 ceph-mon[32525]: pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:46.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:46 smithi130 ceph-mon[32525]: Deploying daemon osd.11 on smithi130 2024-04-25T00:29:46.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:46 smithi032 ceph-mon[29357]: pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:46.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:46 smithi032 ceph-mon[29357]: Deploying daemon osd.11 on smithi130 2024-04-25T00:29:47.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:46 smithi121 ceph-mon[32231]: pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:47.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:46 smithi121 ceph-mon[32231]: Deploying daemon osd.11 on smithi130 2024-04-25T00:29:49.008 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:48 smithi130 ceph-mon[32525]: pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:49.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:48 smithi121 ceph-mon[32231]: pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:49.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:48 smithi032 ceph-mon[29357]: pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:50.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:49 smithi121 ceph-mon[32231]: pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:50.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:49 smithi130 ceph-mon[32525]: pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:50.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:49 smithi032 ceph-mon[29357]: pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:52.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:51 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:52.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:51 smithi130 ceph-mon[32525]: pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:52.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:51 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:52.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:51 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:29:52.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:51 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:52.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:51 smithi032 ceph-mon[29357]: pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:52.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:51 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:52.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:51 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:29:52.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:51 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:52.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:51 smithi121 ceph-mon[32231]: pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:52.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:51 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:52.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:51 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:29:54.139 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:53 smithi121 ceph-mon[32231]: pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:54.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:53 smithi130 ceph-mon[32525]: pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:54.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:53 smithi032 ceph-mon[29357]: pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:54.946 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:54.945+0000 7f6da5ffb700 1 -- 172.21.15.130:0/2086418676 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+38 (secure 0 0 0) 0x7f6da8062670 con 0x7f6d980a2c20 2024-04-25T00:29:54.946 INFO:teuthology.orchestra.run.smithi130.stdout:Created osd(s) 11 on host 'smithi130' 2024-04-25T00:29:54.948 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:54.947+0000 7f6daeb14700 1 -- 172.21.15.130:0/2086418676 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f6d980a2c20 msgr2=0x7f6d980a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:29:54.948 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:54.947+0000 7f6daeb14700 1 --2- 172.21.15.130:0/2086418676 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f6d980a2c20 0x7f6d980a50e0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f6d9c005fd0 tx=0x7f6d9c00a560 comp rx=0 tx=0).stop 2024-04-25T00:29:54.948 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:54.947+0000 7f6daeb14700 1 -- 172.21.15.130:0/2086418676 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6da8103ed0 msgr2=0x7f6da819fa50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:29:54.948 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:54.947+0000 7f6daeb14700 1 --2- 172.21.15.130:0/2086418676 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6da8103ed0 0x7f6da819fa50 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f6da000d010 tx=0x7f6da000bf10 comp rx=0 tx=0).stop 2024-04-25T00:29:54.948 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:54.947+0000 7f6daeb14700 1 -- 172.21.15.130:0/2086418676 shutdown_connections 2024-04-25T00:29:54.948 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:54.947+0000 7f6daeb14700 1 --2- 172.21.15.130:0/2086418676 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f6d980a2c20 0x7f6d980a50e0 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:54.948 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:54.947+0000 7f6daeb14700 1 --2- 172.21.15.130:0/2086418676 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6da8103ed0 0x7f6da819fa50 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:54.948 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:54.947+0000 7f6daeb14700 1 --2- 172.21.15.130:0/2086418676 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6da8100550 0x7f6da8198490 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:54.949 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:54.947+0000 7f6daeb14700 1 --2- 172.21.15.130:0/2086418676 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6da8103530 0x7f6da81989d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:54.949 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:54.947+0000 7f6daeb14700 1 -- 172.21.15.130:0/2086418676 >> 172.21.15.130:0/2086418676 conn(0x7f6da80fbe90 msgr2=0x7f6da8106830 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:29:54.949 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:54.947+0000 7f6daeb14700 1 -- 172.21.15.130:0/2086418676 shutdown_connections 2024-04-25T00:29:54.949 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:29:54.947+0000 7f6daeb14700 1 -- 172.21.15.130:0/2086418676 wait complete. 2024-04-25T00:29:55.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:55 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:55.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:55 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:55.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:55 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:55.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:55 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:29:55.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:55 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:55.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:55 smithi130 ceph-mon[32525]: from='osd.11 [v2:172.21.15.130:6824/2386788822,v1:172.21.15.130:6825/2386788822]' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2024-04-25T00:29:55.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:55 smithi130 ceph-mon[32525]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2024-04-25T00:29:55.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:55 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:55.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:55 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:55.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:55 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:29:55.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:55 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:55.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:55 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:55.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:55 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:55.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:55 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:29:55.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:55 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:55.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:55 smithi121 ceph-mon[32231]: from='osd.11 [v2:172.21.15.130:6824/2386788822,v1:172.21.15.130:6825/2386788822]' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2024-04-25T00:29:55.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:55 smithi121 ceph-mon[32231]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2024-04-25T00:29:55.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:55 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:55.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:55 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:55.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:55 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:29:55.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:55 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:55.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:55 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:55.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:55 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:55.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:55 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:29:55.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:55 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:55.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:55 smithi032 ceph-mon[29357]: from='osd.11 [v2:172.21.15.130:6824/2386788822,v1:172.21.15.130:6825/2386788822]' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2024-04-25T00:29:55.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:55 smithi032 ceph-mon[29357]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2024-04-25T00:29:55.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:55 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:55.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:55 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:55.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:55 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:29:55.615 DEBUG:teuthology.orchestra.run.smithi130:osd.11> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.11.service 2024-04-25T00:29:55.617 INFO:tasks.cephadm:Waiting for 12 OSDs to come up... 2024-04-25T00:29:55.617 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-25T00:29:55.877 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:29:56.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:56 smithi130 ceph-mon[32525]: pgmap v253: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:56.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:56 smithi130 ceph-mon[32525]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]': finished 2024-04-25T00:29:56.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:56 smithi130 ceph-mon[32525]: osdmap e78: 12 total, 11 up, 12 in 2024-04-25T00:29:56.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:56 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-25T00:29:56.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:56 smithi130 ceph-mon[32525]: from='osd.11 [v2:172.21.15.130:6824/2386788822,v1:172.21.15.130:6825/2386788822]' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-25T00:29:56.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:56 smithi130 ceph-mon[32525]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-25T00:29:56.269 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:56 smithi032 ceph-mon[29357]: pgmap v253: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:56.270 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:56 smithi032 ceph-mon[29357]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]': finished 2024-04-25T00:29:56.270 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:56 smithi032 ceph-mon[29357]: osdmap e78: 12 total, 11 up, 12 in 2024-04-25T00:29:56.270 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:56 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-25T00:29:56.270 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:56 smithi032 ceph-mon[29357]: from='osd.11 [v2:172.21.15.130:6824/2386788822,v1:172.21.15.130:6825/2386788822]' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-25T00:29:56.270 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:56 smithi032 ceph-mon[29357]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-25T00:29:56.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:56 smithi121 ceph-mon[32231]: pgmap v253: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:56.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:56 smithi121 ceph-mon[32231]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]': finished 2024-04-25T00:29:56.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:56 smithi121 ceph-mon[32231]: osdmap e78: 12 total, 11 up, 12 in 2024-04-25T00:29:56.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:56 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-25T00:29:56.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:56 smithi121 ceph-mon[32231]: from='osd.11 [v2:172.21.15.130:6824/2386788822,v1:172.21.15.130:6825/2386788822]' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-25T00:29:56.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:56 smithi121 ceph-mon[32231]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-25T00:29:56.812 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:29:56 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:29:56.743+0000 7f81e8785700 -1 osd.11 0 waiting for initial osdmap 2024-04-25T00:29:56.812 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:29:56 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:29:56.760+0000 7f81dfd6f700 -1 osd.11 79 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T00:29:57.132 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:57 smithi130 ceph-mon[32525]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-04-25T00:29:57.133 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:57 smithi130 ceph-mon[32525]: osdmap e79: 12 total, 11 up, 12 in 2024-04-25T00:29:57.133 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:57 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-25T00:29:57.133 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:57 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-25T00:29:57.133 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:57 smithi130 ceph-mon[32525]: from='osd.11 ' entity='osd.11' 2024-04-25T00:29:57.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:57 smithi032 ceph-mon[29357]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-04-25T00:29:57.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:57 smithi032 ceph-mon[29357]: osdmap e79: 12 total, 11 up, 12 in 2024-04-25T00:29:57.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:57 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-25T00:29:57.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:57 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-25T00:29:57.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:57 smithi032 ceph-mon[29357]: from='osd.11 ' entity='osd.11' 2024-04-25T00:29:57.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:57 smithi121 ceph-mon[32231]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-04-25T00:29:57.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:57 smithi121 ceph-mon[32231]: osdmap e79: 12 total, 11 up, 12 in 2024-04-25T00:29:57.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:57 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-25T00:29:57.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:57 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-25T00:29:57.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:57 smithi121 ceph-mon[32231]: from='osd.11 ' entity='osd.11' 2024-04-25T00:29:57.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.820+0000 7f5da9ad5700 1 -- 172.21.15.32:0/1690461998 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5da4102d30 msgr2=0x7f5da41095e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:29:57.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.820+0000 7f5da9ad5700 1 --2- 172.21.15.32:0/1690461998 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5da4102d30 0x7f5da41095e0 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f5d9c00b0d0 tx=0x7f5d9c00b3e0 comp rx=0 tx=0).stop 2024-04-25T00:29:57.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.820+0000 7f5da9ad5700 1 -- 172.21.15.32:0/1690461998 shutdown_connections 2024-04-25T00:29:57.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.820+0000 7f5da9ad5700 1 --2- 172.21.15.32:0/1690461998 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5da4102d30 0x7f5da41095e0 unknown :-1 s=CLOSED pgs=143 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:57.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.820+0000 7f5da9ad5700 1 --2- 172.21.15.32:0/1690461998 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5da4101190 0x7f5da41015b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:57.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.820+0000 7f5da9ad5700 1 --2- 172.21.15.32:0/1690461998 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5da4102390 0x7f5da41027f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:57.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.820+0000 7f5da9ad5700 1 -- 172.21.15.32:0/1690461998 >> 172.21.15.32:0/1690461998 conn(0x7f5da40fc730 msgr2=0x7f5da40feb70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:29:57.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.820+0000 7f5da9ad5700 1 -- 172.21.15.32:0/1690461998 shutdown_connections 2024-04-25T00:29:57.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.820+0000 7f5da9ad5700 1 -- 172.21.15.32:0/1690461998 wait complete. 2024-04-25T00:29:57.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.821+0000 7f5da9ad5700 1 Processor -- start 2024-04-25T00:29:57.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.821+0000 7f5da9ad5700 1 -- start start 2024-04-25T00:29:57.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.821+0000 7f5da9ad5700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5da4101190 0x7f5da4105120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:29:57.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.821+0000 7f5da9ad5700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5da4102390 0x7f5da4103770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:29:57.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.821+0000 7f5da9ad5700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5da4102d30 0x7f5da4103cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:29:57.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.821+0000 7f5da9ad5700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f5da410cdc0 con 0x7f5da4101190 2024-04-25T00:29:57.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.821+0000 7f5da9ad5700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f5da410cc40 con 0x7f5da4102d30 2024-04-25T00:29:57.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.821+0000 7f5da9ad5700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f5da410cf40 con 0x7f5da4102390 2024-04-25T00:29:57.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.821+0000 7f5da37fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5da4101190 0x7f5da4105120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:29:57.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.821+0000 7f5da2ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5da4102390 0x7f5da4103770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:29:57.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.821+0000 7f5da37fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5da4101190 0x7f5da4105120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53488/0 (socket says 172.21.15.32:53488) 2024-04-25T00:29:57.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.821+0000 7f5da37fe700 1 -- 172.21.15.32:0/694001823 learned_addr learned my addr 172.21.15.32:0/694001823 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:29:57.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.821+0000 7f5da3fff700 1 --2- 172.21.15.32:0/694001823 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5da4102d30 0x7f5da4103cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:29:57.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.822+0000 7f5da2ffd700 1 -- 172.21.15.32:0/694001823 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5da4102d30 msgr2=0x7f5da4103cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:29:57.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.822+0000 7f5da2ffd700 1 --2- 172.21.15.32:0/694001823 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5da4102d30 0x7f5da4103cb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:57.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.822+0000 7f5da2ffd700 1 -- 172.21.15.32:0/694001823 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5da4101190 msgr2=0x7f5da4105120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:29:57.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.822+0000 7f5da2ffd700 1 --2- 172.21.15.32:0/694001823 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5da4101190 0x7f5da4105120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:57.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.822+0000 7f5da2ffd700 1 -- 172.21.15.32:0/694001823 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5da4104560 con 0x7f5da4102390 2024-04-25T00:29:57.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.822+0000 7f5da37fe700 1 --2- 172.21.15.32:0/694001823 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5da4101190 0x7f5da4105120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:29:57.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.822+0000 7f5da2ffd700 1 --2- 172.21.15.32:0/694001823 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5da4102390 0x7f5da4103770 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f5d9800e9f0 tx=0x7f5d9800ed00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:29:57.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.822+0000 7f5da0ff9700 1 -- 172.21.15.32:0/694001823 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5d9800cbd0 con 0x7f5da4102390 2024-04-25T00:29:57.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.822+0000 7f5da0ff9700 1 -- 172.21.15.32:0/694001823 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5d9800cd70 con 0x7f5da4102390 2024-04-25T00:29:57.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.822+0000 7f5da9ad5700 1 -- 172.21.15.32:0/694001823 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5da41a2680 con 0x7f5da4102390 2024-04-25T00:29:57.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.822+0000 7f5da0ff9700 1 -- 172.21.15.32:0/694001823 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5d98018710 con 0x7f5da4102390 2024-04-25T00:29:57.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.822+0000 7f5da9ad5700 1 -- 172.21.15.32:0/694001823 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5da41a2b90 con 0x7f5da4102390 2024-04-25T00:29:57.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.823+0000 7f5da9ad5700 1 -- 172.21.15.32:0/694001823 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5da4068f20 con 0x7f5da4102390 2024-04-25T00:29:57.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.824+0000 7f5da0ff9700 1 -- 172.21.15.32:0/694001823 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f5d98010430 con 0x7f5da4102390 2024-04-25T00:29:57.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.825+0000 7f5da0ff9700 1 --2- 172.21.15.32:0/694001823 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f5d8c0a2c20 0x7f5d8c0a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:29:57.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.825+0000 7f5da0ff9700 1 -- 172.21.15.32:0/694001823 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 7127+0+0 (secure 0 0 0) 0x7f5d980cf130 con 0x7f5da4102390 2024-04-25T00:29:57.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.825+0000 7f5da37fe700 1 --2- 172.21.15.32:0/694001823 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f5d8c0a2c20 0x7f5d8c0a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:29:57.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.825+0000 7f5da37fe700 1 --2- 172.21.15.32:0/694001823 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f5d8c0a2c20 0x7f5d8c0a50e0 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f5d940105c0 tx=0x7f5d9400b4a0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:29:57.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:57.829+0000 7f5da0ff9700 1 -- 172.21.15.32:0/694001823 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f5d9809c530 con 0x7f5da4102390 2024-04-25T00:29:58.036 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:58.036+0000 7f5da9ad5700 1 -- 172.21.15.32:0/694001823 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd stat", "format": "json"} v 0) -- 0x7f5da41a2fb0 con 0x7f5da4102390 2024-04-25T00:29:58.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:58.036+0000 7f5da0ff9700 1 -- 172.21.15.32:0/694001823 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd stat", "format": "json"}]=0 v80) ==== 74+0+133 (secure 0 0 0) 0x7f5d9809be30 con 0x7f5da4102390 2024-04-25T00:29:58.037 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:29:58.039 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:58.038+0000 7f5da9ad5700 1 -- 172.21.15.32:0/694001823 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f5d8c0a2c20 msgr2=0x7f5d8c0a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:29:58.039 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:58.038+0000 7f5da9ad5700 1 --2- 172.21.15.32:0/694001823 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f5d8c0a2c20 0x7f5d8c0a50e0 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f5d940105c0 tx=0x7f5d9400b4a0 comp rx=0 tx=0).stop 2024-04-25T00:29:58.039 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:58.039+0000 7f5da9ad5700 1 -- 172.21.15.32:0/694001823 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5da4102390 msgr2=0x7f5da4103770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:29:58.039 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:58.039+0000 7f5da9ad5700 1 --2- 172.21.15.32:0/694001823 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5da4102390 0x7f5da4103770 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f5d9800e9f0 tx=0x7f5d9800ed00 comp rx=0 tx=0).stop 2024-04-25T00:29:58.039 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:58.039+0000 7f5da9ad5700 1 -- 172.21.15.32:0/694001823 shutdown_connections 2024-04-25T00:29:58.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:58.039+0000 7f5da9ad5700 1 --2- 172.21.15.32:0/694001823 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f5d8c0a2c20 0x7f5d8c0a50e0 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:58.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:58.039+0000 7f5da9ad5700 1 --2- 172.21.15.32:0/694001823 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5da4102d30 0x7f5da4103cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:58.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:58.039+0000 7f5da9ad5700 1 --2- 172.21.15.32:0/694001823 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5da4101190 0x7f5da4105120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:58.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:58.039+0000 7f5da9ad5700 1 --2- 172.21.15.32:0/694001823 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5da4102390 0x7f5da4103770 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:29:58.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:58.039+0000 7f5da9ad5700 1 -- 172.21.15.32:0/694001823 >> 172.21.15.32:0/694001823 conn(0x7f5da40fc730 msgr2=0x7f5da40fe180 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:29:58.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:58.039+0000 7f5da9ad5700 1 -- 172.21.15.32:0/694001823 shutdown_connections 2024-04-25T00:29:58.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:29:58.039+0000 7f5da9ad5700 1 -- 172.21.15.32:0/694001823 wait complete. 2024-04-25T00:29:58.278 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:58 smithi130 ceph-mon[32525]: purged_snaps scrub starts 2024-04-25T00:29:58.278 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:58 smithi130 ceph-mon[32525]: purged_snaps scrub ok 2024-04-25T00:29:58.278 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:58 smithi130 ceph-mon[32525]: pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:58.278 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:58 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-25T00:29:58.278 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:58 smithi130 ceph-mon[32525]: osd.11 [v2:172.21.15.130:6824/2386788822,v1:172.21.15.130:6825/2386788822] boot 2024-04-25T00:29:58.278 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:58 smithi130 ceph-mon[32525]: osdmap e80: 12 total, 12 up, 12 in 2024-04-25T00:29:58.279 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:58 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-25T00:29:58.279 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:58 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/694001823' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-25T00:29:58.330 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:58 smithi032 ceph-mon[29357]: purged_snaps scrub starts 2024-04-25T00:29:58.331 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:58 smithi032 ceph-mon[29357]: purged_snaps scrub ok 2024-04-25T00:29:58.331 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:58 smithi032 ceph-mon[29357]: pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:58.331 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:58 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-25T00:29:58.331 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:58 smithi032 ceph-mon[29357]: osd.11 [v2:172.21.15.130:6824/2386788822,v1:172.21.15.130:6825/2386788822] boot 2024-04-25T00:29:58.331 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:58 smithi032 ceph-mon[29357]: osdmap e80: 12 total, 12 up, 12 in 2024-04-25T00:29:58.331 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:58 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-25T00:29:58.331 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:58 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/694001823' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-25T00:29:58.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:58 smithi121 ceph-mon[32231]: purged_snaps scrub starts 2024-04-25T00:29:58.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:58 smithi121 ceph-mon[32231]: purged_snaps scrub ok 2024-04-25T00:29:58.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:58 smithi121 ceph-mon[32231]: pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 333 MiB used, 983 GiB / 983 GiB avail 2024-04-25T00:29:58.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:58 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-25T00:29:58.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:58 smithi121 ceph-mon[32231]: osd.11 [v2:172.21.15.130:6824/2386788822,v1:172.21.15.130:6825/2386788822] boot 2024-04-25T00:29:58.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:58 smithi121 ceph-mon[32231]: osdmap e80: 12 total, 12 up, 12 in 2024-04-25T00:29:58.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:58 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-25T00:29:58.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:58 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/694001823' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-25T00:29:58.847 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":80,"num_osds":12,"num_up_osds":12,"osd_up_since":1714004997,"num_in_osds":12,"osd_in_since":1714004977,"num_remapped_pgs":0} 2024-04-25T00:29:58.847 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-25T00:29:59.101 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:29:59.970 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:59 smithi032 ceph-mon[29357]: Detected new or changed devices on smithi130 2024-04-25T00:29:59.970 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:59 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:59.970 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:59 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:59.970 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:59 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:59.970 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:59 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:59.970 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:59 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:59.970 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:59 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:29:59.971 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:59 smithi032 ceph-mon[29357]: Adjusting osd_memory_target on smithi130 to 4272M 2024-04-25T00:29:59.971 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:59 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:29:59.971 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:59 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:29:59.971 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:59 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:29:59.971 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:29:59 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:00.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:59 smithi121 ceph-mon[32231]: Detected new or changed devices on smithi130 2024-04-25T00:30:00.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:59 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:00.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:59 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:00.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:59 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:30:00.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:59 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:30:00.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:59 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:30:00.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:59 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:30:00.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:59 smithi121 ceph-mon[32231]: Adjusting osd_memory_target on smithi130 to 4272M 2024-04-25T00:30:00.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:59 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:00.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:59 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:00.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:59 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:30:00.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:29:59 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:00.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:59 smithi130 ceph-mon[32525]: Detected new or changed devices on smithi130 2024-04-25T00:30:00.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:59 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:00.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:59 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:00.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:59 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:30:00.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:59 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:30:00.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:59 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:30:00.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:59 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:30:00.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:59 smithi130 ceph-mon[32525]: Adjusting osd_memory_target on smithi130 to 4272M 2024-04-25T00:30:00.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:59 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:00.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:59 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:00.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:59 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:30:00.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:29:59 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:00.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.831+0000 7f083f285700 1 -- 172.21.15.32:0/484762902 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f08381008d0 msgr2=0x7f0838100d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:00.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.831+0000 7f083f285700 1 --2- 172.21.15.32:0/484762902 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f08381008d0 0x7f0838100d50 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f0828009990 tx=0x7f0828009ca0 comp rx=0 tx=0).stop 2024-04-25T00:30:00.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.831+0000 7f083f285700 1 -- 172.21.15.32:0/484762902 shutdown_connections 2024-04-25T00:30:00.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.831+0000 7f083f285700 1 --2- 172.21.15.32:0/484762902 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0838101290 0x7f083810d7a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:00.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.831+0000 7f083f285700 1 --2- 172.21.15.32:0/484762902 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f08381068d0 0x7f0838106cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:00.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.831+0000 7f083f285700 1 --2- 172.21.15.32:0/484762902 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f08381008d0 0x7f0838100d50 unknown :-1 s=CLOSED pgs=144 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:00.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.831+0000 7f083f285700 1 -- 172.21.15.32:0/484762902 >> 172.21.15.32:0/484762902 conn(0x7f0838077730 msgr2=0x7f08380feab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:00.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.831+0000 7f083f285700 1 -- 172.21.15.32:0/484762902 shutdown_connections 2024-04-25T00:30:00.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.831+0000 7f083f285700 1 -- 172.21.15.32:0/484762902 wait complete. 2024-04-25T00:30:00.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.831+0000 7f083f285700 1 Processor -- start 2024-04-25T00:30:00.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.831+0000 7f083f285700 1 -- start start 2024-04-25T00:30:00.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.832+0000 7f083f285700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f08381008d0 0x7f083819ba90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:00.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.832+0000 7f083f285700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0838101290 0x7f083819bfd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:00.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.832+0000 7f083f285700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f08381068d0 0x7f08381a6940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:00.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.832+0000 7f083f285700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f083810f2d0 con 0x7f0838101290 2024-04-25T00:30:00.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.832+0000 7f083f285700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f083810f150 con 0x7f08381008d0 2024-04-25T00:30:00.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.832+0000 7f083f285700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f0838110590 con 0x7f08381068d0 2024-04-25T00:30:00.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.832+0000 7f083c820700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0838101290 0x7f083819bfd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:00.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.832+0000 7f083c820700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0838101290 0x7f083819bfd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53506/0 (socket says 172.21.15.32:53506) 2024-04-25T00:30:00.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.832+0000 7f083c820700 1 -- 172.21.15.32:0/2095692155 learned_addr learned my addr 172.21.15.32:0/2095692155 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:00.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.832+0000 7f083d021700 1 --2- 172.21.15.32:0/2095692155 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f08381008d0 0x7f083819ba90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:00.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.832+0000 7f083d822700 1 --2- 172.21.15.32:0/2095692155 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f08381068d0 0x7f08381a6940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:00.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.832+0000 7f083c820700 1 -- 172.21.15.32:0/2095692155 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f08381068d0 msgr2=0x7f08381a6940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:00.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.832+0000 7f083c820700 1 --2- 172.21.15.32:0/2095692155 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f08381068d0 0x7f08381a6940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:00.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.832+0000 7f083c820700 1 -- 172.21.15.32:0/2095692155 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f08381008d0 msgr2=0x7f083819ba90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:00.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.832+0000 7f083c820700 1 --2- 172.21.15.32:0/2095692155 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f08381008d0 0x7f083819ba90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:00.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.832+0000 7f083c820700 1 -- 172.21.15.32:0/2095692155 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f08381a6f00 con 0x7f0838101290 2024-04-25T00:30:00.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.832+0000 7f083d822700 1 --2- 172.21.15.32:0/2095692155 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f08381068d0 0x7f08381a6940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:30:00.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.832+0000 7f083c820700 1 --2- 172.21.15.32:0/2095692155 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0838101290 0x7f083819bfd0 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f0828005910 tx=0x7f08280047b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:00.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.833+0000 7f082e7fc700 1 -- 172.21.15.32:0/2095692155 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0828027070 con 0x7f0838101290 2024-04-25T00:30:00.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.833+0000 7f083f285700 1 -- 172.21.15.32:0/2095692155 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f08381a7190 con 0x7f0838101290 2024-04-25T00:30:00.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.833+0000 7f082e7fc700 1 -- 172.21.15.32:0/2095692155 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0828004050 con 0x7f0838101290 2024-04-25T00:30:00.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.833+0000 7f082e7fc700 1 -- 172.21.15.32:0/2095692155 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f082800f460 con 0x7f0838101290 2024-04-25T00:30:00.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.833+0000 7f083f285700 1 -- 172.21.15.32:0/2095692155 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f08381a75e0 con 0x7f0838101290 2024-04-25T00:30:00.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.834+0000 7f083f285700 1 -- 172.21.15.32:0/2095692155 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0838068f20 con 0x7f0838101290 2024-04-25T00:30:00.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.835+0000 7f082e7fc700 1 -- 172.21.15.32:0/2095692155 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f082802a070 con 0x7f0838101290 2024-04-25T00:30:00.837 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.835+0000 7f082e7fc700 1 --2- 172.21.15.32:0/2095692155 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f08240a2c20 0x7f08240a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:00.837 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.836+0000 7f082e7fc700 1 -- 172.21.15.32:0/2095692155 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7f08280d01f0 con 0x7f0838101290 2024-04-25T00:30:00.837 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.836+0000 7f083d021700 1 --2- 172.21.15.32:0/2095692155 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f08240a2c20 0x7f08240a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:00.837 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.836+0000 7f083d021700 1 --2- 172.21.15.32:0/2095692155 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f08240a2c20 0x7f08240a50e0 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f0834005950 tx=0x7f083400a300 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:00.839 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:00.839+0000 7f082e7fc700 1 -- 172.21.15.32:0/2095692155 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f0828031070 con 0x7f0838101290 2024-04-25T00:30:00.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:00 smithi032 ceph-mon[29357]: pgmap v258: 1 pgs: 1 active+clean; 577 KiB data, 364 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:00.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:00 smithi032 ceph-mon[29357]: osdmap e81: 12 total, 12 up, 12 in 2024-04-25T00:30:00.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:00 smithi032 ceph-mon[29357]: overall HEALTH_OK 2024-04-25T00:30:01.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:01.050+0000 7f083f285700 1 -- 172.21.15.32:0/2095692155 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f08381a7940 con 0x7f0838101290 2024-04-25T00:30:01.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:01.051+0000 7f082e7fc700 1 -- 172.21.15.32:0/2095692155 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v81) ==== 74+0+18456 (secure 0 0 0) 0x7f0828013070 con 0x7f0838101290 2024-04-25T00:30:01.052 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:01.052 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":81,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:29:59.700603+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:30:01.054 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:01.053+0000 7f083f285700 1 -- 172.21.15.32:0/2095692155 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f08240a2c20 msgr2=0x7f08240a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:01.054 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:01.053+0000 7f083f285700 1 --2- 172.21.15.32:0/2095692155 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f08240a2c20 0x7f08240a50e0 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f0834005950 tx=0x7f083400a300 comp rx=0 tx=0).stop 2024-04-25T00:30:01.054 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:01.053+0000 7f083f285700 1 -- 172.21.15.32:0/2095692155 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0838101290 msgr2=0x7f083819bfd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:01.054 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:01.053+0000 7f083f285700 1 --2- 172.21.15.32:0/2095692155 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0838101290 0x7f083819bfd0 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f0828005910 tx=0x7f08280047b0 comp rx=0 tx=0).stop 2024-04-25T00:30:01.054 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:01.053+0000 7f083f285700 1 -- 172.21.15.32:0/2095692155 shutdown_connections 2024-04-25T00:30:01.054 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:01.053+0000 7f083f285700 1 --2- 172.21.15.32:0/2095692155 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f08240a2c20 0x7f08240a50e0 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:01.054 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:01.053+0000 7f083f285700 1 --2- 172.21.15.32:0/2095692155 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f08381068d0 0x7f08381a6940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:01.054 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:01.053+0000 7f083f285700 1 --2- 172.21.15.32:0/2095692155 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f08381008d0 0x7f083819ba90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:01.054 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:01.053+0000 7f083f285700 1 --2- 172.21.15.32:0/2095692155 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0838101290 0x7f083819bfd0 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:01.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:01.053+0000 7f083f285700 1 -- 172.21.15.32:0/2095692155 >> 172.21.15.32:0/2095692155 conn(0x7f0838077730 msgr2=0x7f0838104920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:01.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:01.053+0000 7f083f285700 1 -- 172.21.15.32:0/2095692155 shutdown_connections 2024-04-25T00:30:01.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:01.053+0000 7f083f285700 1 -- 172.21.15.32:0/2095692155 wait complete. 2024-04-25T00:30:01.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:00 smithi121 ceph-mon[32231]: pgmap v258: 1 pgs: 1 active+clean; 577 KiB data, 364 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:01.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:00 smithi121 ceph-mon[32231]: osdmap e81: 12 total, 12 up, 12 in 2024-04-25T00:30:01.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:00 smithi121 ceph-mon[32231]: overall HEALTH_OK 2024-04-25T00:30:01.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:00 smithi130 ceph-mon[32525]: pgmap v258: 1 pgs: 1 active+clean; 577 KiB data, 364 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:01.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:00 smithi130 ceph-mon[32525]: osdmap e81: 12 total, 12 up, 12 in 2024-04-25T00:30:01.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:00 smithi130 ceph-mon[32525]: overall HEALTH_OK 2024-04-25T00:30:01.653 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-25T00:26:07.306839+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-04-25T00:30:01.653 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-04-25T00:30:01.904 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:30:01.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:01 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2095692155' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:30:02.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:01 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2095692155' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:30:02.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:01 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2095692155' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:30:02.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:02 smithi032 ceph-mon[29357]: pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:03.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:02 smithi121 ceph-mon[32231]: pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:03.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:02 smithi130 ceph-mon[32525]: pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:03.707 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.706+0000 7f5d19071700 1 -- 172.21.15.32:0/178038439 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5d141013c0 msgr2=0x7f5d141017a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:03.707 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.706+0000 7f5d19071700 1 --2- 172.21.15.32:0/178038439 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5d141013c0 0x7f5d141017a0 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f5d040099c0 tx=0x7f5d04009cd0 comp rx=0 tx=0).stop 2024-04-25T00:30:03.707 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.706+0000 7f5d19071700 1 -- 172.21.15.32:0/178038439 shutdown_connections 2024-04-25T00:30:03.707 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.706+0000 7f5d19071700 1 --2- 172.21.15.32:0/178038439 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5d14106450 0x7f5d1410e020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:03.707 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.706+0000 7f5d19071700 1 --2- 172.21.15.32:0/178038439 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5d141013c0 0x7f5d141017a0 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:03.708 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.706+0000 7f5d19071700 1 --2- 172.21.15.32:0/178038439 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5d14101d70 0x7f5d14105d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:03.708 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.706+0000 7f5d19071700 1 -- 172.21.15.32:0/178038439 >> 172.21.15.32:0/178038439 conn(0x7f5d140780e0 msgr2=0x7f5d140784f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:03.708 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.706+0000 7f5d19071700 1 -- 172.21.15.32:0/178038439 shutdown_connections 2024-04-25T00:30:03.708 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.706+0000 7f5d19071700 1 -- 172.21.15.32:0/178038439 wait complete. 2024-04-25T00:30:03.710 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.707+0000 7f5d19071700 1 Processor -- start 2024-04-25T00:30:03.710 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.707+0000 7f5d19071700 1 -- start start 2024-04-25T00:30:03.710 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.707+0000 7f5d19071700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5d141013c0 0x7f5d14199890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:03.710 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.707+0000 7f5d19071700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5d14101d70 0x7f5d14199dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:03.710 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.707+0000 7f5d19071700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5d14106450 0x7f5d141a4910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:03.710 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.707+0000 7f5d19071700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f5d14111430 con 0x7f5d14106450 2024-04-25T00:30:03.710 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.707+0000 7f5d19071700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f5d141112b0 con 0x7f5d141013c0 2024-04-25T00:30:03.711 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.707+0000 7f5d19071700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f5d141115b0 con 0x7f5d14101d70 2024-04-25T00:30:03.711 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.707+0000 7f5d1359e700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5d14106450 0x7f5d141a4910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:03.711 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.707+0000 7f5d1359e700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5d14106450 0x7f5d141a4910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:60062/0 (socket says 172.21.15.32:60062) 2024-04-25T00:30:03.711 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.707+0000 7f5d1359e700 1 -- 172.21.15.32:0/517171145 learned_addr learned my addr 172.21.15.32:0/517171145 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:03.711 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.707+0000 7f5d1259c700 1 --2- 172.21.15.32:0/517171145 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5d14101d70 0x7f5d14199dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:03.711 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.708+0000 7f5d12d9d700 1 --2- 172.21.15.32:0/517171145 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5d141013c0 0x7f5d14199890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:03.711 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.708+0000 7f5d1359e700 1 -- 172.21.15.32:0/517171145 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5d14101d70 msgr2=0x7f5d14199dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:03.711 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.708+0000 7f5d1359e700 1 --2- 172.21.15.32:0/517171145 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5d14101d70 0x7f5d14199dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:03.711 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.708+0000 7f5d1359e700 1 -- 172.21.15.32:0/517171145 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5d141013c0 msgr2=0x7f5d14199890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:03.711 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.708+0000 7f5d1359e700 1 --2- 172.21.15.32:0/517171145 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5d141013c0 0x7f5d14199890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:03.711 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.708+0000 7f5d1359e700 1 -- 172.21.15.32:0/517171145 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5d141a4e50 con 0x7f5d14106450 2024-04-25T00:30:03.712 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.708+0000 7f5d1359e700 1 --2- 172.21.15.32:0/517171145 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5d14106450 0x7f5d141a4910 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7f5d0c007ee0 tx=0x7f5d0c00d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:03.712 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.708+0000 7f5cfbfff700 1 -- 172.21.15.32:0/517171145 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5d0c00dd20 con 0x7f5d14106450 2024-04-25T00:30:03.712 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.708+0000 7f5d19071700 1 -- 172.21.15.32:0/517171145 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5d141a5020 con 0x7f5d14106450 2024-04-25T00:30:03.712 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.708+0000 7f5cfbfff700 1 -- 172.21.15.32:0/517171145 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5d0c004020 con 0x7f5d14106450 2024-04-25T00:30:03.712 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.708+0000 7f5d19071700 1 -- 172.21.15.32:0/517171145 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5d141a5560 con 0x7f5d14106450 2024-04-25T00:30:03.712 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.708+0000 7f5cfbfff700 1 -- 172.21.15.32:0/517171145 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5d0c015660 con 0x7f5d14106450 2024-04-25T00:30:03.712 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.709+0000 7f5d19071700 1 -- 172.21.15.32:0/517171145 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5d14068f20 con 0x7f5d14106450 2024-04-25T00:30:03.713 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.711+0000 7f5cfbfff700 1 -- 172.21.15.32:0/517171145 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f5d0c0041d0 con 0x7f5d14106450 2024-04-25T00:30:03.713 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.711+0000 7f5cfbfff700 1 --2- 172.21.15.32:0/517171145 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f5cfc0a2c20 0x7f5cfc0a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:03.713 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.711+0000 7f5d12d9d700 1 --2- 172.21.15.32:0/517171145 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f5cfc0a2c20 0x7f5cfc0a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:03.713 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.711+0000 7f5cfbfff700 1 -- 172.21.15.32:0/517171145 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7f5d0c0cedd0 con 0x7f5d14106450 2024-04-25T00:30:03.713 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.712+0000 7f5d12d9d700 1 --2- 172.21.15.32:0/517171145 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f5cfc0a2c20 0x7f5cfc0a50e0 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f5d04000c00 tx=0x7f5d04005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:03.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.715+0000 7f5cfbfff700 1 -- 172.21.15.32:0/517171145 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f5d0c010070 con 0x7f5d14106450 2024-04-25T00:30:03.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.918+0000 7f5d19071700 1 -- 172.21.15.32:0/517171145 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7f5d1406af20 con 0x7f5d14106450 2024-04-25T00:30:03.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.919+0000 7f5cfbfff700 1 -- 172.21.15.32:0/517171145 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v81) ==== 93+0+10 (secure 0 0 0) 0x7f5d0c09c1d0 con 0x7f5d14106450 2024-04-25T00:30:03.920 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 1 2024-04-25T00:30:03.922 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.921+0000 7f5d19071700 1 -- 172.21.15.32:0/517171145 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f5cfc0a2c20 msgr2=0x7f5cfc0a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:03.922 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.921+0000 7f5d19071700 1 --2- 172.21.15.32:0/517171145 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f5cfc0a2c20 0x7f5cfc0a50e0 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f5d04000c00 tx=0x7f5d04005fb0 comp rx=0 tx=0).stop 2024-04-25T00:30:03.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.921+0000 7f5d19071700 1 -- 172.21.15.32:0/517171145 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5d14106450 msgr2=0x7f5d141a4910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:03.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.921+0000 7f5d19071700 1 --2- 172.21.15.32:0/517171145 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5d14106450 0x7f5d141a4910 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7f5d0c007ee0 tx=0x7f5d0c00d3b0 comp rx=0 tx=0).stop 2024-04-25T00:30:03.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.921+0000 7f5d19071700 1 -- 172.21.15.32:0/517171145 shutdown_connections 2024-04-25T00:30:03.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.921+0000 7f5d19071700 1 --2- 172.21.15.32:0/517171145 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f5cfc0a2c20 0x7f5cfc0a50e0 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:03.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.921+0000 7f5d19071700 1 --2- 172.21.15.32:0/517171145 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5d14106450 0x7f5d141a4910 unknown :-1 s=CLOSED pgs=147 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:03.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.921+0000 7f5d19071700 1 --2- 172.21.15.32:0/517171145 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5d141013c0 0x7f5d14199890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:03.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.921+0000 7f5d19071700 1 --2- 172.21.15.32:0/517171145 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5d14101d70 0x7f5d14199dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:03.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.921+0000 7f5d19071700 1 -- 172.21.15.32:0/517171145 >> 172.21.15.32:0/517171145 conn(0x7f5d140780e0 msgr2=0x7f5d140ff410 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:03.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.921+0000 7f5d19071700 1 -- 172.21.15.32:0/517171145 shutdown_connections 2024-04-25T00:30:03.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:03.922+0000 7f5d19071700 1 -- 172.21.15.32:0/517171145 wait complete. 2024-04-25T00:30:04.731 INFO:tasks.cephadm:Adding mds.a on smithi032 2024-04-25T00:30:04.731 INFO:tasks.cephadm:Adding mds.d on smithi032 2024-04-25T00:30:04.731 INFO:tasks.cephadm:Adding mds.g on smithi032 2024-04-25T00:30:04.731 INFO:tasks.cephadm:Adding mds.j on smithi032 2024-04-25T00:30:04.731 INFO:tasks.cephadm:Adding mds.b on smithi121 2024-04-25T00:30:04.732 INFO:tasks.cephadm:Adding mds.e on smithi121 2024-04-25T00:30:04.732 INFO:tasks.cephadm:Adding mds.h on smithi121 2024-04-25T00:30:04.732 INFO:tasks.cephadm:Adding mds.k on smithi121 2024-04-25T00:30:04.732 INFO:tasks.cephadm:Adding mds.c on smithi130 2024-04-25T00:30:04.732 INFO:tasks.cephadm:Adding mds.f on smithi130 2024-04-25T00:30:04.732 INFO:tasks.cephadm:Adding mds.i on smithi130 2024-04-25T00:30:04.732 INFO:tasks.cephadm:Adding mds.l on smithi130 2024-04-25T00:30:04.732 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph orch apply mds all '12;smithi032=a;smithi032=d;smithi032=g;smithi032=j;smithi121=b;smithi121=e;smithi121=h;smithi121=k;smithi130=c;smithi130=f;smithi130=i;smithi130=l' 2024-04-25T00:30:04.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:04 smithi130 ceph-mon[32525]: pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:04.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:04 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/517171145' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:30:04.976 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.c/config 2024-04-25T00:30:04.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:04 smithi032 ceph-mon[29357]: pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:04.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:04 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/517171145' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:30:05.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:04 smithi121 ceph-mon[32231]: pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:05.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:04 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/517171145' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:30:06.663 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.662+0000 7f145d24f700 1 -- 172.21.15.130:0/3890728011 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f14581011b0 msgr2=0x7f14581015d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:06.664 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.662+0000 7f145d24f700 1 --2- 172.21.15.130:0/3890728011 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f14581011b0 0x7f14581015d0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f1448009990 tx=0x7f1448009ca0 comp rx=0 tx=0).stop 2024-04-25T00:30:06.664 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.662+0000 7f145d24f700 1 -- 172.21.15.130:0/3890728011 shutdown_connections 2024-04-25T00:30:06.664 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.662+0000 7f145d24f700 1 --2- 172.21.15.130:0/3890728011 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1458102d50 0x7f1458109600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:06.664 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.662+0000 7f145d24f700 1 --2- 172.21.15.130:0/3890728011 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f14581011b0 0x7f14581015d0 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:06.664 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.662+0000 7f145d24f700 1 --2- 172.21.15.130:0/3890728011 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f14581023b0 0x7f1458102810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:06.664 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.662+0000 7f145d24f700 1 -- 172.21.15.130:0/3890728011 >> 172.21.15.130:0/3890728011 conn(0x7f14580fc790 msgr2=0x7f14580febb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:06.664 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.662+0000 7f145d24f700 1 -- 172.21.15.130:0/3890728011 shutdown_connections 2024-04-25T00:30:06.664 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.662+0000 7f145d24f700 1 -- 172.21.15.130:0/3890728011 wait complete. 2024-04-25T00:30:06.664 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.663+0000 7f145d24f700 1 Processor -- start 2024-04-25T00:30:06.666 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.663+0000 7f145d24f700 1 -- start start 2024-04-25T00:30:06.666 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.663+0000 7f145d24f700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f14581011b0 0x7f1458197420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:06.666 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.663+0000 7f145d24f700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f14581023b0 0x7f1458197960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:06.666 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.663+0000 7f145d24f700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1458102d50 0x7f145819e9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:06.666 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.663+0000 7f145d24f700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f145810ce00 con 0x7f14581011b0 2024-04-25T00:30:06.666 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.663+0000 7f145d24f700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f145810cc80 con 0x7f14581023b0 2024-04-25T00:30:06.666 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.663+0000 7f145d24f700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f145810cf80 con 0x7f1458102d50 2024-04-25T00:30:06.667 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.664+0000 7f145759e700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1458102d50 0x7f145819e9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:06.667 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.664+0000 7f1456d9d700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f14581011b0 0x7f1458197420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:06.667 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.664+0000 7f145759e700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1458102d50 0x7f145819e9e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.130:46704/0 (socket says 172.21.15.130:46704) 2024-04-25T00:30:06.667 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.664+0000 7f1456d9d700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f14581011b0 0x7f1458197420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.130:46648/0 (socket says 172.21.15.130:46648) 2024-04-25T00:30:06.667 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.664+0000 7f145759e700 1 -- 172.21.15.130:0/601409682 learned_addr learned my addr 172.21.15.130:0/601409682 (peer_addr_for_me v2:172.21.15.130:0/0) 2024-04-25T00:30:06.667 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.664+0000 7f145659c700 1 --2- 172.21.15.130:0/601409682 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f14581023b0 0x7f1458197960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:06.667 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.664+0000 7f145759e700 1 -- 172.21.15.130:0/601409682 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f14581023b0 msgr2=0x7f1458197960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:06.667 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.664+0000 7f145759e700 1 --2- 172.21.15.130:0/601409682 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f14581023b0 0x7f1458197960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:06.667 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.664+0000 7f145759e700 1 -- 172.21.15.130:0/601409682 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f14581011b0 msgr2=0x7f1458197420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:06.667 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.664+0000 7f145759e700 1 --2- 172.21.15.130:0/601409682 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f14581011b0 0x7f1458197420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:06.667 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.664+0000 7f145759e700 1 -- 172.21.15.130:0/601409682 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f145819f080 con 0x7f1458102d50 2024-04-25T00:30:06.667 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.664+0000 7f145659c700 1 --2- 172.21.15.130:0/601409682 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f14581023b0 0x7f1458197960 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:30:06.667 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.664+0000 7f145759e700 1 --2- 172.21.15.130:0/601409682 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1458102d50 0x7f145819e9e0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f1450007f40 tx=0x7f145000f5c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:06.668 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.664+0000 7f143ffff700 1 -- 172.21.15.130:0/601409682 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1450016030 con 0x7f1458102d50 2024-04-25T00:30:06.668 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.664+0000 7f143ffff700 1 -- 172.21.15.130:0/601409682 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f145000fde0 con 0x7f1458102d50 2024-04-25T00:30:06.668 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.664+0000 7f145d24f700 1 -- 172.21.15.130:0/601409682 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f145806a9a0 con 0x7f1458102d50 2024-04-25T00:30:06.668 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.664+0000 7f143ffff700 1 -- 172.21.15.130:0/601409682 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1450005020 con 0x7f1458102d50 2024-04-25T00:30:06.668 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.664+0000 7f145d24f700 1 -- 172.21.15.130:0/601409682 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f145806af10 con 0x7f1458102d50 2024-04-25T00:30:06.668 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.665+0000 7f145d24f700 1 -- 172.21.15.130:0/601409682 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1458068f20 con 0x7f1458102d50 2024-04-25T00:30:06.669 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.666+0000 7f143ffff700 1 -- 172.21.15.130:0/601409682 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f1450012460 con 0x7f1458102d50 2024-04-25T00:30:06.669 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.667+0000 7f143ffff700 1 --2- 172.21.15.130:0/601409682 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f14400a2c20 0x7f14400a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:06.669 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.667+0000 7f143ffff700 1 -- 172.21.15.130:0/601409682 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7f14500d0e40 con 0x7f1458102d50 2024-04-25T00:30:06.669 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.667+0000 7f1456d9d700 1 --2- 172.21.15.130:0/601409682 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f14400a2c20 0x7f14400a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:06.669 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.668+0000 7f1456d9d700 1 --2- 172.21.15.130:0/601409682 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f14400a2c20 0x7f14400a50e0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f1448003890 tx=0x7f1448005fd0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:06.672 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.671+0000 7f143ffff700 1 -- 172.21.15.130:0/601409682 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f145009e240 con 0x7f1458102d50 2024-04-25T00:30:06.879 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.878+0000 7f145d24f700 1 -- 172.21.15.130:0/601409682 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch apply mds", "fs_name": "all", "placement": "12;smithi032=a;smithi032=d;smithi032=g;smithi032=j;smithi121=b;smithi121=e;smithi121=h;smithi121=k;smithi130=c;smithi130=f;smithi130=i;smithi130=l", "target": ["mon-mgr", ""]}) -- 0x7f145806b430 con 0x7f14400a2c20 2024-04-25T00:30:06.887 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.886+0000 7f143ffff700 1 -- 172.21.15.130:0/601409682 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+28 (secure 0 0 0) 0x7f145806b430 con 0x7f14400a2c20 2024-04-25T00:30:06.887 INFO:teuthology.orchestra.run.smithi130.stdout:Scheduled mds.all update... 2024-04-25T00:30:06.889 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.888+0000 7f145d24f700 1 -- 172.21.15.130:0/601409682 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f14400a2c20 msgr2=0x7f14400a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:06.889 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.888+0000 7f145d24f700 1 --2- 172.21.15.130:0/601409682 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f14400a2c20 0x7f14400a50e0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f1448003890 tx=0x7f1448005fd0 comp rx=0 tx=0).stop 2024-04-25T00:30:06.889 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.888+0000 7f145d24f700 1 -- 172.21.15.130:0/601409682 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1458102d50 msgr2=0x7f145819e9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:06.889 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.888+0000 7f145d24f700 1 --2- 172.21.15.130:0/601409682 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1458102d50 0x7f145819e9e0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f1450007f40 tx=0x7f145000f5c0 comp rx=0 tx=0).stop 2024-04-25T00:30:06.889 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.889+0000 7f145d24f700 1 -- 172.21.15.130:0/601409682 shutdown_connections 2024-04-25T00:30:06.890 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.889+0000 7f145d24f700 1 --2- 172.21.15.130:0/601409682 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f14400a2c20 0x7f14400a50e0 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:06.890 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.889+0000 7f145d24f700 1 --2- 172.21.15.130:0/601409682 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1458102d50 0x7f145819e9e0 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:06.890 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.889+0000 7f145d24f700 1 --2- 172.21.15.130:0/601409682 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f14581011b0 0x7f1458197420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:06.890 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.889+0000 7f145d24f700 1 --2- 172.21.15.130:0/601409682 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f14581023b0 0x7f1458197960 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:06.890 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.889+0000 7f145d24f700 1 -- 172.21.15.130:0/601409682 >> 172.21.15.130:0/601409682 conn(0x7f14580fc790 msgr2=0x7f1458105780 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:06.890 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.889+0000 7f145d24f700 1 -- 172.21.15.130:0/601409682 shutdown_connections 2024-04-25T00:30:06.890 INFO:teuthology.orchestra.run.smithi130.stderr:2024-04-25T00:30:06.889+0000 7f145d24f700 1 -- 172.21.15.130:0/601409682 wait complete. 2024-04-25T00:30:06.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:06 smithi032 ceph-mon[29357]: pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:07.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:06 smithi121 ceph-mon[32231]: pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:07.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:06 smithi130 ceph-mon[32525]: pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:07.549 DEBUG:teuthology.orchestra.run.smithi032:mds.a> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.a.service 2024-04-25T00:30:07.553 DEBUG:teuthology.orchestra.run.smithi032:mds.d> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.d.service 2024-04-25T00:30:07.596 DEBUG:teuthology.orchestra.run.smithi032:mds.g> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.g.service 2024-04-25T00:30:07.639 DEBUG:teuthology.orchestra.run.smithi032:mds.j> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.j.service 2024-04-25T00:30:07.682 DEBUG:teuthology.orchestra.run.smithi121:mds.b> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.b.service 2024-04-25T00:30:07.685 DEBUG:teuthology.orchestra.run.smithi121:mds.e> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.e.service 2024-04-25T00:30:07.730 DEBUG:teuthology.orchestra.run.smithi121:mds.h> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.h.service 2024-04-25T00:30:07.773 DEBUG:teuthology.orchestra.run.smithi121:mds.k> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.k.service 2024-04-25T00:30:07.816 DEBUG:teuthology.orchestra.run.smithi130:mds.c> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.c.service 2024-04-25T00:30:07.818 DEBUG:teuthology.orchestra.run.smithi130:mds.f> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.f.service 2024-04-25T00:30:07.861 DEBUG:teuthology.orchestra.run.smithi130:mds.i> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.i.service 2024-04-25T00:30:07.904 DEBUG:teuthology.orchestra.run.smithi130:mds.l> sudo journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.l.service 2024-04-25T00:30:07.947 INFO:tasks.cephadm:Setting up CephFS filesystem(s)... 2024-04-25T00:30:07.948 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-25T00:30:08.067 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.063+0000 7f00e6b84640 1 -- 172.21.15.32:0/2830174815 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f00e0102230 msgr2=0x7f00e0108ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:08.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.063+0000 7f00e6b84640 1 --2- 172.21.15.32:0/2830174815 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f00e0102230 0x7f00e0108ac0 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f00c8009900 tx=0x7f00c802a410 comp rx=0 tx=0).stop 2024-04-25T00:30:08.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.063+0000 7f00e6b84640 1 -- 172.21.15.32:0/2830174815 shutdown_connections 2024-04-25T00:30:08.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.063+0000 7f00e6b84640 1 --2- 172.21.15.32:0/2830174815 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f00e0102230 0x7f00e0108ac0 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.063+0000 7f00e6b84640 1 --2- 172.21.15.32:0/2830174815 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f00e0101870 0x7f00e0101cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.063+0000 7f00e6b84640 1 --2- 172.21.15.32:0/2830174815 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f00e0100680 0x7f00e0100aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.063+0000 7f00e6b84640 1 -- 172.21.15.32:0/2830174815 >> 172.21.15.32:0/2830174815 conn(0x7f00e00fbbf0 msgr2=0x7f00e00fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:08.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.063+0000 7f00e6b84640 1 -- 172.21.15.32:0/2830174815 shutdown_connections 2024-04-25T00:30:08.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.063+0000 7f00e6b84640 1 -- 172.21.15.32:0/2830174815 wait complete. 2024-04-25T00:30:08.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.063+0000 7f00e6b84640 1 Processor -- start 2024-04-25T00:30:08.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.063+0000 7f00e6b84640 1 -- start start 2024-04-25T00:30:08.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.063+0000 7f00e6b84640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f00e0100680 0x7f00e0069460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:08.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.063+0000 7f00e6b84640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f00e0101870 0x7f00e019c7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:08.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.063+0000 7f00e6b84640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f00e0102230 0x7f00e019ee00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:08.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.063+0000 7f00e6b84640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f00e010c2e0 con 0x7f00e0100680 2024-04-25T00:30:08.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.063+0000 7f00e6b84640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f00e010c160 con 0x7f00e0102230 2024-04-25T00:30:08.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.063+0000 7f00e6b84640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f00e010c460 con 0x7f00e0101870 2024-04-25T00:30:08.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.064+0000 7f00e4920640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f00e0100680 0x7f00e0069460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:08.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.064+0000 7f00e4920640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f00e0100680 0x7f00e0069460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:60076/0 (socket says 172.21.15.32:60076) 2024-04-25T00:30:08.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.064+0000 7f00e4920640 1 -- 172.21.15.32:0/1432599877 learned_addr learned my addr 172.21.15.32:0/1432599877 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:08.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.064+0000 7f00dffff640 1 --2- 172.21.15.32:0/1432599877 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f00e0101870 0x7f00e019c7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:08.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.064+0000 7f00e5121640 1 --2- 172.21.15.32:0/1432599877 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f00e0102230 0x7f00e019ee00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:08.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.064+0000 7f00e4920640 1 -- 172.21.15.32:0/1432599877 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f00e0101870 msgr2=0x7f00e019c7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:08.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.064+0000 7f00e4920640 1 --2- 172.21.15.32:0/1432599877 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f00e0101870 0x7f00e019c7c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.064+0000 7f00e4920640 1 -- 172.21.15.32:0/1432599877 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f00e0102230 msgr2=0x7f00e019ee00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:08.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.064+0000 7f00e4920640 1 --2- 172.21.15.32:0/1432599877 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f00e0102230 0x7f00e019ee00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.064+0000 7f00e4920640 1 -- 172.21.15.32:0/1432599877 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f00e019f6c0 con 0x7f00e0100680 2024-04-25T00:30:08.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.064+0000 7f00e4920640 1 --2- 172.21.15.32:0/1432599877 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f00e0100680 0x7f00e0069460 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f00d800ccb0 tx=0x7f00d800a620 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:08.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.064+0000 7f00ddffb640 1 -- 172.21.15.32:0/1432599877 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f00d8003990 con 0x7f00e0100680 2024-04-25T00:30:08.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.064+0000 7f00ddffb640 1 -- 172.21.15.32:0/1432599877 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f00d8003b30 con 0x7f00e0100680 2024-04-25T00:30:08.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.064+0000 7f00ddffb640 1 -- 172.21.15.32:0/1432599877 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f00d800f7b0 con 0x7f00e0100680 2024-04-25T00:30:08.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.064+0000 7f00e6b84640 1 -- 172.21.15.32:0/1432599877 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f00e019f9b0 con 0x7f00e0100680 2024-04-25T00:30:08.071 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.064+0000 7f00e6b84640 1 -- 172.21.15.32:0/1432599877 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f00e019feb0 con 0x7f00e0100680 2024-04-25T00:30:08.071 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.065+0000 7f00e6b84640 1 -- 172.21.15.32:0/1432599877 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f00e0107940 con 0x7f00e0100680 2024-04-25T00:30:08.071 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.066+0000 7f00ddffb640 1 -- 172.21.15.32:0/1432599877 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f00d800f950 con 0x7f00e0100680 2024-04-25T00:30:08.072 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.067+0000 7f00ddffb640 1 --2- 172.21.15.32:0/1432599877 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f00cc0a2c10 0x7f00cc0a50d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:08.072 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.067+0000 7f00dffff640 1 --2- 172.21.15.32:0/1432599877 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f00cc0a2c10 0x7f00cc0a50d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:08.072 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.067+0000 7f00ddffb640 1 -- 172.21.15.32:0/1432599877 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7f00d80ced00 con 0x7f00e0100680 2024-04-25T00:30:08.072 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.068+0000 7f00dffff640 1 --2- 172.21.15.32:0/1432599877 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f00cc0a2c10 0x7f00cc0a50d0 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f00e00691d0 tx=0x7f00d000b390 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:08.073 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.072+0000 7f00ddffb640 1 -- 172.21.15.32:0/1432599877 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f00d8014030 con 0x7f00e0100680 2024-04-25T00:30:08.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:07 smithi121 ceph-mon[32231]: pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:08.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:07 smithi121 ceph-mon[32231]: from='client.24388 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "12;smithi032=a;smithi032=d;smithi032=g;smithi032=j;smithi121=b;smithi121=e;smithi121=h;smithi121=k;smithi130=c;smithi130=f;smithi130=i;smithi130=l", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:30:08.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:07 smithi121 ceph-mon[32231]: Saving service mds.all spec with placement smithi032=a;smithi032=d;smithi032=g;smithi032=j;smithi121=b;smithi121=e;smithi121=h;smithi121=k;smithi130=c;smithi130=f;smithi130=i;smithi130=l;count:12 2024-04-25T00:30:08.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:07 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:08.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:07 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:30:08.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:07 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:08.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:07 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:30:08.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:07 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:08.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:07 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:08.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:07 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:08.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:07 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:08.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:07 smithi121 ceph-mon[32231]: Deploying daemon mds.l on smithi130 2024-04-25T00:30:08.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:07 smithi130 ceph-mon[32525]: pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:08.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:07 smithi130 ceph-mon[32525]: from='client.24388 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "12;smithi032=a;smithi032=d;smithi032=g;smithi032=j;smithi121=b;smithi121=e;smithi121=h;smithi121=k;smithi130=c;smithi130=f;smithi130=i;smithi130=l", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:30:08.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:07 smithi130 ceph-mon[32525]: Saving service mds.all spec with placement smithi032=a;smithi032=d;smithi032=g;smithi032=j;smithi121=b;smithi121=e;smithi121=h;smithi121=k;smithi130=c;smithi130=f;smithi130=i;smithi130=l;count:12 2024-04-25T00:30:08.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:07 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:08.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:07 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:30:08.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:07 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:08.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:07 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:30:08.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:07 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:08.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:07 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:08.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:07 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:08.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:07 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:08.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:07 smithi130 ceph-mon[32525]: Deploying daemon mds.l on smithi130 2024-04-25T00:30:08.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:07 smithi032 ceph-mon[29357]: pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:08.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:07 smithi032 ceph-mon[29357]: from='client.24388 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "12;smithi032=a;smithi032=d;smithi032=g;smithi032=j;smithi121=b;smithi121=e;smithi121=h;smithi121=k;smithi130=c;smithi130=f;smithi130=i;smithi130=l", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:30:08.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:07 smithi032 ceph-mon[29357]: Saving service mds.all spec with placement smithi032=a;smithi032=d;smithi032=g;smithi032=j;smithi121=b;smithi121=e;smithi121=h;smithi121=k;smithi130=c;smithi130=f;smithi130=i;smithi130=l;count:12 2024-04-25T00:30:08.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:08.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:30:08.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:08.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:30:08.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:08.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:08.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:08.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:07 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:08.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:07 smithi032 ceph-mon[29357]: Deploying daemon mds.l on smithi130 2024-04-25T00:30:08.261 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.260+0000 7f00e6b84640 1 -- 172.21.15.32:0/1432599877 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f00e0062340 con 0x7f00e0100680 2024-04-25T00:30:08.262 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.261+0000 7f00ddffb640 1 -- 172.21.15.32:0/1432599877 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v81) ==== 74+0+18456 (secure 0 0 0) 0x7f00d809c060 con 0x7f00e0100680 2024-04-25T00:30:08.263 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:08.263 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":81,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:29:59.700603+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:30:08.264 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.263+0000 7f00e6b84640 1 -- 172.21.15.32:0/1432599877 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f00cc0a2c10 msgr2=0x7f00cc0a50d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:08.264 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.263+0000 7f00e6b84640 1 --2- 172.21.15.32:0/1432599877 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f00cc0a2c10 0x7f00cc0a50d0 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f00e00691d0 tx=0x7f00d000b390 comp rx=0 tx=0).stop 2024-04-25T00:30:08.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.263+0000 7f00e6b84640 1 -- 172.21.15.32:0/1432599877 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f00e0100680 msgr2=0x7f00e0069460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:08.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.263+0000 7f00e6b84640 1 --2- 172.21.15.32:0/1432599877 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f00e0100680 0x7f00e0069460 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f00d800ccb0 tx=0x7f00d800a620 comp rx=0 tx=0).stop 2024-04-25T00:30:08.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.263+0000 7f00e6b84640 1 -- 172.21.15.32:0/1432599877 shutdown_connections 2024-04-25T00:30:08.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.263+0000 7f00e6b84640 1 --2- 172.21.15.32:0/1432599877 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f00cc0a2c10 0x7f00cc0a50d0 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.263+0000 7f00e6b84640 1 --2- 172.21.15.32:0/1432599877 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f00e0102230 0x7f00e019ee00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.263+0000 7f00e6b84640 1 --2- 172.21.15.32:0/1432599877 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f00e0101870 0x7f00e019c7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.263+0000 7f00e6b84640 1 --2- 172.21.15.32:0/1432599877 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f00e0100680 0x7f00e0069460 unknown :-1 s=CLOSED pgs=149 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.263+0000 7f00e6b84640 1 -- 172.21.15.32:0/1432599877 >> 172.21.15.32:0/1432599877 conn(0x7f00e00fbbf0 msgr2=0x7f00e0104b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:08.266 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.263+0000 7f00e6b84640 1 -- 172.21.15.32:0/1432599877 shutdown_connections 2024-04-25T00:30:08.266 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.263+0000 7f00e6b84640 1 -- 172.21.15.32:0/1432599877 wait complete. 2024-04-25T00:30:08.276 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-25T00:26:07.306839+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-04-25T00:30:08.276 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-25T00:30:08.384 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.379+0000 7f9feaa1b640 1 -- 172.21.15.32:0/1204790812 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fe4104bf0 msgr2=0x7f9fe4106fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:08.384 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.379+0000 7f9feaa1b640 1 --2- 172.21.15.32:0/1204790812 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fe4104bf0 0x7f9fe4106fe0 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f9fd8009900 tx=0x7f9fd802a410 comp rx=0 tx=0).stop 2024-04-25T00:30:08.384 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.379+0000 7f9feaa1b640 1 -- 172.21.15.32:0/1204790812 shutdown_connections 2024-04-25T00:30:08.384 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.379+0000 7f9feaa1b640 1 --2- 172.21.15.32:0/1204790812 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9fe4107520 0x7f9fe4109910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.384 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.379+0000 7f9feaa1b640 1 --2- 172.21.15.32:0/1204790812 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fe4104bf0 0x7f9fe4106fe0 unknown :-1 s=CLOSED pgs=150 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.384 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.379+0000 7f9feaa1b640 1 --2- 172.21.15.32:0/1204790812 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9fe41022c0 0x7f9fe41046b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.384 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.379+0000 7f9feaa1b640 1 -- 172.21.15.32:0/1204790812 >> 172.21.15.32:0/1204790812 conn(0x7f9fe40fbc30 msgr2=0x7f9fe40fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:08.384 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.379+0000 7f9feaa1b640 1 -- 172.21.15.32:0/1204790812 shutdown_connections 2024-04-25T00:30:08.385 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.379+0000 7f9feaa1b640 1 -- 172.21.15.32:0/1204790812 wait complete. 2024-04-25T00:30:08.385 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.379+0000 7f9feaa1b640 1 Processor -- start 2024-04-25T00:30:08.385 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.380+0000 7f9feaa1b640 1 -- start start 2024-04-25T00:30:08.385 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.380+0000 7f9feaa1b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9fe41022c0 0x7f9fe4101c30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:08.385 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.380+0000 7f9feaa1b640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9fe4104bf0 0x7f9fe4100270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:08.385 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.380+0000 7f9feaa1b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fe4107520 0x7f9fe41007b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:08.385 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.380+0000 7f9feaa1b640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f9fe410d460 con 0x7f9fe4107520 2024-04-25T00:30:08.385 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.380+0000 7f9feaa1b640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f9fe410d2e0 con 0x7f9fe41022c0 2024-04-25T00:30:08.385 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.380+0000 7f9feaa1b640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f9fe410d5e0 con 0x7f9fe4104bf0 2024-04-25T00:30:08.385 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.380+0000 7f9fe8fb8640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fe4107520 0x7f9fe41007b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:08.385 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.380+0000 7f9fe8fb8640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fe4107520 0x7f9fe41007b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:60094/0 (socket says 172.21.15.32:60094) 2024-04-25T00:30:08.385 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.380+0000 7f9fe8fb8640 1 -- 172.21.15.32:0/2459392276 learned_addr learned my addr 172.21.15.32:0/2459392276 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:08.386 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.380+0000 7f9fe37fe640 1 --2- 172.21.15.32:0/2459392276 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9fe4104bf0 0x7f9fe4100270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:08.386 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.380+0000 7f9fe3fff640 1 --2- 172.21.15.32:0/2459392276 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9fe41022c0 0x7f9fe4101c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:08.386 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.380+0000 7f9fe8fb8640 1 -- 172.21.15.32:0/2459392276 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9fe4104bf0 msgr2=0x7f9fe4100270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:08.386 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.380+0000 7f9fe8fb8640 1 --2- 172.21.15.32:0/2459392276 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9fe4104bf0 0x7f9fe4100270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.386 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.380+0000 7f9fe8fb8640 1 -- 172.21.15.32:0/2459392276 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9fe41022c0 msgr2=0x7f9fe4101c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:08.386 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.380+0000 7f9fe8fb8640 1 --2- 172.21.15.32:0/2459392276 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9fe41022c0 0x7f9fe4101c30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.386 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.380+0000 7f9fe8fb8640 1 -- 172.21.15.32:0/2459392276 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9fe4101040 con 0x7f9fe4107520 2024-04-25T00:30:08.386 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.380+0000 7f9fe8fb8640 1 --2- 172.21.15.32:0/2459392276 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fe4107520 0x7f9fe41007b0 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f9fdc00cad0 tx=0x7f9fdc00cf90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:08.386 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.381+0000 7f9fe17fa640 1 -- 172.21.15.32:0/2459392276 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9fdc00ae20 con 0x7f9fe4107520 2024-04-25T00:30:08.386 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.381+0000 7f9feaa1b640 1 -- 172.21.15.32:0/2459392276 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9fe41a1c30 con 0x7f9fe4107520 2024-04-25T00:30:08.386 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.381+0000 7f9fe17fa640 1 -- 172.21.15.32:0/2459392276 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9fdc005020 con 0x7f9fe4107520 2024-04-25T00:30:08.386 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.381+0000 7f9fe17fa640 1 -- 172.21.15.32:0/2459392276 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9fdc00f6f0 con 0x7f9fe4107520 2024-04-25T00:30:08.386 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.381+0000 7f9feaa1b640 1 -- 172.21.15.32:0/2459392276 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9fe41a2150 con 0x7f9fe4107520 2024-04-25T00:30:08.386 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.382+0000 7f9feaa1b640 1 -- 172.21.15.32:0/2459392276 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9fe40689b0 con 0x7f9fe4107520 2024-04-25T00:30:08.387 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.383+0000 7f9fe17fa640 1 -- 172.21.15.32:0/2459392276 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f9fdc0051d0 con 0x7f9fe4107520 2024-04-25T00:30:08.387 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.383+0000 7f9fe17fa640 1 --2- 172.21.15.32:0/2459392276 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f9fcc0a2b90 0x7f9fcc0a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:08.387 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.384+0000 7f9fe3fff640 1 --2- 172.21.15.32:0/2459392276 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f9fcc0a2b90 0x7f9fcc0a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:08.388 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.384+0000 7f9fe17fa640 1 -- 172.21.15.32:0/2459392276 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7f9fdc0ceca0 con 0x7f9fe4107520 2024-04-25T00:30:08.388 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.384+0000 7f9fe3fff640 1 --2- 172.21.15.32:0/2459392276 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f9fcc0a2b90 0x7f9fcc0a5050 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f9fd4009bd0 tx=0x7f9fd4005f90 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:08.389 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.388+0000 7f9fe17fa640 1 -- 172.21.15.32:0/2459392276 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f9fdc014030 con 0x7f9fe4107520 2024-04-25T00:30:08.574 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.572+0000 7f9feaa1b640 1 -- 172.21.15.32:0/2459392276 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7f9fe4062340 con 0x7f9fe4107520 2024-04-25T00:30:08.574 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.573+0000 7f9fe17fa640 1 -- 172.21.15.32:0/2459392276 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v81) ==== 93+0+10 (secure 0 0 0) 0x7f9fdc09b9b0 con 0x7f9fe4107520 2024-04-25T00:30:08.575 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 1 2024-04-25T00:30:08.576 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.575+0000 7f9feaa1b640 1 -- 172.21.15.32:0/2459392276 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f9fcc0a2b90 msgr2=0x7f9fcc0a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:08.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.575+0000 7f9feaa1b640 1 --2- 172.21.15.32:0/2459392276 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f9fcc0a2b90 0x7f9fcc0a5050 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f9fd4009bd0 tx=0x7f9fd4005f90 comp rx=0 tx=0).stop 2024-04-25T00:30:08.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.575+0000 7f9feaa1b640 1 -- 172.21.15.32:0/2459392276 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fe4107520 msgr2=0x7f9fe41007b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:08.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.575+0000 7f9feaa1b640 1 --2- 172.21.15.32:0/2459392276 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fe4107520 0x7f9fe41007b0 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f9fdc00cad0 tx=0x7f9fdc00cf90 comp rx=0 tx=0).stop 2024-04-25T00:30:08.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.575+0000 7f9feaa1b640 1 -- 172.21.15.32:0/2459392276 shutdown_connections 2024-04-25T00:30:08.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.575+0000 7f9feaa1b640 1 --2- 172.21.15.32:0/2459392276 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f9fcc0a2b90 0x7f9fcc0a5050 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.575+0000 7f9feaa1b640 1 --2- 172.21.15.32:0/2459392276 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fe4107520 0x7f9fe41007b0 unknown :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.575+0000 7f9feaa1b640 1 --2- 172.21.15.32:0/2459392276 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9fe4104bf0 0x7f9fe4100270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.575+0000 7f9feaa1b640 1 --2- 172.21.15.32:0/2459392276 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9fe41022c0 0x7f9fe4101c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.575+0000 7f9feaa1b640 1 -- 172.21.15.32:0/2459392276 >> 172.21.15.32:0/2459392276 conn(0x7f9fe40fbc30 msgr2=0x7f9fe4102ce0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:08.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.575+0000 7f9feaa1b640 1 -- 172.21.15.32:0/2459392276 shutdown_connections 2024-04-25T00:30:08.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.575+0000 7f9feaa1b640 1 -- 172.21.15.32:0/2459392276 wait complete. 2024-04-25T00:30:08.588 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:30:08.692 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.688+0000 7fa3bd5b6640 1 -- 172.21.15.32:0/1785758292 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa3b8101890 msgr2=0x7fa3b8101d10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:08.692 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.688+0000 7fa3bd5b6640 1 --2- 172.21.15.32:0/1785758292 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa3b8101890 0x7fa3b8101d10 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7fa3ac009900 tx=0x7fa3ac02a410 comp rx=0 tx=0).stop 2024-04-25T00:30:08.692 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.688+0000 7fa3bd5b6640 1 -- 172.21.15.32:0/1785758292 shutdown_connections 2024-04-25T00:30:08.692 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.688+0000 7fa3bd5b6640 1 --2- 172.21.15.32:0/1785758292 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa3b8102250 0x7fa3b8108ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.692 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.688+0000 7fa3bd5b6640 1 --2- 172.21.15.32:0/1785758292 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa3b8101890 0x7fa3b8101d10 unknown :-1 s=CLOSED pgs=152 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.692 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.688+0000 7fa3bd5b6640 1 --2- 172.21.15.32:0/1785758292 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa3b81006a0 0x7fa3b8100ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.692 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.688+0000 7fa3bd5b6640 1 -- 172.21.15.32:0/1785758292 >> 172.21.15.32:0/1785758292 conn(0x7fa3b80fbc30 msgr2=0x7fa3b80fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:08.692 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.688+0000 7fa3bd5b6640 1 -- 172.21.15.32:0/1785758292 shutdown_connections 2024-04-25T00:30:08.693 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.688+0000 7fa3bd5b6640 1 -- 172.21.15.32:0/1785758292 wait complete. 2024-04-25T00:30:08.693 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.689+0000 7fa3bd5b6640 1 Processor -- start 2024-04-25T00:30:08.693 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.689+0000 7fa3bd5b6640 1 -- start start 2024-04-25T00:30:08.693 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.689+0000 7fa3bd5b6640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa3b81006a0 0x7fa3b8197af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:08.693 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.689+0000 7fa3bd5b6640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa3b8101890 0x7fa3b8198030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:08.693 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.689+0000 7fa3bd5b6640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa3b8102250 0x7fa3b819f050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:08.693 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.689+0000 7fa3bd5b6640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fa3b810d4a0 con 0x7fa3b8101890 2024-04-25T00:30:08.693 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.689+0000 7fa3bd5b6640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fa3b810d320 con 0x7fa3b81006a0 2024-04-25T00:30:08.693 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.689+0000 7fa3bd5b6640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fa3b810d620 con 0x7fa3b8102250 2024-04-25T00:30:08.693 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.689+0000 7fa3b67fc640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa3b8101890 0x7fa3b8198030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:08.693 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.689+0000 7fa3b67fc640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa3b8101890 0x7fa3b8198030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:60106/0 (socket says 172.21.15.32:60106) 2024-04-25T00:30:08.693 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.689+0000 7fa3b67fc640 1 -- 172.21.15.32:0/979193458 learned_addr learned my addr 172.21.15.32:0/979193458 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:08.693 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.689+0000 7fa3b77fe640 1 --2- 172.21.15.32:0/979193458 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa3b8102250 0x7fa3b819f050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:08.693 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.689+0000 7fa3b6ffd640 1 --2- 172.21.15.32:0/979193458 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa3b81006a0 0x7fa3b8197af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:08.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.689+0000 7fa3b67fc640 1 -- 172.21.15.32:0/979193458 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa3b8102250 msgr2=0x7fa3b819f050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:08.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.689+0000 7fa3b67fc640 1 --2- 172.21.15.32:0/979193458 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa3b8102250 0x7fa3b819f050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.689+0000 7fa3b67fc640 1 -- 172.21.15.32:0/979193458 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa3b81006a0 msgr2=0x7fa3b8197af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:08.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.689+0000 7fa3b67fc640 1 --2- 172.21.15.32:0/979193458 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa3b81006a0 0x7fa3b8197af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.689+0000 7fa3b67fc640 1 -- 172.21.15.32:0/979193458 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa3b819f6c0 con 0x7fa3b8101890 2024-04-25T00:30:08.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.689+0000 7fa3b77fe640 1 --2- 172.21.15.32:0/979193458 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa3b8102250 0x7fa3b819f050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:30:08.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.690+0000 7fa3b67fc640 1 --2- 172.21.15.32:0/979193458 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa3b8101890 0x7fa3b8198030 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7fa3ac031440 tx=0x7fa3ac032e20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:08.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.690+0000 7fa39ffff640 1 -- 172.21.15.32:0/979193458 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa3ac03f030 con 0x7fa3b8101890 2024-04-25T00:30:08.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.690+0000 7fa3bd5b6640 1 -- 172.21.15.32:0/979193458 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa3b819f950 con 0x7fa3b8101890 2024-04-25T00:30:08.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.690+0000 7fa39ffff640 1 -- 172.21.15.32:0/979193458 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa3ac003ce0 con 0x7fa3b8101890 2024-04-25T00:30:08.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.690+0000 7fa39ffff640 1 -- 172.21.15.32:0/979193458 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa3ac035070 con 0x7fa3b8101890 2024-04-25T00:30:08.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.690+0000 7fa3bd5b6640 1 -- 172.21.15.32:0/979193458 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa3b819fe40 con 0x7fa3b8101890 2024-04-25T00:30:08.695 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.691+0000 7fa3bd5b6640 1 -- 172.21.15.32:0/979193458 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa3b8107a40 con 0x7fa3b8101890 2024-04-25T00:30:08.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.692+0000 7fa39ffff640 1 -- 172.21.15.32:0/979193458 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7fa3ac042030 con 0x7fa3b8101890 2024-04-25T00:30:08.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.692+0000 7fa39ffff640 1 --2- 172.21.15.32:0/979193458 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa3a00a2b90 0x7fa3a00a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:08.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.693+0000 7fa3b6ffd640 1 --2- 172.21.15.32:0/979193458 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa3a00a2b90 0x7fa3a00a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:08.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.693+0000 7fa39ffff640 1 -- 172.21.15.32:0/979193458 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7fa3ac0ef110 con 0x7fa3b8101890 2024-04-25T00:30:08.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.693+0000 7fa3b6ffd640 1 --2- 172.21.15.32:0/979193458 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa3a00a2b90 0x7fa3a00a5050 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7fa3b81016f0 tx=0x7fa3a800b390 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:08.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.697+0000 7fa39ffff640 1 -- 172.21.15.32:0/979193458 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fa3ac0bbec0 con 0x7fa3b8101890 2024-04-25T00:30:08.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.921+0000 7fa3bd5b6640 1 -- 172.21.15.32:0/979193458 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fa3b802d3c0 con 0x7fa3b8101890 2024-04-25T00:30:08.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.921+0000 7fa39ffff640 1 -- 172.21.15.32:0/979193458 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 1 v1) ==== 93+0+569 (secure 0 0 0) 0x7fa3ac0c01c0 con 0x7fa3b8101890 2024-04-25T00:30:08.923 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:08.923 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":1,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[]} 2024-04-25T00:30:08.923 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 1 2024-04-25T00:30:08.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.923+0000 7fa3bd5b6640 1 -- 172.21.15.32:0/979193458 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa3a00a2b90 msgr2=0x7fa3a00a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:08.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.923+0000 7fa3bd5b6640 1 --2- 172.21.15.32:0/979193458 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa3a00a2b90 0x7fa3a00a5050 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7fa3b81016f0 tx=0x7fa3a800b390 comp rx=0 tx=0).stop 2024-04-25T00:30:08.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.923+0000 7fa3bd5b6640 1 -- 172.21.15.32:0/979193458 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa3b8101890 msgr2=0x7fa3b8198030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:08.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.923+0000 7fa3bd5b6640 1 --2- 172.21.15.32:0/979193458 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa3b8101890 0x7fa3b8198030 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7fa3ac031440 tx=0x7fa3ac032e20 comp rx=0 tx=0).stop 2024-04-25T00:30:08.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.923+0000 7fa3bd5b6640 1 -- 172.21.15.32:0/979193458 shutdown_connections 2024-04-25T00:30:08.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.923+0000 7fa3bd5b6640 1 --2- 172.21.15.32:0/979193458 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa3a00a2b90 0x7fa3a00a5050 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.923+0000 7fa3bd5b6640 1 --2- 172.21.15.32:0/979193458 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa3b8102250 0x7fa3b819f050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.923+0000 7fa3bd5b6640 1 --2- 172.21.15.32:0/979193458 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa3b8101890 0x7fa3b8198030 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.923+0000 7fa3bd5b6640 1 --2- 172.21.15.32:0/979193458 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa3b81006a0 0x7fa3b8197af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:08.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.923+0000 7fa3bd5b6640 1 -- 172.21.15.32:0/979193458 >> 172.21.15.32:0/979193458 conn(0x7fa3b80fbc30 msgr2=0x7fa3b8104c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:08.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.924+0000 7fa3bd5b6640 1 -- 172.21.15.32:0/979193458 shutdown_connections 2024-04-25T00:30:08.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:08.924+0000 7fa3bd5b6640 1 -- 172.21.15.32:0/979193458 wait complete. 2024-04-25T00:30:09.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:08 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1432599877' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:30:09.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:08 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2459392276' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:30:09.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:08 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1432599877' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:30:09.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:08 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2459392276' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:30:09.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:08 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1432599877' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:30:09.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:08 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2459392276' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:30:10.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:09 smithi130 ceph-mon[32525]: pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:10.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:09 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/979193458' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:10.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:09 smithi032 ceph-mon[29357]: pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:10.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:09 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/979193458' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:10.313 INFO:journalctl@ceph.mds.l.smithi130.stdout:Apr 25 00:30:10 smithi130 podman[62411]: 2024-04-25 00:30:10.289770682 +0000 UTC m=+0.352885158 container start 62b6c422c9477910b984e324fdb53604e84d60c5f02fb8be002c1571b60f0f98 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-l, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-d2c45d9, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2) 2024-04-25T00:30:10.313 INFO:journalctl@ceph.mds.l.smithi130.stdout:Apr 25 00:30:10 smithi130 ceph-mds[62430]: -- 172.21.15.130:0/2159034844 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5619110cda40 con 0x5619110a0800 2024-04-25T00:30:10.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:09 smithi121 ceph-mon[32231]: pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:10.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:09 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/979193458' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:10.641 INFO:journalctl@ceph.mds.l.smithi130.stdout:Apr 25 00:30:10 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-l[62426]: starting mds.l at 2024-04-25T00:30:10.642 INFO:journalctl@ceph.mds.l.smithi130.stdout:Apr 25 00:30:10 smithi130 bash[62411]: 62b6c422c9477910b984e324fdb53604e84d60c5f02fb8be002c1571b60f0f98 2024-04-25T00:30:10.642 INFO:journalctl@ceph.mds.l.smithi130.stdout:Apr 25 00:30:10 smithi130 systemd[1]: Started Ceph mds.l for cb997156-0299-11ef-bc93-c7b262605968. 2024-04-25T00:30:10.937 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:30:11.054 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.051+0000 7fda703c1640 1 -- 172.21.15.32:0/1917193871 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fda68101870 msgr2=0x7fda68101cf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:11.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.051+0000 7fda703c1640 1 --2- 172.21.15.32:0/1917193871 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fda68101870 0x7fda68101cf0 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7fda58009900 tx=0x7fda5802a410 comp rx=0 tx=0).stop 2024-04-25T00:30:11.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.051+0000 7fda703c1640 1 -- 172.21.15.32:0/1917193871 shutdown_connections 2024-04-25T00:30:11.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.051+0000 7fda703c1640 1 --2- 172.21.15.32:0/1917193871 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fda68102230 0x7fda68108ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:11.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.051+0000 7fda703c1640 1 --2- 172.21.15.32:0/1917193871 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fda68101870 0x7fda68101cf0 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:11.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.051+0000 7fda703c1640 1 --2- 172.21.15.32:0/1917193871 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fda68100680 0x7fda68100aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:11.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.051+0000 7fda703c1640 1 -- 172.21.15.32:0/1917193871 >> 172.21.15.32:0/1917193871 conn(0x7fda680fbbf0 msgr2=0x7fda680fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:11.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.051+0000 7fda703c1640 1 -- 172.21.15.32:0/1917193871 shutdown_connections 2024-04-25T00:30:11.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.051+0000 7fda703c1640 1 -- 172.21.15.32:0/1917193871 wait complete. 2024-04-25T00:30:11.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.051+0000 7fda703c1640 1 Processor -- start 2024-04-25T00:30:11.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.051+0000 7fda703c1640 1 -- start start 2024-04-25T00:30:11.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.052+0000 7fda703c1640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fda68100680 0x7fda68069440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:11.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.052+0000 7fda703c1640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fda68101870 0x7fda6819c7f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:11.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.052+0000 7fda703c1640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fda68102230 0x7fda6819ee10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:11.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.052+0000 7fda703c1640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fda6810c300 con 0x7fda68102230 2024-04-25T00:30:11.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.052+0000 7fda703c1640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fda6810c180 con 0x7fda68101870 2024-04-25T00:30:11.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.052+0000 7fda703c1640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fda6810c480 con 0x7fda68100680 2024-04-25T00:30:11.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.052+0000 7fda6d95c640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fda68101870 0x7fda6819c7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:11.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.052+0000 7fda6e95e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fda68102230 0x7fda6819ee10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:11.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.052+0000 7fda6d95c640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fda68101870 0x7fda6819c7f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:49018/0 (socket says 172.21.15.32:49018) 2024-04-25T00:30:11.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.052+0000 7fda6e95e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fda68102230 0x7fda6819ee10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:60114/0 (socket says 172.21.15.32:60114) 2024-04-25T00:30:11.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.052+0000 7fda6d95c640 1 -- 172.21.15.32:0/2734215135 learned_addr learned my addr 172.21.15.32:0/2734215135 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:11.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.052+0000 7fda6e15d640 1 --2- 172.21.15.32:0/2734215135 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fda68100680 0x7fda68069440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:11.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.052+0000 7fda6e95e640 1 -- 172.21.15.32:0/2734215135 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fda68100680 msgr2=0x7fda68069440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:11.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.052+0000 7fda6e95e640 1 --2- 172.21.15.32:0/2734215135 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fda68100680 0x7fda68069440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:11.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.052+0000 7fda6e95e640 1 -- 172.21.15.32:0/2734215135 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fda68101870 msgr2=0x7fda6819c7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:11.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.052+0000 7fda6e95e640 1 --2- 172.21.15.32:0/2734215135 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fda68101870 0x7fda6819c7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:11.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.052+0000 7fda6e95e640 1 -- 172.21.15.32:0/2734215135 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fda6819f720 con 0x7fda68102230 2024-04-25T00:30:11.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.052+0000 7fda6d95c640 1 --2- 172.21.15.32:0/2734215135 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fda68101870 0x7fda6819c7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:30:11.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.052+0000 7fda6e15d640 1 --2- 172.21.15.32:0/2734215135 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fda68100680 0x7fda68069440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:30:11.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.052+0000 7fda6e95e640 1 --2- 172.21.15.32:0/2734215135 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fda68102230 0x7fda6819ee10 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7fda6400e940 tx=0x7fda6400ee00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:11.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.052+0000 7fda5f7fe640 1 -- 172.21.15.32:0/2734215135 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fda64004d20 con 0x7fda68102230 2024-04-25T00:30:11.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.053+0000 7fda5f7fe640 1 -- 172.21.15.32:0/2734215135 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fda64005020 con 0x7fda68102230 2024-04-25T00:30:11.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.053+0000 7fda703c1640 1 -- 172.21.15.32:0/2734215135 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fda6819f9b0 con 0x7fda68102230 2024-04-25T00:30:11.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.053+0000 7fda5f7fe640 1 -- 172.21.15.32:0/2734215135 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fda640116b0 con 0x7fda68102230 2024-04-25T00:30:11.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.053+0000 7fda703c1640 1 -- 172.21.15.32:0/2734215135 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fda6819fed0 con 0x7fda68102230 2024-04-25T00:30:11.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.054+0000 7fda703c1640 1 -- 172.21.15.32:0/2734215135 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fda681079a0 con 0x7fda68102230 2024-04-25T00:30:11.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.055+0000 7fda5f7fe640 1 -- 172.21.15.32:0/2734215135 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7fda640054d0 con 0x7fda68102230 2024-04-25T00:30:11.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.055+0000 7fda5f7fe640 1 --2- 172.21.15.32:0/2734215135 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fda540a2b90 0x7fda540a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:11.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.056+0000 7fda5f7fe640 1 -- 172.21.15.32:0/2734215135 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7fda640cefd0 con 0x7fda68102230 2024-04-25T00:30:11.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.056+0000 7fda6e15d640 1 --2- 172.21.15.32:0/2734215135 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fda540a2b90 0x7fda540a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:11.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.056+0000 7fda6e15d640 1 --2- 172.21.15.32:0/2734215135 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fda540a2b90 0x7fda540a5050 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7fda681016d0 tx=0x7fda60005f90 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:11.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.060+0000 7fda5f7fe640 1 -- 172.21.15.32:0/2734215135 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fda64016030 con 0x7fda68102230 2024-04-25T00:30:11.285 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.283+0000 7fda703c1640 1 -- 172.21.15.32:0/2734215135 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fda6802d1c0 con 0x7fda68102230 2024-04-25T00:30:11.285 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.284+0000 7fda5f7fe640 1 -- 172.21.15.32:0/2734215135 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 2 v2) ==== 93+0+1396 (secure 0 0 0) 0x7fda6409c3d0 con 0x7fda68102230 2024-04-25T00:30:11.285 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:11.286 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":2,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2}],"filesystems":[]} 2024-04-25T00:30:11.286 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 2 2024-04-25T00:30:11.287 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.286+0000 7fda703c1640 1 -- 172.21.15.32:0/2734215135 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fda540a2b90 msgr2=0x7fda540a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:11.287 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.286+0000 7fda703c1640 1 --2- 172.21.15.32:0/2734215135 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fda540a2b90 0x7fda540a5050 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7fda681016d0 tx=0x7fda60005f90 comp rx=0 tx=0).stop 2024-04-25T00:30:11.287 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.286+0000 7fda703c1640 1 -- 172.21.15.32:0/2734215135 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fda68102230 msgr2=0x7fda6819ee10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:11.287 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.286+0000 7fda703c1640 1 --2- 172.21.15.32:0/2734215135 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fda68102230 0x7fda6819ee10 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7fda6400e940 tx=0x7fda6400ee00 comp rx=0 tx=0).stop 2024-04-25T00:30:11.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.286+0000 7fda703c1640 1 -- 172.21.15.32:0/2734215135 shutdown_connections 2024-04-25T00:30:11.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.286+0000 7fda703c1640 1 --2- 172.21.15.32:0/2734215135 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fda540a2b90 0x7fda540a5050 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:11.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.286+0000 7fda703c1640 1 --2- 172.21.15.32:0/2734215135 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fda68102230 0x7fda6819ee10 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:11.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.286+0000 7fda703c1640 1 --2- 172.21.15.32:0/2734215135 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fda68101870 0x7fda6819c7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:11.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.286+0000 7fda703c1640 1 --2- 172.21.15.32:0/2734215135 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fda68100680 0x7fda68069440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:11.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.286+0000 7fda703c1640 1 -- 172.21.15.32:0/2734215135 >> 172.21.15.32:0/2734215135 conn(0x7fda680fbbf0 msgr2=0x7fda68104b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:11.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.286+0000 7fda703c1640 1 -- 172.21.15.32:0/2734215135 shutdown_connections 2024-04-25T00:30:11.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:11.286+0000 7fda703c1640 1 -- 172.21.15.32:0/2734215135 wait complete. 2024-04-25T00:30:11.438 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:11 smithi121 ceph-mon[32231]: mds.? [v2:172.21.15.130:6832/3000660373,v1:172.21.15.130:6833/3000660373] up:boot 2024-04-25T00:30:11.438 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:11 smithi121 ceph-mon[32231]: fsmap 1 up:standby 2024-04-25T00:30:11.438 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:11 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2024-04-25T00:30:11.438 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:11 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:11.438 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:11 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:11.438 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:11 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:11.438 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:11 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:11.438 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:11 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:11.438 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:11 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:11.438 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:11 smithi121 ceph-mon[32231]: Deploying daemon mds.k on smithi121 2024-04-25T00:30:11.438 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:11 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2734215135' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:11.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:11 smithi130 ceph-mon[32525]: mds.? [v2:172.21.15.130:6832/3000660373,v1:172.21.15.130:6833/3000660373] up:boot 2024-04-25T00:30:11.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:11 smithi130 ceph-mon[32525]: fsmap 1 up:standby 2024-04-25T00:30:11.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:11 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2024-04-25T00:30:11.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:11 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:11.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:11 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:11.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:11 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:11.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:11 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:11.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:11 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:11.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:11 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:11.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:11 smithi130 ceph-mon[32525]: Deploying daemon mds.k on smithi121 2024-04-25T00:30:11.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:11 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2734215135' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:11.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:11 smithi032 ceph-mon[29357]: mds.? [v2:172.21.15.130:6832/3000660373,v1:172.21.15.130:6833/3000660373] up:boot 2024-04-25T00:30:11.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:11 smithi032 ceph-mon[29357]: fsmap 1 up:standby 2024-04-25T00:30:11.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:11 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2024-04-25T00:30:11.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:11 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:11.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:11 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:11.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:11 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:11.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:11 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:11.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:11 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:11.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:11 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:11.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:11 smithi032 ceph-mon[29357]: Deploying daemon mds.k on smithi121 2024-04-25T00:30:11.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:11 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2734215135' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:12.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:12 smithi121 ceph-mon[32231]: pgmap v265: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:12.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:12 smithi130 ceph-mon[32525]: pgmap v265: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:12.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:12 smithi032 ceph-mon[29357]: pgmap v265: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:13.299 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:30:13.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.414+0000 7fce59d70640 1 -- 172.21.15.32:0/3252086476 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fce541029a0 msgr2=0x7fce54102e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:13.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.414+0000 7fce59d70640 1 --2- 172.21.15.32:0/3252086476 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fce541029a0 0x7fce54102e20 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7fce48009900 tx=0x7fce4802a410 comp rx=0 tx=0).stop 2024-04-25T00:30:13.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.414+0000 7fce59d70640 1 -- 172.21.15.32:0/3252086476 shutdown_connections 2024-04-25T00:30:13.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.414+0000 7fce59d70640 1 --2- 172.21.15.32:0/3252086476 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fce54103360 0x7fce54109bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:13.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.414+0000 7fce59d70640 1 --2- 172.21.15.32:0/3252086476 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fce541029a0 0x7fce54102e20 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:13.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.414+0000 7fce59d70640 1 --2- 172.21.15.32:0/3252086476 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fce541017b0 0x7fce54101bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:13.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.414+0000 7fce59d70640 1 -- 172.21.15.32:0/3252086476 >> 172.21.15.32:0/3252086476 conn(0x7fce540fcd40 msgr2=0x7fce540ff160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:13.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.414+0000 7fce59d70640 1 -- 172.21.15.32:0/3252086476 shutdown_connections 2024-04-25T00:30:13.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.414+0000 7fce59d70640 1 -- 172.21.15.32:0/3252086476 wait complete. 2024-04-25T00:30:13.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.415+0000 7fce59d70640 1 Processor -- start 2024-04-25T00:30:13.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.415+0000 7fce59d70640 1 -- start start 2024-04-25T00:30:13.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.415+0000 7fce59d70640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fce541017b0 0x7fce54197a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:13.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.415+0000 7fce59d70640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fce541029a0 0x7fce54197f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:13.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.415+0000 7fce59d70640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fce54103360 0x7fce5419efb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:13.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.415+0000 7fce59d70640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fce5410d410 con 0x7fce541017b0 2024-04-25T00:30:13.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.415+0000 7fce59d70640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fce5410d290 con 0x7fce54103360 2024-04-25T00:30:13.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.415+0000 7fce59d70640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fce5410d590 con 0x7fce541029a0 2024-04-25T00:30:13.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.415+0000 7fce537fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fce541017b0 0x7fce54197a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:13.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.415+0000 7fce537fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fce541017b0 0x7fce54197a00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:36522/0 (socket says 172.21.15.32:36522) 2024-04-25T00:30:13.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.415+0000 7fce537fe640 1 -- 172.21.15.32:0/2512578128 learned_addr learned my addr 172.21.15.32:0/2512578128 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:13.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.415+0000 7fce53fff640 1 --2- 172.21.15.32:0/2512578128 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fce54103360 0x7fce5419efb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:13.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.415+0000 7fce52ffd640 1 --2- 172.21.15.32:0/2512578128 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fce541029a0 0x7fce54197f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:13.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.416+0000 7fce537fe640 1 -- 172.21.15.32:0/2512578128 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fce541029a0 msgr2=0x7fce54197f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:13.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.416+0000 7fce537fe640 1 --2- 172.21.15.32:0/2512578128 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fce541029a0 0x7fce54197f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:13.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.416+0000 7fce537fe640 1 -- 172.21.15.32:0/2512578128 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fce54103360 msgr2=0x7fce5419efb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:13.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.416+0000 7fce537fe640 1 --2- 172.21.15.32:0/2512578128 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fce54103360 0x7fce5419efb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:13.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.416+0000 7fce537fe640 1 -- 172.21.15.32:0/2512578128 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fce5419f620 con 0x7fce541017b0 2024-04-25T00:30:13.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.416+0000 7fce52ffd640 1 --2- 172.21.15.32:0/2512578128 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fce541029a0 0x7fce54197f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:30:13.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.416+0000 7fce537fe640 1 --2- 172.21.15.32:0/2512578128 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fce541017b0 0x7fce54197a00 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7fce4400c970 tx=0x7fce4400ce30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:13.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.416+0000 7fce50ff9640 1 -- 172.21.15.32:0/2512578128 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fce44007c10 con 0x7fce541017b0 2024-04-25T00:30:13.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.416+0000 7fce50ff9640 1 -- 172.21.15.32:0/2512578128 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fce44007db0 con 0x7fce541017b0 2024-04-25T00:30:13.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.416+0000 7fce59d70640 1 -- 172.21.15.32:0/2512578128 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fce5419f910 con 0x7fce541017b0 2024-04-25T00:30:13.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.416+0000 7fce50ff9640 1 -- 172.21.15.32:0/2512578128 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fce4400f940 con 0x7fce541017b0 2024-04-25T00:30:13.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.416+0000 7fce59d70640 1 -- 172.21.15.32:0/2512578128 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fce5419fe60 con 0x7fce541017b0 2024-04-25T00:30:13.420 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.417+0000 7fce59d70640 1 -- 172.21.15.32:0/2512578128 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fce54108a70 con 0x7fce541017b0 2024-04-25T00:30:13.420 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.418+0000 7fce50ff9640 1 -- 172.21.15.32:0/2512578128 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7fce4400fae0 con 0x7fce541017b0 2024-04-25T00:30:13.420 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.419+0000 7fce50ff9640 1 --2- 172.21.15.32:0/2512578128 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fce3c0a2b90 0x7fce3c0a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:13.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.419+0000 7fce52ffd640 1 --2- 172.21.15.32:0/2512578128 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fce3c0a2b90 0x7fce3c0a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:13.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.419+0000 7fce50ff9640 1 -- 172.21.15.32:0/2512578128 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7fce440cf0c0 con 0x7fce541017b0 2024-04-25T00:30:13.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.419+0000 7fce52ffd640 1 --2- 172.21.15.32:0/2512578128 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fce3c0a2b90 0x7fce3c0a5050 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7fce48000c00 tx=0x7fce48031400 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:13.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.423+0000 7fce50ff9640 1 -- 172.21.15.32:0/2512578128 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fce4409c4b0 con 0x7fce541017b0 2024-04-25T00:30:13.587 INFO:journalctl@ceph.mds.k.smithi121.stdout:Apr 25 00:30:13 smithi121 systemd[1]: Started Ceph mds.k for cb997156-0299-11ef-bc93-c7b262605968. 2024-04-25T00:30:13.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.650+0000 7fce59d70640 1 -- 172.21.15.32:0/2512578128 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fce5402d170 con 0x7fce541017b0 2024-04-25T00:30:13.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.651+0000 7fce50ff9640 1 -- 172.21.15.32:0/2512578128 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 3 v3) ==== 93+0+2221 (secure 0 0 0) 0x7fce4409bdb0 con 0x7fce541017b0 2024-04-25T00:30:13.655 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:13.655 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":3,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3}],"filesystems":[]} 2024-04-25T00:30:13.656 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 3 2024-04-25T00:30:13.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.653+0000 7fce59d70640 1 -- 172.21.15.32:0/2512578128 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fce3c0a2b90 msgr2=0x7fce3c0a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:13.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.653+0000 7fce59d70640 1 --2- 172.21.15.32:0/2512578128 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fce3c0a2b90 0x7fce3c0a5050 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7fce48000c00 tx=0x7fce48031400 comp rx=0 tx=0).stop 2024-04-25T00:30:13.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.653+0000 7fce59d70640 1 -- 172.21.15.32:0/2512578128 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fce541017b0 msgr2=0x7fce54197a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:13.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.653+0000 7fce59d70640 1 --2- 172.21.15.32:0/2512578128 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fce541017b0 0x7fce54197a00 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7fce4400c970 tx=0x7fce4400ce30 comp rx=0 tx=0).stop 2024-04-25T00:30:13.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.653+0000 7fce59d70640 1 -- 172.21.15.32:0/2512578128 shutdown_connections 2024-04-25T00:30:13.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.653+0000 7fce59d70640 1 --2- 172.21.15.32:0/2512578128 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fce3c0a2b90 0x7fce3c0a5050 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:13.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.653+0000 7fce59d70640 1 --2- 172.21.15.32:0/2512578128 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fce54103360 0x7fce5419efb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:13.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.653+0000 7fce59d70640 1 --2- 172.21.15.32:0/2512578128 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fce541029a0 0x7fce54197f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:13.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.653+0000 7fce59d70640 1 --2- 172.21.15.32:0/2512578128 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fce541017b0 0x7fce54197a00 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:13.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.653+0000 7fce59d70640 1 -- 172.21.15.32:0/2512578128 >> 172.21.15.32:0/2512578128 conn(0x7fce540fcd40 msgr2=0x7fce54105c40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:13.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.653+0000 7fce59d70640 1 -- 172.21.15.32:0/2512578128 shutdown_connections 2024-04-25T00:30:13.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:13.653+0000 7fce59d70640 1 -- 172.21.15.32:0/2512578128 wait complete. 2024-04-25T00:30:14.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:14 smithi032 ceph-mon[29357]: pgmap v266: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:14.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:14 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:14.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:14 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:14.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:14 smithi032 ceph-mon[29357]: mds.? [v2:172.21.15.121:6832/436688740,v1:172.21.15.121:6833/436688740] up:boot 2024-04-25T00:30:14.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:14 smithi032 ceph-mon[29357]: fsmap 2 up:standby 2024-04-25T00:30:14.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:14 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2024-04-25T00:30:14.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:14 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:14.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:14 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:14.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:14 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:14.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:14 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:14.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:14 smithi032 ceph-mon[29357]: Deploying daemon mds.j on smithi032 2024-04-25T00:30:14.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:14 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2512578128' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:14.756 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:14 smithi121 ceph-mon[32231]: pgmap v266: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:14.756 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:14 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:14.756 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:14 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:14.756 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:14 smithi121 ceph-mon[32231]: mds.? [v2:172.21.15.121:6832/436688740,v1:172.21.15.121:6833/436688740] up:boot 2024-04-25T00:30:14.756 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:14 smithi121 ceph-mon[32231]: fsmap 2 up:standby 2024-04-25T00:30:14.756 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:14 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2024-04-25T00:30:14.756 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:14 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:14.756 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:14 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:14.756 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:14 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:14.756 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:14 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:14.756 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:14 smithi121 ceph-mon[32231]: Deploying daemon mds.j on smithi032 2024-04-25T00:30:14.756 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:14 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2512578128' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:14.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:14 smithi130 ceph-mon[32525]: pgmap v266: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:14.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:14 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:14.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:14 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:14.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:14 smithi130 ceph-mon[32525]: mds.? [v2:172.21.15.121:6832/436688740,v1:172.21.15.121:6833/436688740] up:boot 2024-04-25T00:30:14.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:14 smithi130 ceph-mon[32525]: fsmap 2 up:standby 2024-04-25T00:30:14.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:14 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2024-04-25T00:30:14.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:14 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:14.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:14 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:14.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:14 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:14.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:14 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:14.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:14 smithi130 ceph-mon[32525]: Deploying daemon mds.j on smithi032 2024-04-25T00:30:14.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:14 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2512578128' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:15.665 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:30:15.828 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.825+0000 7ff6600ba640 1 -- 172.21.15.32:0/900973409 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff658102240 msgr2=0x7ff658108ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:15.828 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.825+0000 7ff6600ba640 1 --2- 172.21.15.32:0/900973409 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff658102240 0x7ff658108ad0 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7ff650009ca0 tx=0x7ff65002a8c0 comp rx=0 tx=0).stop 2024-04-25T00:30:15.828 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.825+0000 7ff6600ba640 1 -- 172.21.15.32:0/900973409 shutdown_connections 2024-04-25T00:30:15.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.826+0000 7ff6600ba640 1 --2- 172.21.15.32:0/900973409 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff658102240 0x7ff658108ad0 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:15.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.826+0000 7ff6600ba640 1 --2- 172.21.15.32:0/900973409 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff658101880 0x7ff658101d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:15.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.826+0000 7ff6600ba640 1 --2- 172.21.15.32:0/900973409 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff658100690 0x7ff658100ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:15.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.826+0000 7ff6600ba640 1 -- 172.21.15.32:0/900973409 >> 172.21.15.32:0/900973409 conn(0x7ff6580fbc20 msgr2=0x7ff6580fe040 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:15.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.826+0000 7ff6600ba640 1 -- 172.21.15.32:0/900973409 shutdown_connections 2024-04-25T00:30:15.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.826+0000 7ff6600ba640 1 -- 172.21.15.32:0/900973409 wait complete. 2024-04-25T00:30:15.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.826+0000 7ff6600ba640 1 Processor -- start 2024-04-25T00:30:15.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.826+0000 7ff6600ba640 1 -- start start 2024-04-25T00:30:15.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.826+0000 7ff6600ba640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff658100690 0x7ff6581979c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:15.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.826+0000 7ff6600ba640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff658101880 0x7ff658197f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:15.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.826+0000 7ff6600ba640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff658102240 0x7ff65819ef70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:15.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.826+0000 7ff6600ba640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7ff65810c360 con 0x7ff658102240 2024-04-25T00:30:15.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.826+0000 7ff6600ba640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7ff65810c1e0 con 0x7ff658100690 2024-04-25T00:30:15.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.826+0000 7ff6600ba640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7ff65810c4e0 con 0x7ff658101880 2024-04-25T00:30:15.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.827+0000 7ff65e657640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff658102240 0x7ff65819ef70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:15.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.827+0000 7ff65e657640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff658102240 0x7ff65819ef70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:36544/0 (socket says 172.21.15.32:36544) 2024-04-25T00:30:15.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.827+0000 7ff65e657640 1 -- 172.21.15.32:0/946072263 learned_addr learned my addr 172.21.15.32:0/946072263 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:15.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.827+0000 7ff65de56640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff658100690 0x7ff6581979c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:15.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.827+0000 7ff65d655640 1 --2- 172.21.15.32:0/946072263 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff658101880 0x7ff658197f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:15.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.827+0000 7ff65e657640 1 -- 172.21.15.32:0/946072263 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff658101880 msgr2=0x7ff658197f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:15.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.827+0000 7ff65e657640 1 --2- 172.21.15.32:0/946072263 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff658101880 0x7ff658197f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:15.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.827+0000 7ff65e657640 1 -- 172.21.15.32:0/946072263 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff658100690 msgr2=0x7ff6581979c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:15.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.827+0000 7ff65e657640 1 --2- 172.21.15.32:0/946072263 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff658100690 0x7ff6581979c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:15.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.827+0000 7ff65e657640 1 -- 172.21.15.32:0/946072263 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff65819f5e0 con 0x7ff658102240 2024-04-25T00:30:15.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.827+0000 7ff65de56640 1 --2- 172.21.15.32:0/946072263 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff658100690 0x7ff6581979c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:30:15.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.827+0000 7ff65e657640 1 --2- 172.21.15.32:0/946072263 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff658102240 0x7ff65819ef70 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7ff650003ba0 tx=0x7ff650003bd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:15.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.827+0000 7ff64effd640 1 -- 172.21.15.32:0/946072263 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff650004040 con 0x7ff658102240 2024-04-25T00:30:15.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.827+0000 7ff64effd640 1 -- 172.21.15.32:0/946072263 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff6500046c0 con 0x7ff658102240 2024-04-25T00:30:15.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.827+0000 7ff6600ba640 1 -- 172.21.15.32:0/946072263 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff65819f870 con 0x7ff658102240 2024-04-25T00:30:15.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.827+0000 7ff64effd640 1 -- 172.21.15.32:0/946072263 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff65002fea0 con 0x7ff658102240 2024-04-25T00:30:15.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.827+0000 7ff6600ba640 1 -- 172.21.15.32:0/946072263 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff65819fdc0 con 0x7ff658102240 2024-04-25T00:30:15.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.829+0000 7ff6600ba640 1 -- 172.21.15.32:0/946072263 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff65819ffc0 con 0x7ff658102240 2024-04-25T00:30:15.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.830+0000 7ff64effd640 1 -- 172.21.15.32:0/946072263 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7ff65002f610 con 0x7ff658102240 2024-04-25T00:30:15.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.830+0000 7ff64effd640 1 --2- 172.21.15.32:0/946072263 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff6440a2b90 0x7ff6440a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:15.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.831+0000 7ff65de56640 1 --2- 172.21.15.32:0/946072263 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff6440a2b90 0x7ff6440a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:15.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.831+0000 7ff64effd640 1 -- 172.21.15.32:0/946072263 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7ff6500f0360 con 0x7ff658102240 2024-04-25T00:30:15.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.831+0000 7ff65de56640 1 --2- 172.21.15.32:0/946072263 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff6440a2b90 0x7ff6440a5050 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7ff6480058c0 tx=0x7ff64800a2d0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:15.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:15.835+0000 7ff64effd640 1 -- 172.21.15.32:0/946072263 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7ff6500bd760 con 0x7ff658102240 2024-04-25T00:30:15.988 INFO:journalctl@ceph.mds.j.smithi032.stdout:Apr 25 00:30:15 smithi032 systemd[1]: Starting Ceph mds.j for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:30:16.065 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:16.065+0000 7ff6600ba640 1 -- 172.21.15.32:0/946072263 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7ff6581a0350 con 0x7ff658102240 2024-04-25T00:30:16.066 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:16.065+0000 7ff64effd640 1 -- 172.21.15.32:0/946072263 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 3 v3) ==== 93+0+2221 (secure 0 0 0) 0x7ff6500bd060 con 0x7ff658102240 2024-04-25T00:30:16.066 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:16.066 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":3,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3}],"filesystems":[]} 2024-04-25T00:30:16.066 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 3 2024-04-25T00:30:16.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:16.067+0000 7ff6600ba640 1 -- 172.21.15.32:0/946072263 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff6440a2b90 msgr2=0x7ff6440a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:16.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:16.067+0000 7ff6600ba640 1 --2- 172.21.15.32:0/946072263 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff6440a2b90 0x7ff6440a5050 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7ff6480058c0 tx=0x7ff64800a2d0 comp rx=0 tx=0).stop 2024-04-25T00:30:16.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:16.067+0000 7ff6600ba640 1 -- 172.21.15.32:0/946072263 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff658102240 msgr2=0x7ff65819ef70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:16.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:16.067+0000 7ff6600ba640 1 --2- 172.21.15.32:0/946072263 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff658102240 0x7ff65819ef70 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7ff650003ba0 tx=0x7ff650003bd0 comp rx=0 tx=0).stop 2024-04-25T00:30:16.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:16.067+0000 7ff6600ba640 1 -- 172.21.15.32:0/946072263 shutdown_connections 2024-04-25T00:30:16.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:16.067+0000 7ff6600ba640 1 --2- 172.21.15.32:0/946072263 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff6440a2b90 0x7ff6440a5050 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:16.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:16.067+0000 7ff6600ba640 1 --2- 172.21.15.32:0/946072263 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff658102240 0x7ff65819ef70 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:16.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:16.067+0000 7ff6600ba640 1 --2- 172.21.15.32:0/946072263 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff658101880 0x7ff658197f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:16.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:16.067+0000 7ff6600ba640 1 --2- 172.21.15.32:0/946072263 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff658100690 0x7ff6581979c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:16.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:16.067+0000 7ff6600ba640 1 -- 172.21.15.32:0/946072263 >> 172.21.15.32:0/946072263 conn(0x7ff6580fbc20 msgr2=0x7ff6580fd5d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:16.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:16.067+0000 7ff6600ba640 1 -- 172.21.15.32:0/946072263 shutdown_connections 2024-04-25T00:30:16.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:16.067+0000 7ff6600ba640 1 -- 172.21.15.32:0/946072263 wait complete. 2024-04-25T00:30:16.248 INFO:journalctl@ceph.mds.j.smithi032.stdout:Apr 25 00:30:15 smithi032 podman[69921]: 2024-04-25 00:30:15.89197397 +0000 UTC m=+0.019004260 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901 2024-04-25T00:30:16.248 INFO:journalctl@ceph.mds.j.smithi032.stdout:Apr 25 00:30:16 smithi032 podman[69921]: 2024-04-25 00:30:16.014456818 +0000 UTC m=+0.141487092 container create 96ce71e67c0415c22f9ebc82ad29f552940a63c19f88a2ad84807b2e82dd6b33 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-j, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-d2c45d9, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2) 2024-04-25T00:30:16.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:16 smithi032 ceph-mon[29357]: pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:16.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:16 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/946072263' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:16.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:16 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:16.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:16 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:16.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:16 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:16.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:16 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:16.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:16 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:16.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:16 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:16.749 INFO:journalctl@ceph.mds.j.smithi032.stdout:Apr 25 00:30:16 smithi032 podman[69921]: 2024-04-25 00:30:16.250784658 +0000 UTC m=+0.377814935 container init 96ce71e67c0415c22f9ebc82ad29f552940a63c19f88a2ad84807b2e82dd6b33 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-j, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-d2c45d9, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-25T00:30:16.749 INFO:journalctl@ceph.mds.j.smithi032.stdout:Apr 25 00:30:16 smithi032 podman[69921]: 2024-04-25 00:30:16.253519377 +0000 UTC m=+0.380549662 container start 96ce71e67c0415c22f9ebc82ad29f552940a63c19f88a2ad84807b2e82dd6b33 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-j, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, RELEASE=squid-d2c45d9, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-25T00:30:16.749 INFO:journalctl@ceph.mds.j.smithi032.stdout:Apr 25 00:30:16 smithi032 ceph-mds[69942]: -- 172.21.15.32:0/676387327 <== mon.0 v2:172.21.15.32:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5609bb9efa40 con 0x5609bb9c3000 2024-04-25T00:30:16.750 INFO:journalctl@ceph.mds.j.smithi032.stdout:Apr 25 00:30:16 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-j[69938]: starting mds.j at 2024-04-25T00:30:16.750 INFO:journalctl@ceph.mds.j.smithi032.stdout:Apr 25 00:30:16 smithi032 bash[69921]: 96ce71e67c0415c22f9ebc82ad29f552940a63c19f88a2ad84807b2e82dd6b33 2024-04-25T00:30:16.750 INFO:journalctl@ceph.mds.j.smithi032.stdout:Apr 25 00:30:16 smithi032 systemd[1]: Started Ceph mds.j for cb997156-0299-11ef-bc93-c7b262605968. 2024-04-25T00:30:16.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:16 smithi121 ceph-mon[32231]: pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:16.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:16 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/946072263' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:16.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:16 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:16.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:16 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:16.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:16 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:16.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:16 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:16.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:16 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:16.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:16 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:16.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:16 smithi130 ceph-mon[32525]: pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:16.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:16 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/946072263' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:16.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:16 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:16.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:16 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:16.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:16 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:16.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:16 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:16.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:16 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:16.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:16 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:17.758 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:17 smithi032 ceph-mon[29357]: Deploying daemon mds.d on smithi032 2024-04-25T00:30:17.758 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:17 smithi032 ceph-mon[29357]: mds.? [v2:172.21.15.32:6834/3511697670,v1:172.21.15.32:6835/3511697670] up:boot 2024-04-25T00:30:17.758 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:17 smithi032 ceph-mon[29357]: fsmap 3 up:standby 2024-04-25T00:30:17.758 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:17 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2024-04-25T00:30:17.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:17 smithi121 ceph-mon[32231]: Deploying daemon mds.d on smithi032 2024-04-25T00:30:17.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:17 smithi121 ceph-mon[32231]: mds.? [v2:172.21.15.32:6834/3511697670,v1:172.21.15.32:6835/3511697670] up:boot 2024-04-25T00:30:17.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:17 smithi121 ceph-mon[32231]: fsmap 3 up:standby 2024-04-25T00:30:17.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:17 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2024-04-25T00:30:17.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:17 smithi130 ceph-mon[32525]: Deploying daemon mds.d on smithi032 2024-04-25T00:30:17.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:17 smithi130 ceph-mon[32525]: mds.? [v2:172.21.15.32:6834/3511697670,v1:172.21.15.32:6835/3511697670] up:boot 2024-04-25T00:30:17.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:17 smithi130 ceph-mon[32525]: fsmap 3 up:standby 2024-04-25T00:30:17.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:17 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2024-04-25T00:30:18.080 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:30:18.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.201+0000 7fb97be58640 1 -- 172.21.15.32:0/1745638419 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb9741017b0 msgr2=0x7fb974101bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:18.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.201+0000 7fb97be58640 1 --2- 172.21.15.32:0/1745638419 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb9741017b0 0x7fb974101bd0 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7fb970009900 tx=0x7fb97002a410 comp rx=0 tx=0).stop 2024-04-25T00:30:18.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.201+0000 7fb97be58640 1 -- 172.21.15.32:0/1745638419 shutdown_connections 2024-04-25T00:30:18.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.201+0000 7fb97be58640 1 --2- 172.21.15.32:0/1745638419 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb974103360 0x7fb974109bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:18.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.201+0000 7fb97be58640 1 --2- 172.21.15.32:0/1745638419 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb9741029a0 0x7fb974102e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:18.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.201+0000 7fb97be58640 1 --2- 172.21.15.32:0/1745638419 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb9741017b0 0x7fb974101bd0 unknown :-1 s=CLOSED pgs=162 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:18.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.201+0000 7fb97be58640 1 -- 172.21.15.32:0/1745638419 >> 172.21.15.32:0/1745638419 conn(0x7fb9740fcd20 msgr2=0x7fb9740ff160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:18.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.201+0000 7fb97be58640 1 -- 172.21.15.32:0/1745638419 shutdown_connections 2024-04-25T00:30:18.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.201+0000 7fb97be58640 1 -- 172.21.15.32:0/1745638419 wait complete. 2024-04-25T00:30:18.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.202+0000 7fb97be58640 1 Processor -- start 2024-04-25T00:30:18.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.202+0000 7fb97be58640 1 -- start start 2024-04-25T00:30:18.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.202+0000 7fb97be58640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb9741017b0 0x7fb974197b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:18.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.202+0000 7fb97be58640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb9741029a0 0x7fb974198060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:18.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.202+0000 7fb97be58640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb974103360 0x7fb97419f0d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:18.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.202+0000 7fb97be58640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fb97410d450 con 0x7fb974103360 2024-04-25T00:30:18.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.202+0000 7fb97be58640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fb97410d2d0 con 0x7fb9741017b0 2024-04-25T00:30:18.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.202+0000 7fb97be58640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fb97410d5d0 con 0x7fb9741029a0 2024-04-25T00:30:18.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.202+0000 7fb97a3f5640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb974103360 0x7fb97419f0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:18.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.202+0000 7fb97a3f5640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb974103360 0x7fb97419f0d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:36586/0 (socket says 172.21.15.32:36586) 2024-04-25T00:30:18.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.202+0000 7fb97a3f5640 1 -- 172.21.15.32:0/2471189158 learned_addr learned my addr 172.21.15.32:0/2471189158 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:18.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.202+0000 7fb9793f3640 1 --2- 172.21.15.32:0/2471189158 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb9741029a0 0x7fb974198060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:18.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.202+0000 7fb979bf4640 1 --2- 172.21.15.32:0/2471189158 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb9741017b0 0x7fb974197b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:18.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.203+0000 7fb97a3f5640 1 -- 172.21.15.32:0/2471189158 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb9741029a0 msgr2=0x7fb974198060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:18.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.203+0000 7fb97a3f5640 1 --2- 172.21.15.32:0/2471189158 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb9741029a0 0x7fb974198060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:18.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.203+0000 7fb97a3f5640 1 -- 172.21.15.32:0/2471189158 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb9741017b0 msgr2=0x7fb974197b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:18.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.203+0000 7fb97a3f5640 1 --2- 172.21.15.32:0/2471189158 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb9741017b0 0x7fb974197b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:18.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.203+0000 7fb97a3f5640 1 -- 172.21.15.32:0/2471189158 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb97419f740 con 0x7fb974103360 2024-04-25T00:30:18.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.203+0000 7fb979bf4640 1 --2- 172.21.15.32:0/2471189158 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb9741017b0 0x7fb974197b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:30:18.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.203+0000 7fb97a3f5640 1 --2- 172.21.15.32:0/2471189158 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb974103360 0x7fb97419f0d0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7fb96c00cad0 tx=0x7fb96c00cf90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:18.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.203+0000 7fb96affd640 1 -- 172.21.15.32:0/2471189158 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb96c00ae20 con 0x7fb974103360 2024-04-25T00:30:18.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.203+0000 7fb97be58640 1 -- 172.21.15.32:0/2471189158 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb97419fa30 con 0x7fb974103360 2024-04-25T00:30:18.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.203+0000 7fb96affd640 1 -- 172.21.15.32:0/2471189158 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb96c005020 con 0x7fb974103360 2024-04-25T00:30:18.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.203+0000 7fb96affd640 1 -- 172.21.15.32:0/2471189158 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb96c00f6f0 con 0x7fb974103360 2024-04-25T00:30:18.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.203+0000 7fb97be58640 1 -- 172.21.15.32:0/2471189158 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb97419ff30 con 0x7fb974103360 2024-04-25T00:30:18.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.204+0000 7fb97be58640 1 -- 172.21.15.32:0/2471189158 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb974108ad0 con 0x7fb974103360 2024-04-25T00:30:18.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.205+0000 7fb96affd640 1 -- 172.21.15.32:0/2471189158 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7fb96c0051d0 con 0x7fb974103360 2024-04-25T00:30:18.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.206+0000 7fb96affd640 1 --2- 172.21.15.32:0/2471189158 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb9600a2b90 0x7fb9600a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:18.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.206+0000 7fb979bf4640 1 --2- 172.21.15.32:0/2471189158 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb9600a2b90 0x7fb9600a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:18.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.206+0000 7fb96affd640 1 -- 172.21.15.32:0/2471189158 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7fb96c0cec40 con 0x7fb974103360 2024-04-25T00:30:18.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.206+0000 7fb979bf4640 1 --2- 172.21.15.32:0/2471189158 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb9600a2b90 0x7fb9600a5050 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7fb97002aa00 tx=0x7fb970000bc0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:18.211 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.210+0000 7fb96affd640 1 -- 172.21.15.32:0/2471189158 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fb96c014030 con 0x7fb974103360 2024-04-25T00:30:18.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.446+0000 7fb97be58640 1 -- 172.21.15.32:0/2471189158 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fb97402d170 con 0x7fb974103360 2024-04-25T00:30:18.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.447+0000 7fb96affd640 1 -- 172.21.15.32:0/2471189158 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 4 v4) ==== 93+0+3046 (secure 0 0 0) 0x7fb96c09c040 con 0x7fb974103360 2024-04-25T00:30:18.448 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:18.448 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":4,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3}],"filesystems":[]} 2024-04-25T00:30:18.448 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 4 2024-04-25T00:30:18.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.449+0000 7fb97be58640 1 -- 172.21.15.32:0/2471189158 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb9600a2b90 msgr2=0x7fb9600a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:18.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.449+0000 7fb97be58640 1 --2- 172.21.15.32:0/2471189158 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb9600a2b90 0x7fb9600a5050 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7fb97002aa00 tx=0x7fb970000bc0 comp rx=0 tx=0).stop 2024-04-25T00:30:18.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.449+0000 7fb97be58640 1 -- 172.21.15.32:0/2471189158 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb974103360 msgr2=0x7fb97419f0d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:18.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.449+0000 7fb97be58640 1 --2- 172.21.15.32:0/2471189158 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb974103360 0x7fb97419f0d0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7fb96c00cad0 tx=0x7fb96c00cf90 comp rx=0 tx=0).stop 2024-04-25T00:30:18.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.449+0000 7fb97be58640 1 -- 172.21.15.32:0/2471189158 shutdown_connections 2024-04-25T00:30:18.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.449+0000 7fb97be58640 1 --2- 172.21.15.32:0/2471189158 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb9600a2b90 0x7fb9600a5050 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:18.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.449+0000 7fb97be58640 1 --2- 172.21.15.32:0/2471189158 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb974103360 0x7fb97419f0d0 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:18.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.449+0000 7fb97be58640 1 --2- 172.21.15.32:0/2471189158 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb9741029a0 0x7fb974198060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:18.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.449+0000 7fb97be58640 1 --2- 172.21.15.32:0/2471189158 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb9741017b0 0x7fb974197b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:18.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.449+0000 7fb97be58640 1 -- 172.21.15.32:0/2471189158 >> 172.21.15.32:0/2471189158 conn(0x7fb9740fcd20 msgr2=0x7fb974105ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:18.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.450+0000 7fb97be58640 1 -- 172.21.15.32:0/2471189158 shutdown_connections 2024-04-25T00:30:18.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:18.450+0000 7fb97be58640 1 -- 172.21.15.32:0/2471189158 wait complete. 2024-04-25T00:30:18.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:18 smithi130 ceph-mon[32525]: pgmap v268: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:18.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:18 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2471189158' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:18.739 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:18 smithi032 ceph-mon[29357]: pgmap v268: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:18.739 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:18 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2471189158' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:18.739 INFO:journalctl@ceph.mds.d.smithi032.stdout:Apr 25 00:30:18 smithi032 systemd[1]: Starting Ceph mds.d for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:30:18.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:18 smithi121 ceph-mon[32231]: pgmap v268: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:18.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:18 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2471189158' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:18.996 INFO:journalctl@ceph.mds.d.smithi032.stdout:Apr 25 00:30:18 smithi032 podman[70270]: 2024-04-25 00:30:18.863442542 +0000 UTC m=+0.018898889 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901 2024-04-25T00:30:18.996 INFO:journalctl@ceph.mds.d.smithi032.stdout:Apr 25 00:30:18 smithi032 podman[70270]: 2024-04-25 00:30:18.996191696 +0000 UTC m=+0.151648017 container create 0003cea6ab589bd36a1d02551fe2ceacc8acc7563e540888692a6b17282ecf79 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-d, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-d2c45d9) 2024-04-25T00:30:19.248 INFO:journalctl@ceph.mds.d.smithi032.stdout:Apr 25 00:30:19 smithi032 podman[70270]: 2024-04-25 00:30:19.223740575 +0000 UTC m=+0.379196898 container init 0003cea6ab589bd36a1d02551fe2ceacc8acc7563e540888692a6b17282ecf79 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-d, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=squid-d2c45d9, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0) 2024-04-25T00:30:19.248 INFO:journalctl@ceph.mds.d.smithi032.stdout:Apr 25 00:30:19 smithi032 podman[70270]: 2024-04-25 00:30:19.226755239 +0000 UTC m=+0.382211562 container start 0003cea6ab589bd36a1d02551fe2ceacc8acc7563e540888692a6b17282ecf79 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-d, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=squid-d2c45d9, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-25T00:30:19.748 INFO:journalctl@ceph.mds.d.smithi032.stdout:Apr 25 00:30:19 smithi032 ceph-mds[70289]: -- 172.21.15.32:0/3304236090 <== mon.0 v2:172.21.15.32:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56228dc27a40 con 0x56228dbfb000 2024-04-25T00:30:19.748 INFO:journalctl@ceph.mds.d.smithi032.stdout:Apr 25 00:30:19 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-d[70285]: starting mds.d at 2024-04-25T00:30:19.748 INFO:journalctl@ceph.mds.d.smithi032.stdout:Apr 25 00:30:19 smithi032 bash[70270]: 0003cea6ab589bd36a1d02551fe2ceacc8acc7563e540888692a6b17282ecf79 2024-04-25T00:30:19.748 INFO:journalctl@ceph.mds.d.smithi032.stdout:Apr 25 00:30:19 smithi032 systemd[1]: Started Ceph mds.d for cb997156-0299-11ef-bc93-c7b262605968. 2024-04-25T00:30:20.461 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:30:20.581 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.580+0000 7fe5a4ca7640 1 -- 172.21.15.32:0/205090794 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe5a0101880 msgr2=0x7fe5a0101d00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:20.581 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.580+0000 7fe5a4ca7640 1 --2- 172.21.15.32:0/205090794 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe5a0101880 0x7fe5a0101d00 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7fe594009900 tx=0x7fe59402a410 comp rx=0 tx=0).stop 2024-04-25T00:30:20.581 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.580+0000 7fe5a4ca7640 1 -- 172.21.15.32:0/205090794 shutdown_connections 2024-04-25T00:30:20.581 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.580+0000 7fe5a4ca7640 1 --2- 172.21.15.32:0/205090794 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe5a0102240 0x7fe5a0108ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:20.581 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.580+0000 7fe5a4ca7640 1 --2- 172.21.15.32:0/205090794 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe5a0101880 0x7fe5a0101d00 unknown :-1 s=CLOSED pgs=166 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:20.581 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.580+0000 7fe5a4ca7640 1 --2- 172.21.15.32:0/205090794 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe5a0100690 0x7fe5a0100ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:20.582 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.580+0000 7fe5a4ca7640 1 -- 172.21.15.32:0/205090794 >> 172.21.15.32:0/205090794 conn(0x7fe5a00fbc20 msgr2=0x7fe5a00fe040 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:20.582 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.580+0000 7fe5a4ca7640 1 -- 172.21.15.32:0/205090794 shutdown_connections 2024-04-25T00:30:20.582 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.580+0000 7fe5a4ca7640 1 -- 172.21.15.32:0/205090794 wait complete. 2024-04-25T00:30:20.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.581+0000 7fe5a4ca7640 1 Processor -- start 2024-04-25T00:30:20.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.581+0000 7fe5a4ca7640 1 -- start start 2024-04-25T00:30:20.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.581+0000 7fe5a4ca7640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe5a0100690 0x7fe5a0196930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:20.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.581+0000 7fe5a4ca7640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe5a0101880 0x7fe5a0196e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:20.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.581+0000 7fe5a4ca7640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe5a0102240 0x7fe5a019dee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:20.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.581+0000 7fe5a4ca7640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fe5a010c2f0 con 0x7fe5a0101880 2024-04-25T00:30:20.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.581+0000 7fe5a4ca7640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fe5a010c170 con 0x7fe5a0102240 2024-04-25T00:30:20.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.581+0000 7fe5a4ca7640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fe5a010c470 con 0x7fe5a0100690 2024-04-25T00:30:20.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.581+0000 7fe59dd9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe5a0101880 0x7fe5a0196e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:20.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.581+0000 7fe59dd9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe5a0101880 0x7fe5a0196e70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:36636/0 (socket says 172.21.15.32:36636) 2024-04-25T00:30:20.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.581+0000 7fe59dd9b640 1 -- 172.21.15.32:0/2079611455 learned_addr learned my addr 172.21.15.32:0/2079611455 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:20.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.581+0000 7fe59e59c640 1 --2- 172.21.15.32:0/2079611455 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe5a0100690 0x7fe5a0196930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:20.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.581+0000 7fe59ed9d640 1 --2- 172.21.15.32:0/2079611455 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe5a0102240 0x7fe5a019dee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:20.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.581+0000 7fe59dd9b640 1 -- 172.21.15.32:0/2079611455 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe5a0100690 msgr2=0x7fe5a0196930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:20.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.581+0000 7fe59dd9b640 1 --2- 172.21.15.32:0/2079611455 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe5a0100690 0x7fe5a0196930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:20.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.581+0000 7fe59dd9b640 1 -- 172.21.15.32:0/2079611455 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe5a0102240 msgr2=0x7fe5a019dee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:20.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.581+0000 7fe59dd9b640 1 --2- 172.21.15.32:0/2079611455 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe5a0102240 0x7fe5a019dee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:20.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.581+0000 7fe59dd9b640 1 -- 172.21.15.32:0/2079611455 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe5a019e550 con 0x7fe5a0101880 2024-04-25T00:30:20.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.582+0000 7fe59dd9b640 1 --2- 172.21.15.32:0/2079611455 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe5a0101880 0x7fe5a0196e70 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fe594009230 tx=0x7fe594004360 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:20.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.582+0000 7fe58f7fe640 1 -- 172.21.15.32:0/2079611455 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe59403f030 con 0x7fe5a0101880 2024-04-25T00:30:20.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.582+0000 7fe5a4ca7640 1 -- 172.21.15.32:0/2079611455 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe5a019e7e0 con 0x7fe5a0101880 2024-04-25T00:30:20.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.582+0000 7fe58f7fe640 1 -- 172.21.15.32:0/2079611455 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe594003ce0 con 0x7fe5a0101880 2024-04-25T00:30:20.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.582+0000 7fe58f7fe640 1 -- 172.21.15.32:0/2079611455 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe594035070 con 0x7fe5a0101880 2024-04-25T00:30:20.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.582+0000 7fe5a4ca7640 1 -- 172.21.15.32:0/2079611455 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe5a019ed20 con 0x7fe5a0101880 2024-04-25T00:30:20.586 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.583+0000 7fe5a4ca7640 1 -- 172.21.15.32:0/2079611455 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe5a0107950 con 0x7fe5a0101880 2024-04-25T00:30:20.586 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.584+0000 7fe58f7fe640 1 -- 172.21.15.32:0/2079611455 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7fe594042030 con 0x7fe5a0101880 2024-04-25T00:30:20.586 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.585+0000 7fe58f7fe640 1 --2- 172.21.15.32:0/2079611455 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe5880a2b90 0x7fe5880a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:20.587 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.585+0000 7fe59e59c640 1 --2- 172.21.15.32:0/2079611455 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe5880a2b90 0x7fe5880a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:20.587 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.585+0000 7fe58f7fe640 1 -- 172.21.15.32:0/2079611455 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7fe5940ef5e0 con 0x7fe5a0101880 2024-04-25T00:30:20.587 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.585+0000 7fe59e59c640 1 --2- 172.21.15.32:0/2079611455 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe5880a2b90 0x7fe5880a5050 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7fe590005fd0 tx=0x7fe59000b410 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:20.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:20 smithi121 ceph-mon[32231]: pgmap v269: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:20.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:20 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:20.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:20 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:20.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:20 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:20.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:20 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:20.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:20 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:20.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:20 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:20.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:20 smithi121 ceph-mon[32231]: Deploying daemon mds.e on smithi121 2024-04-25T00:30:20.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:20 smithi121 ceph-mon[32231]: mds.? [v2:172.21.15.32:6836/2212842316,v1:172.21.15.32:6837/2212842316] up:boot 2024-04-25T00:30:20.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:20 smithi121 ceph-mon[32231]: fsmap 4 up:standby 2024-04-25T00:30:20.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:20 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2024-04-25T00:30:20.589 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.589+0000 7fe58f7fe640 1 -- 172.21.15.32:0/2079611455 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fe5940bc9e0 con 0x7fe5a0101880 2024-04-25T00:30:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:20 smithi130 ceph-mon[32525]: pgmap v269: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:20 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:20 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:20 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:20 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:20 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:20 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:20 smithi130 ceph-mon[32525]: Deploying daemon mds.e on smithi121 2024-04-25T00:30:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:20 smithi130 ceph-mon[32525]: mds.? [v2:172.21.15.32:6836/2212842316,v1:172.21.15.32:6837/2212842316] up:boot 2024-04-25T00:30:20.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:20 smithi130 ceph-mon[32525]: fsmap 4 up:standby 2024-04-25T00:30:20.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:20 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2024-04-25T00:30:20.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:20 smithi032 ceph-mon[29357]: pgmap v269: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:20.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:20 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:20.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:20 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:20.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:20 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:20.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:20 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:20.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:20 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:20.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:20 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:20.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:20 smithi032 ceph-mon[29357]: Deploying daemon mds.e on smithi121 2024-04-25T00:30:20.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:20 smithi032 ceph-mon[29357]: mds.? [v2:172.21.15.32:6836/2212842316,v1:172.21.15.32:6837/2212842316] up:boot 2024-04-25T00:30:20.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:20 smithi032 ceph-mon[29357]: fsmap 4 up:standby 2024-04-25T00:30:20.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:20 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2024-04-25T00:30:20.815 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.814+0000 7fe5a4ca7640 1 -- 172.21.15.32:0/2079611455 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fe5a002d340 con 0x7fe5a0101880 2024-04-25T00:30:20.815 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.815+0000 7fe58f7fe640 1 -- 172.21.15.32:0/2079611455 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 5 v5) ==== 93+0+3871 (secure 0 0 0) 0x7fe5940bc2e0 con 0x7fe5a0101880 2024-04-25T00:30:20.816 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:20.816 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":5,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3}],"filesystems":[]} 2024-04-25T00:30:20.816 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 5 2024-04-25T00:30:20.817 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.817+0000 7fe5a4ca7640 1 -- 172.21.15.32:0/2079611455 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe5880a2b90 msgr2=0x7fe5880a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:20.818 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.817+0000 7fe5a4ca7640 1 --2- 172.21.15.32:0/2079611455 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe5880a2b90 0x7fe5880a5050 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7fe590005fd0 tx=0x7fe59000b410 comp rx=0 tx=0).stop 2024-04-25T00:30:20.818 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.817+0000 7fe5a4ca7640 1 -- 172.21.15.32:0/2079611455 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe5a0101880 msgr2=0x7fe5a0196e70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:20.818 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.817+0000 7fe5a4ca7640 1 --2- 172.21.15.32:0/2079611455 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe5a0101880 0x7fe5a0196e70 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fe594009230 tx=0x7fe594004360 comp rx=0 tx=0).stop 2024-04-25T00:30:20.818 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.817+0000 7fe5a4ca7640 1 -- 172.21.15.32:0/2079611455 shutdown_connections 2024-04-25T00:30:20.818 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.817+0000 7fe5a4ca7640 1 --2- 172.21.15.32:0/2079611455 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe5880a2b90 0x7fe5880a5050 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:20.818 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.817+0000 7fe5a4ca7640 1 --2- 172.21.15.32:0/2079611455 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe5a0102240 0x7fe5a019dee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:20.818 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.817+0000 7fe5a4ca7640 1 --2- 172.21.15.32:0/2079611455 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe5a0101880 0x7fe5a0196e70 unknown :-1 s=CLOSED pgs=167 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:20.818 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.817+0000 7fe5a4ca7640 1 --2- 172.21.15.32:0/2079611455 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe5a0100690 0x7fe5a0196930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:20.818 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.817+0000 7fe5a4ca7640 1 -- 172.21.15.32:0/2079611455 >> 172.21.15.32:0/2079611455 conn(0x7fe5a00fbc20 msgr2=0x7fe5a0104b20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:20.818 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.817+0000 7fe5a4ca7640 1 -- 172.21.15.32:0/2079611455 shutdown_connections 2024-04-25T00:30:20.818 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:20.817+0000 7fe5a4ca7640 1 -- 172.21.15.32:0/2079611455 wait complete. 2024-04-25T00:30:21.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:21 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2079611455' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:21.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:21 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2079611455' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:21.810 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:21 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2079611455' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:21.810 INFO:journalctl@ceph.mds.e.smithi121.stdout:Apr 25 00:30:21 smithi121 systemd[1]: Starting Ceph mds.e for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:30:22.066 INFO:journalctl@ceph.mds.e.smithi121.stdout:Apr 25 00:30:21 smithi121 podman[61945]: 2024-04-25 00:30:21.712705845 +0000 UTC m=+0.017413557 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901 2024-04-25T00:30:22.067 INFO:journalctl@ceph.mds.e.smithi121.stdout:Apr 25 00:30:21 smithi121 podman[61945]: 2024-04-25 00:30:21.820800004 +0000 UTC m=+0.125507745 container create 489b0aa7cac675b6ab6b9303fc31dab886ac498c2f04ed26210219a249bcd217 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-e, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=squid-d2c45d9, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-04-25T00:30:22.067 INFO:journalctl@ceph.mds.e.smithi121.stdout:Apr 25 00:30:22 smithi121 podman[61945]: 2024-04-25 00:30:22.040649798 +0000 UTC m=+0.345357498 container init 489b0aa7cac675b6ab6b9303fc31dab886ac498c2f04ed26210219a249bcd217 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-d2c45d9, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux ) 2024-04-25T00:30:22.067 INFO:journalctl@ceph.mds.e.smithi121.stdout:Apr 25 00:30:22 smithi121 podman[61945]: 2024-04-25 00:30:22.043288294 +0000 UTC m=+0.347996002 container start 489b0aa7cac675b6ab6b9303fc31dab886ac498c2f04ed26210219a249bcd217 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-e, GIT_BRANCH=HEAD, RELEASE=squid-d2c45d9, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-25T00:30:22.337 INFO:journalctl@ceph.mds.e.smithi121.stdout:Apr 25 00:30:22 smithi121 ceph-mds[61964]: -- 172.21.15.121:0/2731741490 <== mon.2 v2:172.21.15.121:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f0f6427a40 con 0x55f0f63fa800 2024-04-25T00:30:22.337 INFO:journalctl@ceph.mds.e.smithi121.stdout:Apr 25 00:30:22 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-e[61960]: starting mds.e at 2024-04-25T00:30:22.337 INFO:journalctl@ceph.mds.e.smithi121.stdout:Apr 25 00:30:22 smithi121 bash[61945]: 489b0aa7cac675b6ab6b9303fc31dab886ac498c2f04ed26210219a249bcd217 2024-04-25T00:30:22.338 INFO:journalctl@ceph.mds.e.smithi121.stdout:Apr 25 00:30:22 smithi121 systemd[1]: Started Ceph mds.e for cb997156-0299-11ef-bc93-c7b262605968. 2024-04-25T00:30:22.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:22 smithi130 ceph-mon[32525]: pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:22.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:22 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:22.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:22 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:22.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:22 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:22.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:22 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:22.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:22 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:22.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:22 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:22.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:22 smithi032 ceph-mon[29357]: pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:22.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:22 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:22.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:22 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:22.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:22 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:22.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:22 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:22.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:22 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:22.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:22 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:22.829 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:30:22.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:22 smithi121 ceph-mon[32231]: pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:22.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:22 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:22.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:22 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:22.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:22 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:22.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:22 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:22.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:22 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:22.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:22 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:22.947 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.944+0000 7f7c985b7640 1 -- 172.21.15.32:0/1973189406 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7c900ff810 msgr2=0x7f7c900ffc90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:22.947 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.944+0000 7f7c985b7640 1 --2- 172.21.15.32:0/1973189406 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7c900ff810 0x7f7c900ffc90 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7f7c8c009900 tx=0x7f7c8c02a410 comp rx=0 tx=0).stop 2024-04-25T00:30:22.947 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.945+0000 7f7c985b7640 1 -- 172.21.15.32:0/1973189406 shutdown_connections 2024-04-25T00:30:22.947 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.945+0000 7f7c985b7640 1 --2- 172.21.15.32:0/1973189406 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7c901001d0 0x7f7c90100680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:22.947 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.945+0000 7f7c985b7640 1 --2- 172.21.15.32:0/1973189406 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7c900ff810 0x7f7c900ffc90 unknown :-1 s=CLOSED pgs=168 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:22.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.945+0000 7f7c985b7640 1 --2- 172.21.15.32:0/1973189406 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7c90101d20 0x7f7c90102140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:22.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.945+0000 7f7c985b7640 1 -- 172.21.15.32:0/1973189406 >> 172.21.15.32:0/1973189406 conn(0x7f7c900fb3b0 msgr2=0x7f7c900fd7d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:22.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.945+0000 7f7c985b7640 1 -- 172.21.15.32:0/1973189406 shutdown_connections 2024-04-25T00:30:22.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.945+0000 7f7c985b7640 1 -- 172.21.15.32:0/1973189406 wait complete. 2024-04-25T00:30:22.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.945+0000 7f7c985b7640 1 Processor -- start 2024-04-25T00:30:22.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.945+0000 7f7c985b7640 1 -- start start 2024-04-25T00:30:22.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.945+0000 7f7c985b7640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7c900ff810 0x7f7c90199c90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:22.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.945+0000 7f7c985b7640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7c901001d0 0x7f7c9019a1d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:22.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.945+0000 7f7c985b7640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7c90101d20 0x7f7c901a1230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:22.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.945+0000 7f7c985b7640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f7c9010f610 con 0x7f7c901001d0 2024-04-25T00:30:22.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.945+0000 7f7c985b7640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f7c9010f490 con 0x7f7c900ff810 2024-04-25T00:30:22.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.945+0000 7f7c985b7640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f7c9010f790 con 0x7f7c90101d20 2024-04-25T00:30:22.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.946+0000 7f7c95b52640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7c901001d0 0x7f7c9019a1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:22.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.946+0000 7f7c95b52640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7c901001d0 0x7f7c9019a1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:49474/0 (socket says 172.21.15.32:49474) 2024-04-25T00:30:22.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.946+0000 7f7c95b52640 1 -- 172.21.15.32:0/727619650 learned_addr learned my addr 172.21.15.32:0/727619650 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:22.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.946+0000 7f7c96b54640 1 --2- 172.21.15.32:0/727619650 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7c90101d20 0x7f7c901a1230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:22.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.946+0000 7f7c96353640 1 --2- 172.21.15.32:0/727619650 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7c900ff810 0x7f7c90199c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:22.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.946+0000 7f7c95b52640 1 -- 172.21.15.32:0/727619650 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7c90101d20 msgr2=0x7f7c901a1230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:22.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.946+0000 7f7c95b52640 1 --2- 172.21.15.32:0/727619650 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7c90101d20 0x7f7c901a1230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:22.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.946+0000 7f7c95b52640 1 -- 172.21.15.32:0/727619650 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7c900ff810 msgr2=0x7f7c90199c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:22.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.946+0000 7f7c95b52640 1 --2- 172.21.15.32:0/727619650 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7c900ff810 0x7f7c90199c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:22.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.946+0000 7f7c95b52640 1 -- 172.21.15.32:0/727619650 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7c901a18d0 con 0x7f7c901001d0 2024-04-25T00:30:22.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.946+0000 7f7c95b52640 1 --2- 172.21.15.32:0/727619650 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7c901001d0 0x7f7c9019a1d0 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f7c8c031610 tx=0x7f7c8c004270 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:22.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.946+0000 7f7c877fe640 1 -- 172.21.15.32:0/727619650 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7c8c03f030 con 0x7f7c901001d0 2024-04-25T00:30:22.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.946+0000 7f7c985b7640 1 -- 172.21.15.32:0/727619650 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7c901a1b60 con 0x7f7c901001d0 2024-04-25T00:30:22.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.946+0000 7f7c877fe640 1 -- 172.21.15.32:0/727619650 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7c8c003ad0 con 0x7f7c901001d0 2024-04-25T00:30:22.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.946+0000 7f7c877fe640 1 -- 172.21.15.32:0/727619650 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7c8c035070 con 0x7f7c901001d0 2024-04-25T00:30:22.950 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.946+0000 7f7c985b7640 1 -- 172.21.15.32:0/727619650 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7c901a20a0 con 0x7f7c901001d0 2024-04-25T00:30:22.950 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.947+0000 7f7c985b7640 1 -- 172.21.15.32:0/727619650 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7c9010b7e0 con 0x7f7c901001d0 2024-04-25T00:30:22.950 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.948+0000 7f7c877fe640 1 -- 172.21.15.32:0/727619650 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f7c8c042030 con 0x7f7c901001d0 2024-04-25T00:30:22.951 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.949+0000 7f7c877fe640 1 --2- 172.21.15.32:0/727619650 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f7c7c0a2b90 0x7f7c7c0a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:22.951 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.949+0000 7f7c96353640 1 --2- 172.21.15.32:0/727619650 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f7c7c0a2b90 0x7f7c7c0a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:22.951 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.949+0000 7f7c877fe640 1 -- 172.21.15.32:0/727619650 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7f7c8c0ef310 con 0x7f7c901001d0 2024-04-25T00:30:22.951 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.950+0000 7f7c96353640 1 --2- 172.21.15.32:0/727619650 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f7c7c0a2b90 0x7f7c7c0a5050 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f7c9019ab60 tx=0x7f7c80005f90 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:22.954 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:22.954+0000 7f7c877fe640 1 -- 172.21.15.32:0/727619650 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f7c8c0bc020 con 0x7f7c901001d0 2024-04-25T00:30:23.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:23.178+0000 7f7c985b7640 1 -- 172.21.15.32:0/727619650 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f7c90062340 con 0x7f7c901001d0 2024-04-25T00:30:23.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:23.179+0000 7f7c877fe640 1 -- 172.21.15.32:0/727619650 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 6 v6) ==== 93+0+4696 (secure 0 0 0) 0x7f7c8c0c0320 con 0x7f7c901001d0 2024-04-25T00:30:23.180 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:23.180 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":6,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24443,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6}],"filesystems":[]} 2024-04-25T00:30:23.180 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 6 2024-04-25T00:30:23.181 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:23.181+0000 7f7c985b7640 1 -- 172.21.15.32:0/727619650 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f7c7c0a2b90 msgr2=0x7f7c7c0a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:23.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:23.181+0000 7f7c985b7640 1 --2- 172.21.15.32:0/727619650 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f7c7c0a2b90 0x7f7c7c0a5050 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f7c9019ab60 tx=0x7f7c80005f90 comp rx=0 tx=0).stop 2024-04-25T00:30:23.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:23.181+0000 7f7c985b7640 1 -- 172.21.15.32:0/727619650 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7c901001d0 msgr2=0x7f7c9019a1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:23.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:23.181+0000 7f7c985b7640 1 --2- 172.21.15.32:0/727619650 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7c901001d0 0x7f7c9019a1d0 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f7c8c031610 tx=0x7f7c8c004270 comp rx=0 tx=0).stop 2024-04-25T00:30:23.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:23.181+0000 7f7c985b7640 1 -- 172.21.15.32:0/727619650 shutdown_connections 2024-04-25T00:30:23.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:23.181+0000 7f7c985b7640 1 --2- 172.21.15.32:0/727619650 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f7c7c0a2b90 0x7f7c7c0a5050 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:23.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:23.181+0000 7f7c985b7640 1 --2- 172.21.15.32:0/727619650 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7c90101d20 0x7f7c901a1230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:23.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:23.181+0000 7f7c985b7640 1 --2- 172.21.15.32:0/727619650 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7c901001d0 0x7f7c9019a1d0 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:23.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:23.181+0000 7f7c985b7640 1 --2- 172.21.15.32:0/727619650 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7c900ff810 0x7f7c90199c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:23.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:23.181+0000 7f7c985b7640 1 -- 172.21.15.32:0/727619650 >> 172.21.15.32:0/727619650 conn(0x7f7c900fb3b0 msgr2=0x7f7c900fca80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:23.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:23.181+0000 7f7c985b7640 1 -- 172.21.15.32:0/727619650 shutdown_connections 2024-04-25T00:30:23.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:23.181+0000 7f7c985b7640 1 -- 172.21.15.32:0/727619650 wait complete. 2024-04-25T00:30:23.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:23 smithi032 ceph-mon[29357]: Deploying daemon mds.f on smithi130 2024-04-25T00:30:23.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:23 smithi032 ceph-mon[29357]: mds.? [v2:172.21.15.121:6834/271690252,v1:172.21.15.121:6835/271690252] up:boot 2024-04-25T00:30:23.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:23 smithi032 ceph-mon[29357]: fsmap 5 up:standby 2024-04-25T00:30:23.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:23 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-04-25T00:30:23.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:23 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/727619650' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:23.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:23 smithi130 ceph-mon[32525]: Deploying daemon mds.f on smithi130 2024-04-25T00:30:23.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:23 smithi130 ceph-mon[32525]: mds.? [v2:172.21.15.121:6834/271690252,v1:172.21.15.121:6835/271690252] up:boot 2024-04-25T00:30:23.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:23 smithi130 ceph-mon[32525]: fsmap 5 up:standby 2024-04-25T00:30:23.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:23 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-04-25T00:30:23.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:23 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/727619650' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:23.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:23 smithi121 ceph-mon[32231]: Deploying daemon mds.f on smithi130 2024-04-25T00:30:23.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:23 smithi121 ceph-mon[32231]: mds.? [v2:172.21.15.121:6834/271690252,v1:172.21.15.121:6835/271690252] up:boot 2024-04-25T00:30:23.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:23 smithi121 ceph-mon[32231]: fsmap 5 up:standby 2024-04-25T00:30:23.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:23 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-04-25T00:30:23.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:23 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/727619650' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:24.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:24 smithi130 ceph-mon[32525]: pgmap v271: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:24.738 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:24 smithi121 ceph-mon[32231]: pgmap v271: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:24.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:24 smithi032 ceph-mon[29357]: pgmap v271: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:24.810 INFO:journalctl@ceph.mds.f.smithi130.stdout:Apr 25 00:30:24 smithi130 podman[62735]: 2024-04-25 00:30:24.787164448 +0000 UTC m=+0.358380176 container start 39daa494086d752c79c258faaf2b5852116a92d1f857a0cb1c66ce4a8ca8461e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-f, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-d2c45d9, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS) 2024-04-25T00:30:24.810 INFO:journalctl@ceph.mds.f.smithi130.stdout:Apr 25 00:30:24 smithi130 ceph-mds[62754]: -- 172.21.15.130:0/3185057172 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55925e3b7a40 con 0x55925e38a800 2024-04-25T00:30:25.142 INFO:journalctl@ceph.mds.f.smithi130.stdout:Apr 25 00:30:24 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-f[62750]: starting mds.f at 2024-04-25T00:30:25.142 INFO:journalctl@ceph.mds.f.smithi130.stdout:Apr 25 00:30:24 smithi130 bash[62735]: 39daa494086d752c79c258faaf2b5852116a92d1f857a0cb1c66ce4a8ca8461e 2024-04-25T00:30:25.142 INFO:journalctl@ceph.mds.f.smithi130.stdout:Apr 25 00:30:24 smithi130 systemd[1]: Started Ceph mds.f for cb997156-0299-11ef-bc93-c7b262605968. 2024-04-25T00:30:25.194 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:30:25.312 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.308+0000 7fa1bb9f9640 1 -- 172.21.15.32:0/2698596624 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa1b4101310 msgr2=0x7fa1b4101790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:25.312 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.308+0000 7fa1bb9f9640 1 --2- 172.21.15.32:0/2698596624 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa1b4101310 0x7fa1b4101790 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7fa1ac009900 tx=0x7fa1ac02a410 comp rx=0 tx=0).stop 2024-04-25T00:30:25.312 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.309+0000 7fa1bb9f9640 1 -- 172.21.15.32:0/2698596624 shutdown_connections 2024-04-25T00:30:25.312 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.309+0000 7fa1bb9f9640 1 --2- 172.21.15.32:0/2698596624 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa1b4101310 0x7fa1b4101790 unknown :-1 s=CLOSED pgs=170 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:25.312 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.309+0000 7fa1bb9f9640 1 --2- 172.21.15.32:0/2698596624 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa1b4100970 0x7fa1b4100dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:25.312 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.309+0000 7fa1bb9f9640 1 --2- 172.21.15.32:0/2698596624 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa1b4100010 0x7fa1b4100430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:25.312 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.309+0000 7fa1bb9f9640 1 -- 172.21.15.32:0/2698596624 >> 172.21.15.32:0/2698596624 conn(0x7fa1b40fbc30 msgr2=0x7fa1b40fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:25.312 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.309+0000 7fa1bb9f9640 1 -- 172.21.15.32:0/2698596624 shutdown_connections 2024-04-25T00:30:25.312 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.309+0000 7fa1bb9f9640 1 -- 172.21.15.32:0/2698596624 wait complete. 2024-04-25T00:30:25.312 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.309+0000 7fa1bb9f9640 1 Processor -- start 2024-04-25T00:30:25.312 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.309+0000 7fa1bb9f9640 1 -- start start 2024-04-25T00:30:25.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.309+0000 7fa1bb9f9640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa1b4100010 0x7fa1b4199be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:25.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.310+0000 7fa1bb9f9640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa1b4100970 0x7fa1b419a120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:25.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.310+0000 7fa1bb9f9640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa1b4101310 0x7fa1b41a1190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:25.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.310+0000 7fa1bb9f9640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fa1b410f5f0 con 0x7fa1b4100970 2024-04-25T00:30:25.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.310+0000 7fa1bb9f9640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fa1b410f470 con 0x7fa1b4101310 2024-04-25T00:30:25.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.310+0000 7fa1bb9f9640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fa1b410f770 con 0x7fa1b4100010 2024-04-25T00:30:25.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.310+0000 7fa1b8f94640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa1b4100970 0x7fa1b419a120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:25.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.310+0000 7fa1b8f94640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa1b4100970 0x7fa1b419a120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:49488/0 (socket says 172.21.15.32:49488) 2024-04-25T00:30:25.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.310+0000 7fa1b8f94640 1 -- 172.21.15.32:0/2186819127 learned_addr learned my addr 172.21.15.32:0/2186819127 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:25.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.310+0000 7fa1b9795640 1 --2- 172.21.15.32:0/2186819127 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa1b4100010 0x7fa1b4199be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:25.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.310+0000 7fa1b9f96640 1 --2- 172.21.15.32:0/2186819127 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa1b4101310 0x7fa1b41a1190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:25.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.310+0000 7fa1b8f94640 1 -- 172.21.15.32:0/2186819127 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa1b4100010 msgr2=0x7fa1b4199be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:25.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.310+0000 7fa1b8f94640 1 --2- 172.21.15.32:0/2186819127 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa1b4100010 0x7fa1b4199be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:25.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.310+0000 7fa1b8f94640 1 -- 172.21.15.32:0/2186819127 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa1b4101310 msgr2=0x7fa1b41a1190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:25.314 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.310+0000 7fa1b8f94640 1 --2- 172.21.15.32:0/2186819127 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa1b4101310 0x7fa1b41a1190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:25.314 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.310+0000 7fa1b8f94640 1 -- 172.21.15.32:0/2186819127 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa1b41a1800 con 0x7fa1b4100970 2024-04-25T00:30:25.314 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.310+0000 7fa1b8f94640 1 --2- 172.21.15.32:0/2186819127 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa1b4100970 0x7fa1b419a120 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7fa1a800d950 tx=0x7fa1a800de10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:25.314 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.310+0000 7fa1b27fc640 1 -- 172.21.15.32:0/2186819127 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa1a8017030 con 0x7fa1b4100970 2024-04-25T00:30:25.314 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.310+0000 7fa1b27fc640 1 -- 172.21.15.32:0/2186819127 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa1a8004020 con 0x7fa1b4100970 2024-04-25T00:30:25.314 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.310+0000 7fa1bb9f9640 1 -- 172.21.15.32:0/2186819127 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa1b41a1af0 con 0x7fa1b4100970 2024-04-25T00:30:25.314 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.310+0000 7fa1b27fc640 1 -- 172.21.15.32:0/2186819127 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa1a80047a0 con 0x7fa1b4100970 2024-04-25T00:30:25.314 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.310+0000 7fa1bb9f9640 1 -- 172.21.15.32:0/2186819127 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa1b41a2040 con 0x7fa1b4100970 2024-04-25T00:30:25.314 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.311+0000 7fa1bb9f9640 1 -- 172.21.15.32:0/2186819127 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa1b410ad60 con 0x7fa1b4100970 2024-04-25T00:30:25.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.312+0000 7fa1b27fc640 1 -- 172.21.15.32:0/2186819127 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7fa1a80041d0 con 0x7fa1b4100970 2024-04-25T00:30:25.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.313+0000 7fa1b27fc640 1 --2- 172.21.15.32:0/2186819127 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa19c0a2b90 0x7fa19c0a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:25.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.313+0000 7fa1b27fc640 1 -- 172.21.15.32:0/2186819127 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7fa1a80ce4b0 con 0x7fa1b4100970 2024-04-25T00:30:25.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.313+0000 7fa1b9795640 1 --2- 172.21.15.32:0/2186819127 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa19c0a2b90 0x7fa19c0a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:25.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.314+0000 7fa1b9795640 1 --2- 172.21.15.32:0/2186819127 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa19c0a2b90 0x7fa19c0a5050 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7fa1a4005fd0 tx=0x7fa1a4009680 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:25.318 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.318+0000 7fa1b27fc640 1 -- 172.21.15.32:0/2186819127 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fa1a809b8b0 con 0x7fa1b4100970 2024-04-25T00:30:25.544 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.543+0000 7fa1bb9f9640 1 -- 172.21.15.32:0/2186819127 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fa1b402d170 con 0x7fa1b4100970 2024-04-25T00:30:25.544 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.544+0000 7fa1b27fc640 1 -- 172.21.15.32:0/2186819127 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 7 v7) ==== 93+0+5524 (secure 0 0 0) 0x7fa1a809b1b0 con 0x7fa1b4100970 2024-04-25T00:30:25.545 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:25.545 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":7,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24443,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6},{"gid":24445,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":7}],"filesystems":[]} 2024-04-25T00:30:25.545 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 7 2024-04-25T00:30:25.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.546+0000 7fa1bb9f9640 1 -- 172.21.15.32:0/2186819127 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa19c0a2b90 msgr2=0x7fa19c0a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:25.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.546+0000 7fa1bb9f9640 1 --2- 172.21.15.32:0/2186819127 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa19c0a2b90 0x7fa19c0a5050 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7fa1a4005fd0 tx=0x7fa1a4009680 comp rx=0 tx=0).stop 2024-04-25T00:30:25.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.546+0000 7fa1bb9f9640 1 -- 172.21.15.32:0/2186819127 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa1b4100970 msgr2=0x7fa1b419a120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:25.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.546+0000 7fa1bb9f9640 1 --2- 172.21.15.32:0/2186819127 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa1b4100970 0x7fa1b419a120 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7fa1a800d950 tx=0x7fa1a800de10 comp rx=0 tx=0).stop 2024-04-25T00:30:25.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.546+0000 7fa1bb9f9640 1 -- 172.21.15.32:0/2186819127 shutdown_connections 2024-04-25T00:30:25.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.546+0000 7fa1bb9f9640 1 --2- 172.21.15.32:0/2186819127 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa19c0a2b90 0x7fa19c0a5050 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:25.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.546+0000 7fa1bb9f9640 1 --2- 172.21.15.32:0/2186819127 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa1b4101310 0x7fa1b41a1190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:25.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.546+0000 7fa1bb9f9640 1 --2- 172.21.15.32:0/2186819127 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa1b4100970 0x7fa1b419a120 unknown :-1 s=CLOSED pgs=171 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:25.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.546+0000 7fa1bb9f9640 1 --2- 172.21.15.32:0/2186819127 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa1b4100010 0x7fa1b4199be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:25.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.546+0000 7fa1bb9f9640 1 -- 172.21.15.32:0/2186819127 >> 172.21.15.32:0/2186819127 conn(0x7fa1b40fbc30 msgr2=0x7fa1b40fd730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:25.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.546+0000 7fa1bb9f9640 1 -- 172.21.15.32:0/2186819127 shutdown_connections 2024-04-25T00:30:25.548 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:25.546+0000 7fa1bb9f9640 1 -- 172.21.15.32:0/2186819127 wait complete. 2024-04-25T00:30:26.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:25 smithi130 ceph-mon[32525]: pgmap v272: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:26.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:25 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:26.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:25 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:26.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:25 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:26.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:25 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:26.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:25 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:26.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:25 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:26.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:25 smithi130 ceph-mon[32525]: Deploying daemon mds.c on smithi130 2024-04-25T00:30:26.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:25 smithi130 ceph-mon[32525]: mds.? [v2:172.21.15.130:6834/2892666533,v1:172.21.15.130:6835/2892666533] up:boot 2024-04-25T00:30:26.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:25 smithi130 ceph-mon[32525]: fsmap 6 up:standby 2024-04-25T00:30:26.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:25 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-04-25T00:30:26.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:25 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2186819127' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:26.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:25 smithi032 ceph-mon[29357]: pgmap v272: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:26.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:25 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:26.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:25 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:26.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:25 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:26.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:25 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:26.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:25 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:26.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:25 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:26.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:25 smithi032 ceph-mon[29357]: Deploying daemon mds.c on smithi130 2024-04-25T00:30:26.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:25 smithi032 ceph-mon[29357]: mds.? [v2:172.21.15.130:6834/2892666533,v1:172.21.15.130:6835/2892666533] up:boot 2024-04-25T00:30:26.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:25 smithi032 ceph-mon[29357]: fsmap 6 up:standby 2024-04-25T00:30:26.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:25 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-04-25T00:30:26.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:25 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2186819127' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:26.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:25 smithi121 ceph-mon[32231]: pgmap v272: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:26.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:25 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:26.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:25 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:26.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:25 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:26.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:25 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:26.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:25 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:26.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:25 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:26.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:25 smithi121 ceph-mon[32231]: Deploying daemon mds.c on smithi130 2024-04-25T00:30:26.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:25 smithi121 ceph-mon[32231]: mds.? [v2:172.21.15.130:6834/2892666533,v1:172.21.15.130:6835/2892666533] up:boot 2024-04-25T00:30:26.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:25 smithi121 ceph-mon[32231]: fsmap 6 up:standby 2024-04-25T00:30:26.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:25 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-04-25T00:30:26.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:25 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2186819127' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:27.369 INFO:journalctl@ceph.mds.c.smithi130.stdout:Apr 25 00:30:27 smithi130 systemd[1]: Starting Ceph mds.c for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:30:27.370 INFO:journalctl@ceph.mds.c.smithi130.stdout:Apr 25 00:30:27 smithi130 podman[63044]: 2024-04-25 00:30:27.270641583 +0000 UTC m=+0.018655663 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901 2024-04-25T00:30:27.558 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:30:27.619 INFO:journalctl@ceph.mds.c.smithi130.stdout:Apr 25 00:30:27 smithi130 podman[63044]: 2024-04-25 00:30:27.388729321 +0000 UTC m=+0.136743455 container create 73fc15c8c09418ae0a61a9cf9a3803ef3dc18d5f91a6659d86e16fe0e2e2db3d (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-c, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-d2c45d9, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-04-25T00:30:27.619 INFO:journalctl@ceph.mds.c.smithi130.stdout:Apr 25 00:30:27 smithi130 podman[63044]: 2024-04-25 00:30:27.616523504 +0000 UTC m=+0.364537576 container init 73fc15c8c09418ae0a61a9cf9a3803ef3dc18d5f91a6659d86e16fe0e2e2db3d (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-c, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-d2c45d9, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-25T00:30:27.619 INFO:journalctl@ceph.mds.c.smithi130.stdout:Apr 25 00:30:27 smithi130 podman[63044]: 2024-04-25 00:30:27.619164623 +0000 UTC m=+0.367178703 container start 73fc15c8c09418ae0a61a9cf9a3803ef3dc18d5f91a6659d86e16fe0e2e2db3d (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-c, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, RELEASE=squid-d2c45d9, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T00:30:27.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.673+0000 7f12f989e640 1 -- 172.21.15.32:0/4165456981 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f12f4102200 msgr2=0x7f12f41045f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:27.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.673+0000 7f12f989e640 1 --2- 172.21.15.32:0/4165456981 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f12f4102200 0x7f12f41045f0 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f12e4009ca0 tx=0x7f12e402a8c0 comp rx=0 tx=0).stop 2024-04-25T00:30:27.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.673+0000 7f12f989e640 1 -- 172.21.15.32:0/4165456981 shutdown_connections 2024-04-25T00:30:27.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.673+0000 7f12f989e640 1 --2- 172.21.15.32:0/4165456981 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f12f4107460 0x7f12f41098b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:27.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.673+0000 7f12f989e640 1 --2- 172.21.15.32:0/4165456981 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f12f4104b30 0x7f12f4106f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:27.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.673+0000 7f12f989e640 1 --2- 172.21.15.32:0/4165456981 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f12f4102200 0x7f12f41045f0 unknown :-1 s=CLOSED pgs=172 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:27.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.673+0000 7f12f989e640 1 -- 172.21.15.32:0/4165456981 >> 172.21.15.32:0/4165456981 conn(0x7f12f40fbbf0 msgr2=0x7f12f40fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:27.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.673+0000 7f12f989e640 1 -- 172.21.15.32:0/4165456981 shutdown_connections 2024-04-25T00:30:27.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.673+0000 7f12f989e640 1 -- 172.21.15.32:0/4165456981 wait complete. 2024-04-25T00:30:27.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.673+0000 7f12f989e640 1 Processor -- start 2024-04-25T00:30:27.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.674+0000 7f12f989e640 1 -- start start 2024-04-25T00:30:27.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.674+0000 7f12f989e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f12f4102200 0x7f12f4197a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:27.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.674+0000 7f12f989e640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f12f4104b30 0x7f12f4197f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:27.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.674+0000 7f12f989e640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f12f4107460 0x7f12f419eff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:27.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.674+0000 7f12f989e640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f12f410d420 con 0x7f12f4102200 2024-04-25T00:30:27.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.674+0000 7f12f989e640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f12f410d2a0 con 0x7f12f4107460 2024-04-25T00:30:27.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.674+0000 7f12f989e640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f12f410d5a0 con 0x7f12f4104b30 2024-04-25T00:30:27.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.674+0000 7f12f2ffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f12f4102200 0x7f12f4197a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:27.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.674+0000 7f12f2ffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f12f4102200 0x7f12f4197a40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:49492/0 (socket says 172.21.15.32:49492) 2024-04-25T00:30:27.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.674+0000 7f12f2ffd640 1 -- 172.21.15.32:0/2042771133 learned_addr learned my addr 172.21.15.32:0/2042771133 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:27.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.674+0000 7f12f27fc640 1 --2- 172.21.15.32:0/2042771133 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f12f4104b30 0x7f12f4197f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:27.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.674+0000 7f12f37fe640 1 --2- 172.21.15.32:0/2042771133 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f12f4107460 0x7f12f419eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:27.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.674+0000 7f12f2ffd640 1 -- 172.21.15.32:0/2042771133 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f12f4104b30 msgr2=0x7f12f4197f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:27.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.674+0000 7f12f2ffd640 1 --2- 172.21.15.32:0/2042771133 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f12f4104b30 0x7f12f4197f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:27.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.674+0000 7f12f2ffd640 1 -- 172.21.15.32:0/2042771133 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f12f4107460 msgr2=0x7f12f419eff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:27.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.674+0000 7f12f2ffd640 1 --2- 172.21.15.32:0/2042771133 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f12f4107460 0x7f12f419eff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:27.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.674+0000 7f12f2ffd640 1 -- 172.21.15.32:0/2042771133 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f12f419f660 con 0x7f12f4102200 2024-04-25T00:30:27.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.674+0000 7f12f2ffd640 1 --2- 172.21.15.32:0/2042771133 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f12f4102200 0x7f12f4197a40 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f12e4031a40 tx=0x7f12e4031ec0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:27.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.675+0000 7f12f889c640 1 -- 172.21.15.32:0/2042771133 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f12e402faa0 con 0x7f12f4102200 2024-04-25T00:30:27.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.675+0000 7f12f889c640 1 -- 172.21.15.32:0/2042771133 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f12e402fc40 con 0x7f12f4102200 2024-04-25T00:30:27.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.675+0000 7f12f989e640 1 -- 172.21.15.32:0/2042771133 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f12f419f8f0 con 0x7f12f4102200 2024-04-25T00:30:27.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.675+0000 7f12f889c640 1 -- 172.21.15.32:0/2042771133 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f12e4046770 con 0x7f12f4102200 2024-04-25T00:30:27.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.675+0000 7f12f989e640 1 -- 172.21.15.32:0/2042771133 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f12f419fe40 con 0x7f12f4102200 2024-04-25T00:30:27.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.676+0000 7f12f989e640 1 -- 172.21.15.32:0/2042771133 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f12f40fd6e0 con 0x7f12f4102200 2024-04-25T00:30:27.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.677+0000 7f12f889c640 1 -- 172.21.15.32:0/2042771133 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f12e40305c0 con 0x7f12f4102200 2024-04-25T00:30:27.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.677+0000 7f12f889c640 1 --2- 172.21.15.32:0/2042771133 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f12dc0a2b90 0x7f12dc0a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:27.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.678+0000 7f12f889c640 1 -- 172.21.15.32:0/2042771133 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7f12e40f0180 con 0x7f12f4102200 2024-04-25T00:30:27.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.678+0000 7f12f27fc640 1 --2- 172.21.15.32:0/2042771133 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f12dc0a2b90 0x7f12dc0a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:27.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.678+0000 7f12f27fc640 1 --2- 172.21.15.32:0/2042771133 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f12dc0a2b90 0x7f12dc0a5050 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f12f4198f50 tx=0x7f12e800a2d0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:27.682 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.682+0000 7f12f889c640 1 -- 172.21.15.32:0/2042771133 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f12e40bd580 con 0x7f12f4102200 2024-04-25T00:30:27.891 INFO:journalctl@ceph.mds.c.smithi130.stdout:Apr 25 00:30:27 smithi130 ceph-mds[63063]: -- 172.21.15.130:0/930840993 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x560e2d387a40 con 0x560e2d35b000 2024-04-25T00:30:27.892 INFO:journalctl@ceph.mds.c.smithi130.stdout:Apr 25 00:30:27 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-c[63059]: starting mds.c at 2024-04-25T00:30:27.892 INFO:journalctl@ceph.mds.c.smithi130.stdout:Apr 25 00:30:27 smithi130 bash[63044]: 73fc15c8c09418ae0a61a9cf9a3803ef3dc18d5f91a6659d86e16fe0e2e2db3d 2024-04-25T00:30:27.892 INFO:journalctl@ceph.mds.c.smithi130.stdout:Apr 25 00:30:27 smithi130 systemd[1]: Started Ceph mds.c for cb997156-0299-11ef-bc93-c7b262605968. 2024-04-25T00:30:27.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.911+0000 7f12f989e640 1 -- 172.21.15.32:0/2042771133 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f12f402d3c0 con 0x7f12f4102200 2024-04-25T00:30:27.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.911+0000 7f12f889c640 1 -- 172.21.15.32:0/2042771133 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 7 v7) ==== 93+0+5524 (secure 0 0 0) 0x7f12e40c05b0 con 0x7f12f4102200 2024-04-25T00:30:27.912 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:27.912 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":7,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24443,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6},{"gid":24445,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":7}],"filesystems":[]} 2024-04-25T00:30:27.912 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 7 2024-04-25T00:30:27.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.913+0000 7f12f989e640 1 -- 172.21.15.32:0/2042771133 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f12dc0a2b90 msgr2=0x7f12dc0a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:27.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.913+0000 7f12f989e640 1 --2- 172.21.15.32:0/2042771133 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f12dc0a2b90 0x7f12dc0a5050 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f12f4198f50 tx=0x7f12e800a2d0 comp rx=0 tx=0).stop 2024-04-25T00:30:27.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.913+0000 7f12f989e640 1 -- 172.21.15.32:0/2042771133 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f12f4102200 msgr2=0x7f12f4197a40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:27.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.913+0000 7f12f989e640 1 --2- 172.21.15.32:0/2042771133 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f12f4102200 0x7f12f4197a40 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f12e4031a40 tx=0x7f12e4031ec0 comp rx=0 tx=0).stop 2024-04-25T00:30:27.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.913+0000 7f12f989e640 1 -- 172.21.15.32:0/2042771133 shutdown_connections 2024-04-25T00:30:27.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.913+0000 7f12f989e640 1 --2- 172.21.15.32:0/2042771133 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f12dc0a2b90 0x7f12dc0a5050 unknown :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:27.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.913+0000 7f12f989e640 1 --2- 172.21.15.32:0/2042771133 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f12f4107460 0x7f12f419eff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:27.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.913+0000 7f12f989e640 1 --2- 172.21.15.32:0/2042771133 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f12f4104b30 0x7f12f4197f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:27.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.913+0000 7f12f989e640 1 --2- 172.21.15.32:0/2042771133 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f12f4102200 0x7f12f4197a40 unknown :-1 s=CLOSED pgs=173 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:27.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.913+0000 7f12f989e640 1 -- 172.21.15.32:0/2042771133 >> 172.21.15.32:0/2042771133 conn(0x7f12f40fbbf0 msgr2=0x7f12f4102d00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:27.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.914+0000 7f12f989e640 1 -- 172.21.15.32:0/2042771133 shutdown_connections 2024-04-25T00:30:27.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:27.914+0000 7f12f989e640 1 -- 172.21.15.32:0/2042771133 wait complete. 2024-04-25T00:30:28.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:27 smithi032 ceph-mon[29357]: pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:28.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:27 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:28.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:27 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:28.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:27 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:28.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:27 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:28.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:27 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:28.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:27 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:28.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:27 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2042771133' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:28.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:27 smithi121 ceph-mon[32231]: pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:28.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:27 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:28.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:27 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:28.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:27 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:28.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:27 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:28.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:27 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:28.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:27 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:28.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:27 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2042771133' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:28.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:27 smithi130 ceph-mon[32525]: pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:28.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:27 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:28.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:27 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:28.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:27 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:28.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:27 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:28.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:27 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:28.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:27 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:28.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:27 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2042771133' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:29.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:28 smithi121 ceph-mon[32231]: Deploying daemon mds.h on smithi121 2024-04-25T00:30:29.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:28 smithi121 ceph-mon[32231]: mds.? [v2:172.21.15.130:6836/1920876316,v1:172.21.15.130:6837/1920876316] up:boot 2024-04-25T00:30:29.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:28 smithi121 ceph-mon[32231]: fsmap 7 up:standby 2024-04-25T00:30:29.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:28 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-04-25T00:30:29.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:28 smithi032 ceph-mon[29357]: Deploying daemon mds.h on smithi121 2024-04-25T00:30:29.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:28 smithi032 ceph-mon[29357]: mds.? [v2:172.21.15.130:6836/1920876316,v1:172.21.15.130:6837/1920876316] up:boot 2024-04-25T00:30:29.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:28 smithi032 ceph-mon[29357]: fsmap 7 up:standby 2024-04-25T00:30:29.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:28 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-04-25T00:30:29.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:28 smithi130 ceph-mon[32525]: Deploying daemon mds.h on smithi121 2024-04-25T00:30:29.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:28 smithi130 ceph-mon[32525]: mds.? [v2:172.21.15.130:6836/1920876316,v1:172.21.15.130:6837/1920876316] up:boot 2024-04-25T00:30:29.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:28 smithi130 ceph-mon[32525]: fsmap 7 up:standby 2024-04-25T00:30:29.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-04-25T00:30:29.926 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:30:30.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.041+0000 7f2d9668c640 1 -- 172.21.15.32:0/360587636 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2d901029a0 msgr2=0x7f2d90102e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:30.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.041+0000 7f2d9668c640 1 --2- 172.21.15.32:0/360587636 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2d901029a0 0x7f2d90102e20 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f2d84009900 tx=0x7f2d8402a410 comp rx=0 tx=0).stop 2024-04-25T00:30:30.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.041+0000 7f2d9668c640 1 -- 172.21.15.32:0/360587636 shutdown_connections 2024-04-25T00:30:30.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.041+0000 7f2d9668c640 1 --2- 172.21.15.32:0/360587636 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f2d90103360 0x7f2d90109bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:30.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.041+0000 7f2d9668c640 1 --2- 172.21.15.32:0/360587636 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2d901029a0 0x7f2d90102e20 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:30.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.041+0000 7f2d9668c640 1 --2- 172.21.15.32:0/360587636 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2d901017b0 0x7f2d90101bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:30.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.041+0000 7f2d9668c640 1 -- 172.21.15.32:0/360587636 >> 172.21.15.32:0/360587636 conn(0x7f2d900fcd40 msgr2=0x7f2d900ff160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:30.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.041+0000 7f2d9668c640 1 -- 172.21.15.32:0/360587636 shutdown_connections 2024-04-25T00:30:30.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.041+0000 7f2d9668c640 1 -- 172.21.15.32:0/360587636 wait complete. 2024-04-25T00:30:30.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.042+0000 7f2d9668c640 1 Processor -- start 2024-04-25T00:30:30.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.042+0000 7f2d9668c640 1 -- start start 2024-04-25T00:30:30.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.042+0000 7f2d9668c640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f2d901017b0 0x7f2d90197ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:30.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.042+0000 7f2d9668c640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2d901029a0 0x7f2d90198000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:30.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.042+0000 7f2d9668c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2d90103360 0x7f2d9019f070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:30.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.042+0000 7f2d9668c640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f2d9010d410 con 0x7f2d90103360 2024-04-25T00:30:30.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.042+0000 7f2d9668c640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f2d9010d290 con 0x7f2d901017b0 2024-04-25T00:30:30.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.042+0000 7f2d9668c640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f2d9010d590 con 0x7f2d901029a0 2024-04-25T00:30:30.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.043+0000 7f2d8ffff640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f2d901017b0 0x7f2d90197ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:30.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.043+0000 7f2d94c29640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2d90103360 0x7f2d9019f070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:30.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.043+0000 7f2d94c29640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2d90103360 0x7f2d9019f070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:49510/0 (socket says 172.21.15.32:49510) 2024-04-25T00:30:30.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.043+0000 7f2d94c29640 1 -- 172.21.15.32:0/1031009977 learned_addr learned my addr 172.21.15.32:0/1031009977 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:30.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.043+0000 7f2d8ffff640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f2d901017b0 0x7f2d90197ac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:48788/0 (socket says 172.21.15.32:48788) 2024-04-25T00:30:30.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.043+0000 7f2d8f7fe640 1 --2- 172.21.15.32:0/1031009977 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2d901029a0 0x7f2d90198000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:30.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.043+0000 7f2d94c29640 1 -- 172.21.15.32:0/1031009977 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2d901029a0 msgr2=0x7f2d90198000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:30.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.043+0000 7f2d94c29640 1 --2- 172.21.15.32:0/1031009977 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2d901029a0 0x7f2d90198000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:30.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.043+0000 7f2d94c29640 1 -- 172.21.15.32:0/1031009977 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f2d901017b0 msgr2=0x7f2d90197ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:30.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.043+0000 7f2d94c29640 1 --2- 172.21.15.32:0/1031009977 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f2d901017b0 0x7f2d90197ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:30.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.043+0000 7f2d94c29640 1 -- 172.21.15.32:0/1031009977 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2d9019f6e0 con 0x7f2d90103360 2024-04-25T00:30:30.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.043+0000 7f2d94c29640 1 --2- 172.21.15.32:0/1031009977 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2d90103360 0x7f2d9019f070 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f2d8800eb10 tx=0x7f2d8800c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:30.047 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.043+0000 7f2d8d7fa640 1 -- 172.21.15.32:0/1031009977 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2d88004b20 con 0x7f2d90103360 2024-04-25T00:30:30.047 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.043+0000 7f2d9668c640 1 -- 172.21.15.32:0/1031009977 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2d9019f9d0 con 0x7f2d90103360 2024-04-25T00:30:30.047 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.043+0000 7f2d8d7fa640 1 -- 172.21.15.32:0/1031009977 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f2d88004cc0 con 0x7f2d90103360 2024-04-25T00:30:30.047 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.043+0000 7f2d8d7fa640 1 -- 172.21.15.32:0/1031009977 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2d88010760 con 0x7f2d90103360 2024-04-25T00:30:30.047 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.043+0000 7f2d9668c640 1 -- 172.21.15.32:0/1031009977 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2d9019fef0 con 0x7f2d90103360 2024-04-25T00:30:30.047 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.044+0000 7f2d9668c640 1 -- 172.21.15.32:0/1031009977 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2d90108ad0 con 0x7f2d90103360 2024-04-25T00:30:30.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.045+0000 7f2d8d7fa640 1 -- 172.21.15.32:0/1031009977 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f2d88005020 con 0x7f2d90103360 2024-04-25T00:30:30.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.046+0000 7f2d8d7fa640 1 --2- 172.21.15.32:0/1031009977 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f2d780a2b90 0x7f2d780a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:30.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.046+0000 7f2d8d7fa640 1 -- 172.21.15.32:0/1031009977 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7f2d880cf090 con 0x7f2d90103360 2024-04-25T00:30:30.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.046+0000 7f2d8ffff640 1 --2- 172.21.15.32:0/1031009977 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f2d780a2b90 0x7f2d780a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:30.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.047+0000 7f2d8ffff640 1 --2- 172.21.15.32:0/1031009977 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f2d780a2b90 0x7f2d780a5050 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f2d80005fd0 tx=0x7f2d8000b410 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:30.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.051+0000 7f2d8d7fa640 1 -- 172.21.15.32:0/1031009977 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f2d8809c490 con 0x7f2d90103360 2024-04-25T00:30:30.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:29 smithi032 ceph-mon[29357]: pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:30.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.276+0000 7f2d9668c640 1 -- 172.21.15.32:0/1031009977 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f2d90062340 con 0x7f2d90103360 2024-04-25T00:30:30.277 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.276+0000 7f2d8d7fa640 1 -- 172.21.15.32:0/1031009977 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 8 v8) ==== 93+0+6352 (secure 0 0 0) 0x7f2d8809bd90 con 0x7f2d90103360 2024-04-25T00:30:30.277 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:30.277 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":8,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24443,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6},{"gid":24445,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":7},{"gid":24457,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":8}],"filesystems":[]} 2024-04-25T00:30:30.278 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 8 2024-04-25T00:30:30.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.279+0000 7f2d9668c640 1 -- 172.21.15.32:0/1031009977 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f2d780a2b90 msgr2=0x7f2d780a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:30.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.279+0000 7f2d9668c640 1 --2- 172.21.15.32:0/1031009977 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f2d780a2b90 0x7f2d780a5050 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f2d80005fd0 tx=0x7f2d8000b410 comp rx=0 tx=0).stop 2024-04-25T00:30:30.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.279+0000 7f2d9668c640 1 -- 172.21.15.32:0/1031009977 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2d90103360 msgr2=0x7f2d9019f070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:30.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.279+0000 7f2d9668c640 1 --2- 172.21.15.32:0/1031009977 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2d90103360 0x7f2d9019f070 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f2d8800eb10 tx=0x7f2d8800c2d0 comp rx=0 tx=0).stop 2024-04-25T00:30:30.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.279+0000 7f2d9668c640 1 -- 172.21.15.32:0/1031009977 shutdown_connections 2024-04-25T00:30:30.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.279+0000 7f2d9668c640 1 --2- 172.21.15.32:0/1031009977 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f2d780a2b90 0x7f2d780a5050 unknown :-1 s=CLOSED pgs=135 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:30.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.279+0000 7f2d9668c640 1 --2- 172.21.15.32:0/1031009977 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2d90103360 0x7f2d9019f070 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:30.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.279+0000 7f2d9668c640 1 --2- 172.21.15.32:0/1031009977 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2d901029a0 0x7f2d90198000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:30.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.279+0000 7f2d9668c640 1 --2- 172.21.15.32:0/1031009977 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f2d901017b0 0x7f2d90197ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:30.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.279+0000 7f2d9668c640 1 -- 172.21.15.32:0/1031009977 >> 172.21.15.32:0/1031009977 conn(0x7f2d900fcd40 msgr2=0x7f2d90105ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:30.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.279+0000 7f2d9668c640 1 -- 172.21.15.32:0/1031009977 shutdown_connections 2024-04-25T00:30:30.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:30.279+0000 7f2d9668c640 1 -- 172.21.15.32:0/1031009977 wait complete. 2024-04-25T00:30:30.310 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:29 smithi121 ceph-mon[32231]: pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:30.311 INFO:journalctl@ceph.mds.h.smithi121.stdout:Apr 25 00:30:30 smithi121 systemd[1]: Starting Ceph mds.h for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:30:30.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:29 smithi130 ceph-mon[32525]: pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:30.580 INFO:journalctl@ceph.mds.h.smithi121.stdout:Apr 25 00:30:30 smithi121 podman[62258]: 2024-04-25 00:30:30.213463533 +0000 UTC m=+0.017747557 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901 2024-04-25T00:30:30.580 INFO:journalctl@ceph.mds.h.smithi121.stdout:Apr 25 00:30:30 smithi121 podman[62258]: 2024-04-25 00:30:30.318209036 +0000 UTC m=+0.122493083 container create 273b11cef7b658743ab666683522aafac5fe665817fdd44ebe3e69be01a91186 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-h, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=squid-d2c45d9, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-25T00:30:30.580 INFO:journalctl@ceph.mds.h.smithi121.stdout:Apr 25 00:30:30 smithi121 podman[62258]: 2024-04-25 00:30:30.579825297 +0000 UTC m=+0.384109298 container init 273b11cef7b658743ab666683522aafac5fe665817fdd44ebe3e69be01a91186 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-h, ceph=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-d2c45d9, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T00:30:30.837 INFO:journalctl@ceph.mds.h.smithi121.stdout:Apr 25 00:30:30 smithi121 podman[62258]: 2024-04-25 00:30:30.582547451 +0000 UTC m=+0.386831455 container start 273b11cef7b658743ab666683522aafac5fe665817fdd44ebe3e69be01a91186 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-h, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, RELEASE=squid-d2c45d9, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T00:30:30.838 INFO:journalctl@ceph.mds.h.smithi121.stdout:Apr 25 00:30:30 smithi121 ceph-mds[62277]: -- 172.21.15.121:0/3882786863 <== mon.2 v2:172.21.15.121:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d629643a40 con 0x55d629616800 2024-04-25T00:30:30.838 INFO:journalctl@ceph.mds.h.smithi121.stdout:Apr 25 00:30:30 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-h[62273]: starting mds.h at 2024-04-25T00:30:30.838 INFO:journalctl@ceph.mds.h.smithi121.stdout:Apr 25 00:30:30 smithi121 bash[62258]: 273b11cef7b658743ab666683522aafac5fe665817fdd44ebe3e69be01a91186 2024-04-25T00:30:30.838 INFO:journalctl@ceph.mds.h.smithi121.stdout:Apr 25 00:30:30 smithi121 systemd[1]: Started Ceph mds.h for cb997156-0299-11ef-bc93-c7b262605968. 2024-04-25T00:30:31.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:30 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1031009977' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:31.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:30 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:31.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:30 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:31.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:30 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:31.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:30 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:31.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:30 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:31.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:30 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:31.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:30 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1031009977' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:31.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:30 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:31.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:30 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:31.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:30 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:31.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:30 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:31.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:30 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:31.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:30 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:31.385 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:30 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1031009977' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:31.385 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:30 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:31.385 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:30 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:31.385 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:30 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:31.385 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:30 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:31.386 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:30 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:31.386 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:30 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:32.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:31 smithi032 ceph-mon[29357]: Deploying daemon mds.i on smithi130 2024-04-25T00:30:32.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:31 smithi032 ceph-mon[29357]: pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:32.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:31 smithi032 ceph-mon[29357]: mds.? [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] up:boot 2024-04-25T00:30:32.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:31 smithi032 ceph-mon[29357]: fsmap 8 up:standby 2024-04-25T00:30:32.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:31 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-04-25T00:30:32.291 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:30:32.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:31 smithi121 ceph-mon[32231]: Deploying daemon mds.i on smithi130 2024-04-25T00:30:32.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:31 smithi121 ceph-mon[32231]: pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:32.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:31 smithi121 ceph-mon[32231]: mds.? [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] up:boot 2024-04-25T00:30:32.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:31 smithi121 ceph-mon[32231]: fsmap 8 up:standby 2024-04-25T00:30:32.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:31 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-04-25T00:30:32.374 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:31 smithi130 ceph-mon[32525]: Deploying daemon mds.i on smithi130 2024-04-25T00:30:32.374 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:31 smithi130 ceph-mon[32525]: pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:32.375 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:31 smithi130 ceph-mon[32525]: mds.? [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] up:boot 2024-04-25T00:30:32.375 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:31 smithi130 ceph-mon[32525]: fsmap 8 up:standby 2024-04-25T00:30:32.375 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:31 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-04-25T00:30:32.406 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.405+0000 7ff62e212640 1 -- 172.21.15.32:0/1828278443 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff628102250 msgr2=0x7ff628108ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:32.406 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.405+0000 7ff62e212640 1 --2- 172.21.15.32:0/1828278443 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff628102250 0x7ff628108ae0 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7ff620009ca0 tx=0x7ff62002a8c0 comp rx=0 tx=0).stop 2024-04-25T00:30:32.406 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.405+0000 7ff62e212640 1 -- 172.21.15.32:0/1828278443 shutdown_connections 2024-04-25T00:30:32.406 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.405+0000 7ff62e212640 1 --2- 172.21.15.32:0/1828278443 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff628102250 0x7ff628108ae0 unknown :-1 s=CLOSED pgs=176 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:32.407 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.405+0000 7ff62e212640 1 --2- 172.21.15.32:0/1828278443 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff628101890 0x7ff628101d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:32.407 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.405+0000 7ff62e212640 1 --2- 172.21.15.32:0/1828278443 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff6281006a0 0x7ff628100ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:32.407 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.405+0000 7ff62e212640 1 -- 172.21.15.32:0/1828278443 >> 172.21.15.32:0/1828278443 conn(0x7ff6280fbc30 msgr2=0x7ff6280fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:32.407 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.405+0000 7ff62e212640 1 -- 172.21.15.32:0/1828278443 shutdown_connections 2024-04-25T00:30:32.407 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.405+0000 7ff62e212640 1 -- 172.21.15.32:0/1828278443 wait complete. 2024-04-25T00:30:32.407 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.406+0000 7ff62e212640 1 Processor -- start 2024-04-25T00:30:32.409 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.406+0000 7ff62e212640 1 -- start start 2024-04-25T00:30:32.409 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.406+0000 7ff62e212640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff6281006a0 0x7ff628197a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:32.409 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.406+0000 7ff62e212640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff628101890 0x7ff628197f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:32.409 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.406+0000 7ff62e212640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff628102250 0x7ff62819efd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:32.409 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.406+0000 7ff62e212640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7ff62810d4a0 con 0x7ff628102250 2024-04-25T00:30:32.409 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.406+0000 7ff62e212640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7ff62810d320 con 0x7ff6281006a0 2024-04-25T00:30:32.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.406+0000 7ff62e212640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7ff62810d620 con 0x7ff628101890 2024-04-25T00:30:32.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.407+0000 7ff627fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff628102250 0x7ff62819efd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:32.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.407+0000 7ff627fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff628102250 0x7ff62819efd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53782/0 (socket says 172.21.15.32:53782) 2024-04-25T00:30:32.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.407+0000 7ff627fff640 1 -- 172.21.15.32:0/3545858125 learned_addr learned my addr 172.21.15.32:0/3545858125 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:32.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.407+0000 7ff6277fe640 1 --2- 172.21.15.32:0/3545858125 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff6281006a0 0x7ff628197a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:32.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.407+0000 7ff626ffd640 1 --2- 172.21.15.32:0/3545858125 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff628101890 0x7ff628197f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:32.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.407+0000 7ff627fff640 1 -- 172.21.15.32:0/3545858125 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff628101890 msgr2=0x7ff628197f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:32.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.407+0000 7ff627fff640 1 --2- 172.21.15.32:0/3545858125 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff628101890 0x7ff628197f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:32.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.407+0000 7ff627fff640 1 -- 172.21.15.32:0/3545858125 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff6281006a0 msgr2=0x7ff628197a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:32.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.407+0000 7ff627fff640 1 --2- 172.21.15.32:0/3545858125 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff6281006a0 0x7ff628197a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:32.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.407+0000 7ff627fff640 1 -- 172.21.15.32:0/3545858125 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff62819f640 con 0x7ff628102250 2024-04-25T00:30:32.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.407+0000 7ff626ffd640 1 --2- 172.21.15.32:0/3545858125 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff628101890 0x7ff628197f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:30:32.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.407+0000 7ff627fff640 1 --2- 172.21.15.32:0/3545858125 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff628102250 0x7ff62819efd0 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7ff6200319f0 tx=0x7ff62002f450 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:32.411 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.407+0000 7ff624ff9640 1 -- 172.21.15.32:0/3545858125 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff620003c70 con 0x7ff628102250 2024-04-25T00:30:32.411 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.407+0000 7ff624ff9640 1 -- 172.21.15.32:0/3545858125 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff620003e10 con 0x7ff628102250 2024-04-25T00:30:32.411 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.407+0000 7ff62e212640 1 -- 172.21.15.32:0/3545858125 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff62819f8d0 con 0x7ff628102250 2024-04-25T00:30:32.411 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.407+0000 7ff624ff9640 1 -- 172.21.15.32:0/3545858125 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff62002fd30 con 0x7ff628102250 2024-04-25T00:30:32.411 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.407+0000 7ff62e212640 1 -- 172.21.15.32:0/3545858125 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff62819fe20 con 0x7ff628102250 2024-04-25T00:30:32.411 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.408+0000 7ff62e212640 1 -- 172.21.15.32:0/3545858125 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff628107960 con 0x7ff628102250 2024-04-25T00:30:32.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.410+0000 7ff624ff9640 1 -- 172.21.15.32:0/3545858125 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7ff620004040 con 0x7ff628102250 2024-04-25T00:30:32.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.411+0000 7ff624ff9640 1 --2- 172.21.15.32:0/3545858125 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff6100a2b90 0x7ff6100a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:32.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.411+0000 7ff6277fe640 1 --2- 172.21.15.32:0/3545858125 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff6100a2b90 0x7ff6100a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:32.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.411+0000 7ff624ff9640 1 -- 172.21.15.32:0/3545858125 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7ff6200f0310 con 0x7ff628102250 2024-04-25T00:30:32.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.411+0000 7ff6277fe640 1 --2- 172.21.15.32:0/3545858125 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff6100a2b90 0x7ff6100a5050 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7ff618005fd0 tx=0x7ff61800b410 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:32.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.415+0000 7ff624ff9640 1 -- 172.21.15.32:0/3545858125 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7ff6200bd710 con 0x7ff628102250 2024-04-25T00:30:32.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.638+0000 7ff62e212640 1 -- 172.21.15.32:0/3545858125 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7ff62802d170 con 0x7ff628102250 2024-04-25T00:30:32.639 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.638+0000 7ff624ff9640 1 -- 172.21.15.32:0/3545858125 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 9 v9) ==== 93+0+7180 (secure 0 0 0) 0x7ff6200bd010 con 0x7ff628102250 2024-04-25T00:30:32.639 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:32.639 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":9,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24443,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6},{"gid":24445,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":7},{"gid":24457,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":8},{"gid":24476,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":9}],"filesystems":[]} 2024-04-25T00:30:32.639 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 9 2024-04-25T00:30:32.641 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.640+0000 7ff62e212640 1 -- 172.21.15.32:0/3545858125 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff6100a2b90 msgr2=0x7ff6100a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:32.641 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.640+0000 7ff62e212640 1 --2- 172.21.15.32:0/3545858125 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff6100a2b90 0x7ff6100a5050 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7ff618005fd0 tx=0x7ff61800b410 comp rx=0 tx=0).stop 2024-04-25T00:30:32.641 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.640+0000 7ff62e212640 1 -- 172.21.15.32:0/3545858125 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff628102250 msgr2=0x7ff62819efd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:32.641 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.640+0000 7ff62e212640 1 --2- 172.21.15.32:0/3545858125 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff628102250 0x7ff62819efd0 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7ff6200319f0 tx=0x7ff62002f450 comp rx=0 tx=0).stop 2024-04-25T00:30:32.641 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.640+0000 7ff62e212640 1 -- 172.21.15.32:0/3545858125 shutdown_connections 2024-04-25T00:30:32.641 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.640+0000 7ff62e212640 1 --2- 172.21.15.32:0/3545858125 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff6100a2b90 0x7ff6100a5050 unknown :-1 s=CLOSED pgs=137 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:32.641 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.640+0000 7ff62e212640 1 --2- 172.21.15.32:0/3545858125 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff628102250 0x7ff62819efd0 unknown :-1 s=CLOSED pgs=177 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:32.641 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.640+0000 7ff62e212640 1 --2- 172.21.15.32:0/3545858125 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff628101890 0x7ff628197f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:32.641 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.640+0000 7ff62e212640 1 --2- 172.21.15.32:0/3545858125 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff6281006a0 0x7ff628197a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:32.641 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.640+0000 7ff62e212640 1 -- 172.21.15.32:0/3545858125 >> 172.21.15.32:0/3545858125 conn(0x7ff6280fbc30 msgr2=0x7ff628104b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:32.641 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.640+0000 7ff62e212640 1 -- 172.21.15.32:0/3545858125 shutdown_connections 2024-04-25T00:30:32.642 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:32.640+0000 7ff62e212640 1 -- 172.21.15.32:0/3545858125 wait complete. 2024-04-25T00:30:32.951 INFO:journalctl@ceph.mds.i.smithi130.stdout:Apr 25 00:30:32 smithi130 systemd[1]: Starting Ceph mds.i for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:30:33.212 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:32 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3545858125' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:33.213 INFO:journalctl@ceph.mds.i.smithi130.stdout:Apr 25 00:30:33 smithi130 podman[63354]: 2024-04-25 00:30:32.975791133 +0000 UTC m=+0.018164502 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901 2024-04-25T00:30:33.213 INFO:journalctl@ceph.mds.i.smithi130.stdout:Apr 25 00:30:33 smithi130 podman[63354]: 2024-04-25 00:30:33.077951677 +0000 UTC m=+0.120325089 container create 687d1407a90a13bc57c3a8e86839b75b381d397cc1c2003f533cc5f67e872b20 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-i, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-d2c45d9, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T00:30:33.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:32 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3545858125' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:33.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:32 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3545858125' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:33.641 INFO:journalctl@ceph.mds.i.smithi130.stdout:Apr 25 00:30:33 smithi130 podman[63354]: 2024-04-25 00:30:33.314195525 +0000 UTC m=+0.356568895 container init 687d1407a90a13bc57c3a8e86839b75b381d397cc1c2003f533cc5f67e872b20 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-i, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-d2c45d9, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-04-25T00:30:33.642 INFO:journalctl@ceph.mds.i.smithi130.stdout:Apr 25 00:30:33 smithi130 podman[63354]: 2024-04-25 00:30:33.317079722 +0000 UTC m=+0.359453092 container start 687d1407a90a13bc57c3a8e86839b75b381d397cc1c2003f533cc5f67e872b20 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-i, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-d2c45d9, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux ) 2024-04-25T00:30:33.642 INFO:journalctl@ceph.mds.i.smithi130.stdout:Apr 25 00:30:33 smithi130 ceph-mds[63373]: -- 172.21.15.130:0/712262482 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5621f41c9a40 con 0x5621f419cc00 2024-04-25T00:30:33.642 INFO:journalctl@ceph.mds.i.smithi130.stdout:Apr 25 00:30:33 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-i[63369]: starting mds.i at 2024-04-25T00:30:33.642 INFO:journalctl@ceph.mds.i.smithi130.stdout:Apr 25 00:30:33 smithi130 bash[63354]: 687d1407a90a13bc57c3a8e86839b75b381d397cc1c2003f533cc5f67e872b20 2024-04-25T00:30:33.642 INFO:journalctl@ceph.mds.i.smithi130.stdout:Apr 25 00:30:33 smithi130 systemd[1]: Started Ceph mds.i for cb997156-0299-11ef-bc93-c7b262605968. 2024-04-25T00:30:34.653 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:30:34.677 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:34 smithi032 ceph-mon[29357]: pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:34.677 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:34.677 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:34.677 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:34.677 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:34.677 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:34.677 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:34.677 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:34 smithi032 ceph-mon[29357]: Deploying daemon mds.g on smithi032 2024-04-25T00:30:34.677 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:34 smithi032 ceph-mon[29357]: mds.? [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] up:boot 2024-04-25T00:30:34.678 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:34 smithi032 ceph-mon[29357]: fsmap 9 up:standby 2024-04-25T00:30:34.678 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:34 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-04-25T00:30:34.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.760+0000 7f41b7790640 1 -- 172.21.15.32:0/3642259093 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f41b0101890 msgr2=0x7f41b0101d10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:34.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.760+0000 7f41b7790640 1 --2- 172.21.15.32:0/3642259093 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f41b0101890 0x7f41b0101d10 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f41a0009900 tx=0x7f41a002e800 comp rx=0 tx=0).stop 2024-04-25T00:30:34.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.760+0000 7f41b7790640 1 -- 172.21.15.32:0/3642259093 shutdown_connections 2024-04-25T00:30:34.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.760+0000 7f41b7790640 1 --2- 172.21.15.32:0/3642259093 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f41b0102250 0x7f41b0108ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:34.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.760+0000 7f41b7790640 1 --2- 172.21.15.32:0/3642259093 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f41b0101890 0x7f41b0101d10 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:34.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.760+0000 7f41b7790640 1 --2- 172.21.15.32:0/3642259093 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f41b01006a0 0x7f41b0100ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:34.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.760+0000 7f41b7790640 1 -- 172.21.15.32:0/3642259093 >> 172.21.15.32:0/3642259093 conn(0x7f41b00fbc30 msgr2=0x7f41b00fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:34.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.760+0000 7f41b7790640 1 -- 172.21.15.32:0/3642259093 shutdown_connections 2024-04-25T00:30:34.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.760+0000 7f41b7790640 1 -- 172.21.15.32:0/3642259093 wait complete. 2024-04-25T00:30:34.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.761+0000 7f41b7790640 1 Processor -- start 2024-04-25T00:30:34.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.761+0000 7f41b7790640 1 -- start start 2024-04-25T00:30:34.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.761+0000 7f41b7790640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f41b01006a0 0x7f41b01957d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:34.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.761+0000 7f41b7790640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f41b0101890 0x7f41b0195d10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:34.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.761+0000 7f41b7790640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f41b0102250 0x7f41b019cd80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:34.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.761+0000 7f41b7790640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f41b010b230 con 0x7f41b01006a0 2024-04-25T00:30:34.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.761+0000 7f41b7790640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f41b010b0b0 con 0x7f41b0101890 2024-04-25T00:30:34.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.761+0000 7f41b7790640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f41b010b3b0 con 0x7f41b0102250 2024-04-25T00:30:34.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.761+0000 7f41b552c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f41b01006a0 0x7f41b01957d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:34.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.761+0000 7f41b552c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f41b01006a0 0x7f41b01957d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53790/0 (socket says 172.21.15.32:53790) 2024-04-25T00:30:34.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.761+0000 7f41b552c640 1 -- 172.21.15.32:0/2517701134 learned_addr learned my addr 172.21.15.32:0/2517701134 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:34.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.761+0000 7f41b5d2d640 1 --2- 172.21.15.32:0/2517701134 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f41b0102250 0x7f41b019cd80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:34.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.761+0000 7f41b4d2b640 1 --2- 172.21.15.32:0/2517701134 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f41b0101890 0x7f41b0195d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:34.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.761+0000 7f41b552c640 1 -- 172.21.15.32:0/2517701134 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f41b0102250 msgr2=0x7f41b019cd80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:34.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.761+0000 7f41b552c640 1 --2- 172.21.15.32:0/2517701134 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f41b0102250 0x7f41b019cd80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:34.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.761+0000 7f41b552c640 1 -- 172.21.15.32:0/2517701134 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f41b0101890 msgr2=0x7f41b0195d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:34.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.761+0000 7f41b552c640 1 --2- 172.21.15.32:0/2517701134 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f41b0101890 0x7f41b0195d10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:34.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.761+0000 7f41b552c640 1 -- 172.21.15.32:0/2517701134 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f41b019d3f0 con 0x7f41b01006a0 2024-04-25T00:30:34.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.762+0000 7f41b552c640 1 --2- 172.21.15.32:0/2517701134 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f41b01006a0 0x7f41b01957d0 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f41ac00c970 tx=0x7f41ac00ce30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:34.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.762+0000 7f41a67fc640 1 -- 172.21.15.32:0/2517701134 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f41ac007c10 con 0x7f41b01006a0 2024-04-25T00:30:34.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.762+0000 7f41a67fc640 1 -- 172.21.15.32:0/2517701134 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f41ac007db0 con 0x7f41b01006a0 2024-04-25T00:30:34.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.762+0000 7f41b7790640 1 -- 172.21.15.32:0/2517701134 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f41b019d6e0 con 0x7f41b01006a0 2024-04-25T00:30:34.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.762+0000 7f41a67fc640 1 -- 172.21.15.32:0/2517701134 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f41ac00f940 con 0x7f41b01006a0 2024-04-25T00:30:34.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.762+0000 7f41b7790640 1 -- 172.21.15.32:0/2517701134 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f41b019dc00 con 0x7f41b01006a0 2024-04-25T00:30:34.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.763+0000 7f41b7790640 1 -- 172.21.15.32:0/2517701134 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f41b01078f0 con 0x7f41b01006a0 2024-04-25T00:30:34.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.764+0000 7f41a67fc640 1 -- 172.21.15.32:0/2517701134 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f41ac00fae0 con 0x7f41b01006a0 2024-04-25T00:30:34.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.765+0000 7f41a67fc640 1 --2- 172.21.15.32:0/2517701134 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f419c0a2b90 0x7f419c0a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:34.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.765+0000 7f41b4d2b640 1 --2- 172.21.15.32:0/2517701134 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f419c0a2b90 0x7f419c0a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:34.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.765+0000 7f41a67fc640 1 -- 172.21.15.32:0/2517701134 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7f41ac0cf010 con 0x7f41b01006a0 2024-04-25T00:30:34.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.765+0000 7f41b4d2b640 1 --2- 172.21.15.32:0/2517701134 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f419c0a2b90 0x7f419c0a5050 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f41a0006010 tx=0x7f41a0000bc0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:34.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.770+0000 7f41a67fc640 1 -- 172.21.15.32:0/2517701134 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f41ac09c400 con 0x7f41b01006a0 2024-04-25T00:30:34.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:34 smithi121 ceph-mon[32231]: pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:34.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:34.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:34.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:34.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:34.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:34.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:34.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:34 smithi121 ceph-mon[32231]: Deploying daemon mds.g on smithi032 2024-04-25T00:30:34.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:34 smithi121 ceph-mon[32231]: mds.? [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] up:boot 2024-04-25T00:30:34.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:34 smithi121 ceph-mon[32231]: fsmap 9 up:standby 2024-04-25T00:30:34.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:34 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-04-25T00:30:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:34 smithi130 ceph-mon[32525]: pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:34 smithi130 ceph-mon[32525]: Deploying daemon mds.g on smithi032 2024-04-25T00:30:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:34 smithi130 ceph-mon[32525]: mds.? [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] up:boot 2024-04-25T00:30:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:34 smithi130 ceph-mon[32525]: fsmap 9 up:standby 2024-04-25T00:30:34.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:34 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-04-25T00:30:34.995 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.994+0000 7f41b7790640 1 -- 172.21.15.32:0/2517701134 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f41b002d170 con 0x7f41b01006a0 2024-04-25T00:30:34.995 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.995+0000 7f41a67fc640 1 -- 172.21.15.32:0/2517701134 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 10 v10) ==== 94+0+8010 (secure 0 0 0) 0x7f41ac09bd00 con 0x7f41b01006a0 2024-04-25T00:30:34.995 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:34.996 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":10,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24443,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6},{"gid":24445,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":7},{"gid":24457,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":8},{"gid":24476,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":9},{"gid":24481,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6839/4267125809","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":4267125809},{"type":"v1","addr":"172.21.15.130:6839","nonce":4267125809}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":10}],"filesystems":[]} 2024-04-25T00:30:34.996 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 10 2024-04-25T00:30:34.997 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.997+0000 7f41b7790640 1 -- 172.21.15.32:0/2517701134 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f419c0a2b90 msgr2=0x7f419c0a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:34.997 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.997+0000 7f41b7790640 1 --2- 172.21.15.32:0/2517701134 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f419c0a2b90 0x7f419c0a5050 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f41a0006010 tx=0x7f41a0000bc0 comp rx=0 tx=0).stop 2024-04-25T00:30:34.998 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.997+0000 7f41b7790640 1 -- 172.21.15.32:0/2517701134 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f41b01006a0 msgr2=0x7f41b01957d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:34.998 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.997+0000 7f41b7790640 1 --2- 172.21.15.32:0/2517701134 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f41b01006a0 0x7f41b01957d0 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f41ac00c970 tx=0x7f41ac00ce30 comp rx=0 tx=0).stop 2024-04-25T00:30:34.998 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.997+0000 7f41b7790640 1 -- 172.21.15.32:0/2517701134 shutdown_connections 2024-04-25T00:30:34.998 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.997+0000 7f41b7790640 1 --2- 172.21.15.32:0/2517701134 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f419c0a2b90 0x7f419c0a5050 unknown :-1 s=CLOSED pgs=139 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:34.998 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.997+0000 7f41b7790640 1 --2- 172.21.15.32:0/2517701134 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f41b0102250 0x7f41b019cd80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:34.998 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.997+0000 7f41b7790640 1 --2- 172.21.15.32:0/2517701134 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f41b0101890 0x7f41b0195d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:34.998 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.997+0000 7f41b7790640 1 --2- 172.21.15.32:0/2517701134 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f41b01006a0 0x7f41b01957d0 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:34.998 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.997+0000 7f41b7790640 1 -- 172.21.15.32:0/2517701134 >> 172.21.15.32:0/2517701134 conn(0x7f41b00fbc30 msgr2=0x7f41b0104ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:34.998 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.997+0000 7f41b7790640 1 -- 172.21.15.32:0/2517701134 shutdown_connections 2024-04-25T00:30:34.998 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:34.997+0000 7f41b7790640 1 -- 172.21.15.32:0/2517701134 wait complete. 2024-04-25T00:30:35.566 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:35 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2517701134' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:35.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:35 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2517701134' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:35.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:35 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2517701134' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:35.963 INFO:journalctl@ceph.mds.g.smithi032.stdout:Apr 25 00:30:35 smithi032 systemd[1]: Starting Ceph mds.g for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:30:36.223 INFO:journalctl@ceph.mds.g.smithi032.stdout:Apr 25 00:30:35 smithi032 podman[70868]: 2024-04-25 00:30:35.865954671 +0000 UTC m=+0.018577910 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901 2024-04-25T00:30:36.224 INFO:journalctl@ceph.mds.g.smithi032.stdout:Apr 25 00:30:35 smithi032 podman[70868]: 2024-04-25 00:30:35.969047321 +0000 UTC m=+0.121670535 container create 6f2ee1447833de9d1995ecd9d122c18d0382082b72c3c7774dd2b972497e5e9f (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-g, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=squid-d2c45d9) 2024-04-25T00:30:36.224 INFO:journalctl@ceph.mds.g.smithi032.stdout:Apr 25 00:30:36 smithi032 podman[70868]: 2024-04-25 00:30:36.196844822 +0000 UTC m=+0.349468044 container init 6f2ee1447833de9d1995ecd9d122c18d0382082b72c3c7774dd2b972497e5e9f (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-g, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-d2c45d9) 2024-04-25T00:30:36.224 INFO:journalctl@ceph.mds.g.smithi032.stdout:Apr 25 00:30:36 smithi032 podman[70868]: 2024-04-25 00:30:36.199549745 +0000 UTC m=+0.352172967 container start 6f2ee1447833de9d1995ecd9d122c18d0382082b72c3c7774dd2b972497e5e9f (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-g, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=squid-d2c45d9, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, ceph=True, org.label-schema.vendor=CentOS) 2024-04-25T00:30:36.224 INFO:journalctl@ceph.mds.g.smithi032.stdout:Apr 25 00:30:36 smithi032 ceph-mds[70887]: -- 172.21.15.32:0/103836326 <== mon.0 v2:172.21.15.32:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5608ed87ba40 con 0x5608ed84e800 2024-04-25T00:30:36.473 INFO:journalctl@ceph.mds.g.smithi032.stdout:Apr 25 00:30:36 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-g[70883]: starting mds.g at 2024-04-25T00:30:36.474 INFO:journalctl@ceph.mds.g.smithi032.stdout:Apr 25 00:30:36 smithi032 bash[70868]: 6f2ee1447833de9d1995ecd9d122c18d0382082b72c3c7774dd2b972497e5e9f 2024-04-25T00:30:36.474 INFO:journalctl@ceph.mds.g.smithi032.stdout:Apr 25 00:30:36 smithi032 systemd[1]: Started Ceph mds.g for cb997156-0299-11ef-bc93-c7b262605968. 2024-04-25T00:30:36.474 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:36 smithi032 ceph-mon[29357]: pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:36.474 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:36 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:36.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:36 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:36.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:36 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:36.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:36 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:36.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:36 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:36.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:36 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:36.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:36 smithi121 ceph-mon[32231]: pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:36.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:36 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:36.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:36 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:36.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:36 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:36.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:36 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:36.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:36 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:36.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:36 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:36.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:36 smithi130 ceph-mon[32525]: pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:36.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:36 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:36.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:36 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:36.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:36 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:36.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:36 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:36.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:36 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:36.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:36 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:37.009 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:30:37.117 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.114+0000 7fee1adb7640 1 -- 172.21.15.32:0/1593934348 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fee14100680 msgr2=0x7fee14100aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:37.117 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.114+0000 7fee1adb7640 1 --2- 172.21.15.32:0/1593934348 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fee14100680 0x7fee14100aa0 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7fee04009900 tx=0x7fee0402a410 comp rx=0 tx=0).stop 2024-04-25T00:30:37.117 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.114+0000 7fee1adb7640 1 -- 172.21.15.32:0/1593934348 shutdown_connections 2024-04-25T00:30:37.117 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.114+0000 7fee1adb7640 1 --2- 172.21.15.32:0/1593934348 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fee14102230 0x7fee14108ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:37.117 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.114+0000 7fee1adb7640 1 --2- 172.21.15.32:0/1593934348 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fee14101870 0x7fee14101cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:37.118 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.114+0000 7fee1adb7640 1 --2- 172.21.15.32:0/1593934348 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fee14100680 0x7fee14100aa0 unknown :-1 s=CLOSED pgs=182 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:37.118 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.114+0000 7fee1adb7640 1 -- 172.21.15.32:0/1593934348 >> 172.21.15.32:0/1593934348 conn(0x7fee140fbbf0 msgr2=0x7fee140fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:37.118 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.114+0000 7fee1adb7640 1 -- 172.21.15.32:0/1593934348 shutdown_connections 2024-04-25T00:30:37.118 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.114+0000 7fee1adb7640 1 -- 172.21.15.32:0/1593934348 wait complete. 2024-04-25T00:30:37.118 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.115+0000 7fee1adb7640 1 Processor -- start 2024-04-25T00:30:37.118 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.115+0000 7fee1adb7640 1 -- start start 2024-04-25T00:30:37.118 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.115+0000 7fee1adb7640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fee14100680 0x7fee14197ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:37.118 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.115+0000 7fee1adb7640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fee14101870 0x7fee14197ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:37.118 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.115+0000 7fee1adb7640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fee14102230 0x7fee1419f060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:37.118 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.115+0000 7fee1adb7640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fee1410c350 con 0x7fee14100680 2024-04-25T00:30:37.119 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.115+0000 7fee1adb7640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fee1410c1d0 con 0x7fee14102230 2024-04-25T00:30:37.119 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.115+0000 7fee1adb7640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fee1410c4d0 con 0x7fee14101870 2024-04-25T00:30:37.119 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.115+0000 7fee18b53640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fee14100680 0x7fee14197ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:37.119 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.115+0000 7fee18b53640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fee14100680 0x7fee14197ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53838/0 (socket says 172.21.15.32:53838) 2024-04-25T00:30:37.119 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.115+0000 7fee18b53640 1 -- 172.21.15.32:0/144009342 learned_addr learned my addr 172.21.15.32:0/144009342 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:37.119 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.115+0000 7fee13fff640 1 --2- 172.21.15.32:0/144009342 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fee14101870 0x7fee14197ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:37.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.115+0000 7fee19354640 1 --2- 172.21.15.32:0/144009342 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fee14102230 0x7fee1419f060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:37.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.115+0000 7fee18b53640 1 -- 172.21.15.32:0/144009342 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fee14101870 msgr2=0x7fee14197ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:37.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.115+0000 7fee18b53640 1 --2- 172.21.15.32:0/144009342 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fee14101870 0x7fee14197ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:37.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.115+0000 7fee18b53640 1 -- 172.21.15.32:0/144009342 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fee14102230 msgr2=0x7fee1419f060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:37.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.115+0000 7fee18b53640 1 --2- 172.21.15.32:0/144009342 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fee14102230 0x7fee1419f060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:37.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.115+0000 7fee18b53640 1 -- 172.21.15.32:0/144009342 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fee1419f6d0 con 0x7fee14100680 2024-04-25T00:30:37.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.115+0000 7fee18b53640 1 --2- 172.21.15.32:0/144009342 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fee14100680 0x7fee14197ab0 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7fee04031440 tx=0x7fee04032e20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:37.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.115+0000 7fee11ffb640 1 -- 172.21.15.32:0/144009342 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fee0403f030 con 0x7fee14100680 2024-04-25T00:30:37.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.116+0000 7fee11ffb640 1 -- 172.21.15.32:0/144009342 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fee04003ce0 con 0x7fee14100680 2024-04-25T00:30:37.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.116+0000 7fee1adb7640 1 -- 172.21.15.32:0/144009342 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fee1419f960 con 0x7fee14100680 2024-04-25T00:30:37.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.116+0000 7fee11ffb640 1 -- 172.21.15.32:0/144009342 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fee04035070 con 0x7fee14100680 2024-04-25T00:30:37.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.116+0000 7fee1adb7640 1 -- 172.21.15.32:0/144009342 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fee1419fea0 con 0x7fee14100680 2024-04-25T00:30:37.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.117+0000 7fee1adb7640 1 -- 172.21.15.32:0/144009342 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fee141a00d0 con 0x7fee14100680 2024-04-25T00:30:37.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.118+0000 7fee11ffb640 1 -- 172.21.15.32:0/144009342 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7fee04042030 con 0x7fee14100680 2024-04-25T00:30:37.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.118+0000 7fee11ffb640 1 --2- 172.21.15.32:0/144009342 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fee000a2c10 0x7fee000a50d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:37.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.119+0000 7fee13fff640 1 --2- 172.21.15.32:0/144009342 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fee000a2c10 0x7fee000a50d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:37.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.119+0000 7fee11ffb640 1 -- 172.21.15.32:0/144009342 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7fee040ef250 con 0x7fee14100680 2024-04-25T00:30:37.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.119+0000 7fee13fff640 1 --2- 172.21.15.32:0/144009342 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fee000a2c10 0x7fee000a50d0 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7fedfc005fd0 tx=0x7fedfc005850 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:37.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.123+0000 7fee11ffb640 1 -- 172.21.15.32:0/144009342 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fee040bc650 con 0x7fee14100680 2024-04-25T00:30:37.353 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.353+0000 7fee1adb7640 1 -- 172.21.15.32:0/144009342 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fee141a02a0 con 0x7fee14100680 2024-04-25T00:30:37.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.353+0000 7fee11ffb640 1 -- 172.21.15.32:0/144009342 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 11 v11) ==== 94+0+8833 (secure 0 0 0) 0x7fee040bbf50 con 0x7fee14100680 2024-04-25T00:30:37.355 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:37.355 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":11,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14643,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6839/187492674","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6838","nonce":187492674},{"type":"v1","addr":"172.21.15.32:6839","nonce":187492674}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24443,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6},{"gid":24445,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":7},{"gid":24457,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":8},{"gid":24476,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":9},{"gid":24481,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6839/4267125809","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":4267125809},{"type":"v1","addr":"172.21.15.130:6839","nonce":4267125809}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":10}],"filesystems":[]} 2024-04-25T00:30:37.356 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 11 2024-04-25T00:30:37.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.355+0000 7fee1adb7640 1 -- 172.21.15.32:0/144009342 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fee000a2c10 msgr2=0x7fee000a50d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:37.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.355+0000 7fee1adb7640 1 --2- 172.21.15.32:0/144009342 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fee000a2c10 0x7fee000a50d0 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7fedfc005fd0 tx=0x7fedfc005850 comp rx=0 tx=0).stop 2024-04-25T00:30:37.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.355+0000 7fee1adb7640 1 -- 172.21.15.32:0/144009342 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fee14100680 msgr2=0x7fee14197ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:37.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.355+0000 7fee1adb7640 1 --2- 172.21.15.32:0/144009342 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fee14100680 0x7fee14197ab0 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7fee04031440 tx=0x7fee04032e20 comp rx=0 tx=0).stop 2024-04-25T00:30:37.357 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.355+0000 7fee1adb7640 1 -- 172.21.15.32:0/144009342 shutdown_connections 2024-04-25T00:30:37.357 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.355+0000 7fee1adb7640 1 --2- 172.21.15.32:0/144009342 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fee000a2c10 0x7fee000a50d0 unknown :-1 s=CLOSED pgs=141 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:37.357 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.355+0000 7fee1adb7640 1 --2- 172.21.15.32:0/144009342 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fee14102230 0x7fee1419f060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:37.357 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.355+0000 7fee1adb7640 1 --2- 172.21.15.32:0/144009342 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fee14101870 0x7fee14197ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:37.357 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.355+0000 7fee1adb7640 1 --2- 172.21.15.32:0/144009342 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fee14100680 0x7fee14197ab0 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:37.357 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.355+0000 7fee1adb7640 1 -- 172.21.15.32:0/144009342 >> 172.21.15.32:0/144009342 conn(0x7fee140fbbf0 msgr2=0x7fee140fd640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:37.357 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.356+0000 7fee1adb7640 1 -- 172.21.15.32:0/144009342 shutdown_connections 2024-04-25T00:30:37.357 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:37.356+0000 7fee1adb7640 1 -- 172.21.15.32:0/144009342 wait complete. 2024-04-25T00:30:37.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:37 smithi032 ceph-mon[29357]: Deploying daemon mds.a on smithi032 2024-04-25T00:30:37.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:37 smithi032 ceph-mon[29357]: mds.? [v2:172.21.15.32:6838/187492674,v1:172.21.15.32:6839/187492674] up:boot 2024-04-25T00:30:37.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:37 smithi032 ceph-mon[29357]: fsmap 10 up:standby 2024-04-25T00:30:37.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:37 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2024-04-25T00:30:37.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:37 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/144009342' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:37.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:37 smithi121 ceph-mon[32231]: Deploying daemon mds.a on smithi032 2024-04-25T00:30:37.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:37 smithi121 ceph-mon[32231]: mds.? [v2:172.21.15.32:6838/187492674,v1:172.21.15.32:6839/187492674] up:boot 2024-04-25T00:30:37.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:37 smithi121 ceph-mon[32231]: fsmap 10 up:standby 2024-04-25T00:30:37.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:37 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2024-04-25T00:30:37.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:37 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/144009342' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:37.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:37 smithi130 ceph-mon[32525]: Deploying daemon mds.a on smithi032 2024-04-25T00:30:37.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:37 smithi130 ceph-mon[32525]: mds.? [v2:172.21.15.32:6838/187492674,v1:172.21.15.32:6839/187492674] up:boot 2024-04-25T00:30:37.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:37 smithi130 ceph-mon[32525]: fsmap 10 up:standby 2024-04-25T00:30:37.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:37 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2024-04-25T00:30:37.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:37 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/144009342' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:38.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:38 smithi032 ceph-mon[29357]: pgmap v278: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:38.749 INFO:journalctl@ceph.mds.a.smithi032.stdout:Apr 25 00:30:38 smithi032 systemd[1]: Starting Ceph mds.a for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:30:38.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:38 smithi121 ceph-mon[32231]: pgmap v278: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:38.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:38 smithi130 ceph-mon[32525]: pgmap v278: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:39.168 INFO:journalctl@ceph.mds.a.smithi032.stdout:Apr 25 00:30:38 smithi032 podman[71225]: 2024-04-25 00:30:38.807903692 +0000 UTC m=+0.018939474 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901 2024-04-25T00:30:39.168 INFO:journalctl@ceph.mds.a.smithi032.stdout:Apr 25 00:30:38 smithi032 podman[71225]: 2024-04-25 00:30:38.98198434 +0000 UTC m=+0.193020097 container create cdcac5998a3b00dafdc26c4a08c31af58d2a95367e4cee926dfe020010b6fec3 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-a, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-d2c45d9, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5) 2024-04-25T00:30:39.368 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:30:39.478 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.477+0000 7fc36181c640 1 -- 172.21.15.32:0/2582751055 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc35c1006a0 msgr2=0x7fc35c100ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:39.478 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.477+0000 7fc36181c640 1 --2- 172.21.15.32:0/2582751055 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc35c1006a0 0x7fc35c100ac0 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7fc34c009900 tx=0x7fc34c02a410 comp rx=0 tx=0).stop 2024-04-25T00:30:39.478 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.477+0000 7fc36181c640 1 -- 172.21.15.32:0/2582751055 shutdown_connections 2024-04-25T00:30:39.478 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.477+0000 7fc36181c640 1 --2- 172.21.15.32:0/2582751055 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc35c102250 0x7fc35c108ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:39.478 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.477+0000 7fc36181c640 1 --2- 172.21.15.32:0/2582751055 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc35c101890 0x7fc35c101d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:39.478 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.477+0000 7fc36181c640 1 --2- 172.21.15.32:0/2582751055 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc35c1006a0 0x7fc35c100ac0 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:39.478 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.477+0000 7fc36181c640 1 -- 172.21.15.32:0/2582751055 >> 172.21.15.32:0/2582751055 conn(0x7fc35c0fbc30 msgr2=0x7fc35c0fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:39.478 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.477+0000 7fc36181c640 1 -- 172.21.15.32:0/2582751055 shutdown_connections 2024-04-25T00:30:39.479 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.477+0000 7fc36181c640 1 -- 172.21.15.32:0/2582751055 wait complete. 2024-04-25T00:30:39.479 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.477+0000 7fc36181c640 1 Processor -- start 2024-04-25T00:30:39.479 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.478+0000 7fc36181c640 1 -- start start 2024-04-25T00:30:39.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.478+0000 7fc36181c640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc35c1006a0 0x7fc35c197b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:39.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.478+0000 7fc36181c640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc35c101890 0x7fc35c198080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:39.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.478+0000 7fc36181c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc35c102250 0x7fc35c19f0f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:39.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.478+0000 7fc36181c640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fc35c10c2f0 con 0x7fc35c102250 2024-04-25T00:30:39.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.478+0000 7fc36181c640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fc35c10c170 con 0x7fc35c1006a0 2024-04-25T00:30:39.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.478+0000 7fc36181c640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fc35c10c470 con 0x7fc35c101890 2024-04-25T00:30:39.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.478+0000 7fc35b7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc35c102250 0x7fc35c19f0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:39.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.478+0000 7fc35b7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc35c102250 0x7fc35c19f0f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53874/0 (socket says 172.21.15.32:53874) 2024-04-25T00:30:39.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.478+0000 7fc35b7fe640 1 -- 172.21.15.32:0/1088309851 learned_addr learned my addr 172.21.15.32:0/1088309851 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:39.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.478+0000 7fc35a7fc640 1 --2- 172.21.15.32:0/1088309851 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc35c101890 0x7fc35c198080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:39.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.478+0000 7fc35affd640 1 --2- 172.21.15.32:0/1088309851 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc35c1006a0 0x7fc35c197b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:39.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.478+0000 7fc35b7fe640 1 -- 172.21.15.32:0/1088309851 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc35c101890 msgr2=0x7fc35c198080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:39.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.478+0000 7fc35b7fe640 1 --2- 172.21.15.32:0/1088309851 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc35c101890 0x7fc35c198080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:39.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.478+0000 7fc35b7fe640 1 -- 172.21.15.32:0/1088309851 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc35c1006a0 msgr2=0x7fc35c197b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:39.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.478+0000 7fc35b7fe640 1 --2- 172.21.15.32:0/1088309851 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc35c1006a0 0x7fc35c197b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:39.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.478+0000 7fc35b7fe640 1 -- 172.21.15.32:0/1088309851 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc35c19f760 con 0x7fc35c102250 2024-04-25T00:30:39.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.478+0000 7fc35affd640 1 --2- 172.21.15.32:0/1088309851 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc35c1006a0 0x7fc35c197b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:30:39.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.478+0000 7fc35b7fe640 1 --2- 172.21.15.32:0/1088309851 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc35c102250 0x7fc35c19f0f0 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7fc35400eaa0 tx=0x7fc35400ef60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:39.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.479+0000 7fc36081a640 1 -- 172.21.15.32:0/1088309851 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc35400cdb0 con 0x7fc35c102250 2024-04-25T00:30:39.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.479+0000 7fc36081a640 1 -- 172.21.15.32:0/1088309851 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc354005020 con 0x7fc35c102250 2024-04-25T00:30:39.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.479+0000 7fc36081a640 1 -- 172.21.15.32:0/1088309851 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc3540116b0 con 0x7fc35c102250 2024-04-25T00:30:39.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.479+0000 7fc36181c640 1 -- 172.21.15.32:0/1088309851 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc35c19fa50 con 0x7fc35c102250 2024-04-25T00:30:39.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.479+0000 7fc36181c640 1 -- 172.21.15.32:0/1088309851 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc35c19ffa0 con 0x7fc35c102250 2024-04-25T00:30:39.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.480+0000 7fc36181c640 1 -- 172.21.15.32:0/1088309851 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc35c1a0230 con 0x7fc35c102250 2024-04-25T00:30:39.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.481+0000 7fc36081a640 1 -- 172.21.15.32:0/1088309851 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7fc354005560 con 0x7fc35c102250 2024-04-25T00:30:39.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.481+0000 7fc36081a640 1 --2- 172.21.15.32:0/1088309851 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc3440a2c10 0x7fc3440a50d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:39.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.482+0000 7fc35affd640 1 --2- 172.21.15.32:0/1088309851 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc3440a2c10 0x7fc3440a50d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:39.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.482+0000 7fc36081a640 1 -- 172.21.15.32:0/1088309851 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7fc3540cedc0 con 0x7fc35c102250 2024-04-25T00:30:39.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.482+0000 7fc35affd640 1 --2- 172.21.15.32:0/1088309851 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc3440a2c10 0x7fc3440a50d0 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7fc34c000c00 tx=0x7fc34c031400 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:39.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.487+0000 7fc36081a640 1 -- 172.21.15.32:0/1088309851 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fc354016030 con 0x7fc35c102250 2024-04-25T00:30:39.498 INFO:journalctl@ceph.mds.a.smithi032.stdout:Apr 25 00:30:39 smithi032 podman[71225]: 2024-04-25 00:30:39.285894089 +0000 UTC m=+0.496929845 container init cdcac5998a3b00dafdc26c4a08c31af58d2a95367e4cee926dfe020010b6fec3 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-a, RELEASE=squid-d2c45d9, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True) 2024-04-25T00:30:39.499 INFO:journalctl@ceph.mds.a.smithi032.stdout:Apr 25 00:30:39 smithi032 podman[71225]: 2024-04-25 00:30:39.28876516 +0000 UTC m=+0.499800917 container start cdcac5998a3b00dafdc26c4a08c31af58d2a95367e4cee926dfe020010b6fec3 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-a, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=squid-d2c45d9, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-25T00:30:39.499 INFO:journalctl@ceph.mds.a.smithi032.stdout:Apr 25 00:30:39 smithi032 ceph-mds[71244]: -- 172.21.15.32:0/3438348559 <== mon.0 v2:172.21.15.32:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5612bab67a40 con 0x5612bab3ac00 2024-04-25T00:30:39.499 INFO:journalctl@ceph.mds.a.smithi032.stdout:Apr 25 00:30:39 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-a[71240]: starting mds.a at 2024-04-25T00:30:39.499 INFO:journalctl@ceph.mds.a.smithi032.stdout:Apr 25 00:30:39 smithi032 bash[71225]: cdcac5998a3b00dafdc26c4a08c31af58d2a95367e4cee926dfe020010b6fec3 2024-04-25T00:30:39.499 INFO:journalctl@ceph.mds.a.smithi032.stdout:Apr 25 00:30:39 smithi032 systemd[1]: Started Ceph mds.a for cb997156-0299-11ef-bc93-c7b262605968. 2024-04-25T00:30:39.714 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.713+0000 7fc36181c640 1 -- 172.21.15.32:0/1088309851 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fc35c1a0400 con 0x7fc35c102250 2024-04-25T00:30:39.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.714+0000 7fc36081a640 1 -- 172.21.15.32:0/1088309851 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 12 v12) ==== 94+0+9659 (secure 0 0 0) 0x7fc35409c1c0 con 0x7fc35c102250 2024-04-25T00:30:39.715 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:39.716 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":12,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14643,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6839/187492674","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6838","nonce":187492674},{"type":"v1","addr":"172.21.15.32:6839","nonce":187492674}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14655,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6841/1948785545","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6840","nonce":1948785545},{"type":"v1","addr":"172.21.15.32:6841","nonce":1948785545}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24443,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6},{"gid":24445,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":7},{"gid":24457,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":8},{"gid":24476,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":9},{"gid":24481,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6839/4267125809","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":4267125809},{"type":"v1","addr":"172.21.15.130:6839","nonce":4267125809}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":10}],"filesystems":[]} 2024-04-25T00:30:39.718 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 12 2024-04-25T00:30:39.718 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.716+0000 7fc36181c640 1 -- 172.21.15.32:0/1088309851 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc3440a2c10 msgr2=0x7fc3440a50d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:39.718 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.716+0000 7fc36181c640 1 --2- 172.21.15.32:0/1088309851 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc3440a2c10 0x7fc3440a50d0 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7fc34c000c00 tx=0x7fc34c031400 comp rx=0 tx=0).stop 2024-04-25T00:30:39.719 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.716+0000 7fc36181c640 1 -- 172.21.15.32:0/1088309851 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc35c102250 msgr2=0x7fc35c19f0f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:39.719 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.716+0000 7fc36181c640 1 --2- 172.21.15.32:0/1088309851 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc35c102250 0x7fc35c19f0f0 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7fc35400eaa0 tx=0x7fc35400ef60 comp rx=0 tx=0).stop 2024-04-25T00:30:39.719 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.716+0000 7fc36181c640 1 -- 172.21.15.32:0/1088309851 shutdown_connections 2024-04-25T00:30:39.720 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.716+0000 7fc36181c640 1 --2- 172.21.15.32:0/1088309851 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc3440a2c10 0x7fc3440a50d0 unknown :-1 s=CLOSED pgs=142 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:39.720 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.716+0000 7fc36181c640 1 --2- 172.21.15.32:0/1088309851 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc35c102250 0x7fc35c19f0f0 unknown :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:39.720 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.716+0000 7fc36181c640 1 --2- 172.21.15.32:0/1088309851 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc35c101890 0x7fc35c198080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:39.720 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.716+0000 7fc36181c640 1 --2- 172.21.15.32:0/1088309851 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc35c1006a0 0x7fc35c197b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:39.720 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.716+0000 7fc36181c640 1 -- 172.21.15.32:0/1088309851 >> 172.21.15.32:0/1088309851 conn(0x7fc35c0fbc30 msgr2=0x7fc35c0fd740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:39.720 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.716+0000 7fc36181c640 1 -- 172.21.15.32:0/1088309851 shutdown_connections 2024-04-25T00:30:39.720 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:39.716+0000 7fc36181c640 1 -- 172.21.15.32:0/1088309851 wait complete. 2024-04-25T00:30:40.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:40 smithi032 ceph-mon[29357]: pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:40.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:40 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:40.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:40 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:40.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:40 smithi032 ceph-mon[29357]: mds.? [v2:172.21.15.32:6840/1948785545,v1:172.21.15.32:6841/1948785545] up:boot 2024-04-25T00:30:40.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:40 smithi032 ceph-mon[29357]: fsmap 11 up:standby 2024-04-25T00:30:40.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:40 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-04-25T00:30:40.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:40 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:40.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:40 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:40.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:40 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:40.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:40 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:40.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:40 smithi032 ceph-mon[29357]: Deploying daemon mds.b on smithi121 2024-04-25T00:30:40.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:40 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1088309851' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:40.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:40 smithi121 ceph-mon[32231]: pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:40.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:40 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:40.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:40 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:40.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:40 smithi121 ceph-mon[32231]: mds.? [v2:172.21.15.32:6840/1948785545,v1:172.21.15.32:6841/1948785545] up:boot 2024-04-25T00:30:40.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:40 smithi121 ceph-mon[32231]: fsmap 11 up:standby 2024-04-25T00:30:40.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:40 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-04-25T00:30:40.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:40 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:40.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:40 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:40.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:40 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:40.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:40 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:40.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:40 smithi121 ceph-mon[32231]: Deploying daemon mds.b on smithi121 2024-04-25T00:30:40.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:40 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1088309851' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:40.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:40 smithi130 ceph-mon[32525]: pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:40.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:40 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:40.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:40 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:40.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:40 smithi130 ceph-mon[32525]: mds.? [v2:172.21.15.32:6840/1948785545,v1:172.21.15.32:6841/1948785545] up:boot 2024-04-25T00:30:40.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:40 smithi130 ceph-mon[32525]: fsmap 11 up:standby 2024-04-25T00:30:40.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:40 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-04-25T00:30:40.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:40 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:40.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:40 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T00:30:40.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:40 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T00:30:40.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:40 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:40.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:40 smithi130 ceph-mon[32525]: Deploying daemon mds.b on smithi121 2024-04-25T00:30:40.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:40 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1088309851' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:41.729 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:30:41.823 INFO:journalctl@ceph.mds.b.smithi121.stdout:Apr 25 00:30:41 smithi121 systemd[1]: Starting Ceph mds.b for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:30:41.845 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.844+0000 7fa061295640 1 -- 172.21.15.32:0/2934692292 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa05c0ffb10 msgr2=0x7fa05c0fff30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:41.845 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.844+0000 7fa061295640 1 --2- 172.21.15.32:0/2934692292 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa05c0ffb10 0x7fa05c0fff30 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7fa050009930 tx=0x7fa05002a420 comp rx=0 tx=0).stop 2024-04-25T00:30:41.845 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.844+0000 7fa061295640 1 -- 172.21.15.32:0/2934692292 shutdown_connections 2024-04-25T00:30:41.845 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.844+0000 7fa061295640 1 --2- 172.21.15.32:0/2934692292 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa05c103540 0x7fa05c109dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:41.845 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.844+0000 7fa061295640 1 --2- 172.21.15.32:0/2934692292 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa05c102b80 0x7fa05c103000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:41.845 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.844+0000 7fa061295640 1 --2- 172.21.15.32:0/2934692292 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa05c0ffb10 0x7fa05c0fff30 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:41.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.844+0000 7fa061295640 1 -- 172.21.15.32:0/2934692292 >> 172.21.15.32:0/2934692292 conn(0x7fa05c0fb420 msgr2=0x7fa05c0fd860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:41.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.844+0000 7fa061295640 1 -- 172.21.15.32:0/2934692292 shutdown_connections 2024-04-25T00:30:41.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.844+0000 7fa061295640 1 -- 172.21.15.32:0/2934692292 wait complete. 2024-04-25T00:30:41.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.845+0000 7fa061295640 1 Processor -- start 2024-04-25T00:30:41.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.845+0000 7fa061295640 1 -- start start 2024-04-25T00:30:41.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.845+0000 7fa061295640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa05c0ffb10 0x7fa05c1024f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:41.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.845+0000 7fa061295640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa05c102b80 0x7fa05c100b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:41.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.845+0000 7fa061295640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa05c103540 0x7fa05c101070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:41.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.845+0000 7fa061295640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fa05c10d4a0 con 0x7fa05c102b80 2024-04-25T00:30:41.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.845+0000 7fa061295640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fa05c10d320 con 0x7fa05c103540 2024-04-25T00:30:41.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.845+0000 7fa061295640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fa05c10d620 con 0x7fa05c0ffb10 2024-04-25T00:30:41.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.845+0000 7fa05a7fc640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa05c102b80 0x7fa05c100b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:41.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.845+0000 7fa05a7fc640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa05c102b80 0x7fa05c100b30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:36590/0 (socket says 172.21.15.32:36590) 2024-04-25T00:30:41.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.845+0000 7fa05a7fc640 1 -- 172.21.15.32:0/1861919035 learned_addr learned my addr 172.21.15.32:0/1861919035 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:41.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.845+0000 7fa05b7fe640 1 --2- 172.21.15.32:0/1861919035 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa05c103540 0x7fa05c101070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:41.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.845+0000 7fa05affd640 1 --2- 172.21.15.32:0/1861919035 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa05c0ffb10 0x7fa05c1024f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:41.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.845+0000 7fa05a7fc640 1 -- 172.21.15.32:0/1861919035 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa05c0ffb10 msgr2=0x7fa05c1024f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:41.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.846+0000 7fa05a7fc640 1 --2- 172.21.15.32:0/1861919035 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa05c0ffb10 0x7fa05c1024f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:41.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.846+0000 7fa05a7fc640 1 -- 172.21.15.32:0/1861919035 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa05c103540 msgr2=0x7fa05c101070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:41.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.846+0000 7fa05a7fc640 1 --2- 172.21.15.32:0/1861919035 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa05c103540 0x7fa05c101070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:41.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.846+0000 7fa05a7fc640 1 -- 172.21.15.32:0/1861919035 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa05c101900 con 0x7fa05c102b80 2024-04-25T00:30:41.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.846+0000 7fa05a7fc640 1 --2- 172.21.15.32:0/1861919035 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa05c102b80 0x7fa05c100b30 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7fa04c00e9f0 tx=0x7fa04c00eeb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:41.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.846+0000 7fa043fff640 1 -- 172.21.15.32:0/1861919035 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa04c00cd80 con 0x7fa05c102b80 2024-04-25T00:30:41.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.846+0000 7fa061295640 1 -- 172.21.15.32:0/1861919035 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa05c1a1c00 con 0x7fa05c102b80 2024-04-25T00:30:41.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.846+0000 7fa043fff640 1 -- 172.21.15.32:0/1861919035 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa04c010910 con 0x7fa05c102b80 2024-04-25T00:30:41.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.846+0000 7fa043fff640 1 -- 172.21.15.32:0/1861919035 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa04c018950 con 0x7fa05c102b80 2024-04-25T00:30:41.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.846+0000 7fa061295640 1 -- 172.21.15.32:0/1861919035 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa05c1a21a0 con 0x7fa05c102b80 2024-04-25T00:30:41.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.847+0000 7fa061295640 1 -- 172.21.15.32:0/1861919035 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa05c108c50 con 0x7fa05c102b80 2024-04-25T00:30:41.850 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.848+0000 7fa043fff640 1 -- 172.21.15.32:0/1861919035 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7fa04c010ac0 con 0x7fa05c102b80 2024-04-25T00:30:41.850 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.849+0000 7fa043fff640 1 --2- 172.21.15.32:0/1861919035 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa0440a2c10 0x7fa0440a50d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:41.850 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.849+0000 7fa05affd640 1 --2- 172.21.15.32:0/1861919035 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa0440a2c10 0x7fa0440a50d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:41.851 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.849+0000 7fa043fff640 1 -- 172.21.15.32:0/1861919035 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7fa04c0cef50 con 0x7fa05c102b80 2024-04-25T00:30:41.851 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.849+0000 7fa05affd640 1 --2- 172.21.15.32:0/1861919035 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa0440a2c10 0x7fa0440a50d0 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7fa050031610 tx=0x7fa050000bc0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:41.854 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:41.853+0000 7fa043fff640 1 -- 172.21.15.32:0/1861919035 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fa04c09c350 con 0x7fa05c102b80 2024-04-25T00:30:42.077 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:42.077+0000 7fa061295640 1 -- 172.21.15.32:0/1861919035 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fa05c062340 con 0x7fa05c102b80 2024-04-25T00:30:42.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:42.077+0000 7fa043fff640 1 -- 172.21.15.32:0/1861919035 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 12 v12) ==== 94+0+9659 (secure 0 0 0) 0x7fa04c09bc50 con 0x7fa05c102b80 2024-04-25T00:30:42.078 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:42.078 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":12,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14643,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6839/187492674","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6838","nonce":187492674},{"type":"v1","addr":"172.21.15.32:6839","nonce":187492674}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14655,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6841/1948785545","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6840","nonce":1948785545},{"type":"v1","addr":"172.21.15.32:6841","nonce":1948785545}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24443,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6},{"gid":24445,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":7},{"gid":24457,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":8},{"gid":24476,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":9},{"gid":24481,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6839/4267125809","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":4267125809},{"type":"v1","addr":"172.21.15.130:6839","nonce":4267125809}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":10}],"filesystems":[]} 2024-04-25T00:30:42.078 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 12 2024-04-25T00:30:42.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:42.079+0000 7fa061295640 1 -- 172.21.15.32:0/1861919035 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa0440a2c10 msgr2=0x7fa0440a50d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:42.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:42.079+0000 7fa061295640 1 --2- 172.21.15.32:0/1861919035 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa0440a2c10 0x7fa0440a50d0 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7fa050031610 tx=0x7fa050000bc0 comp rx=0 tx=0).stop 2024-04-25T00:30:42.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:42.079+0000 7fa061295640 1 -- 172.21.15.32:0/1861919035 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa05c102b80 msgr2=0x7fa05c100b30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:42.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:42.079+0000 7fa061295640 1 --2- 172.21.15.32:0/1861919035 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa05c102b80 0x7fa05c100b30 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7fa04c00e9f0 tx=0x7fa04c00eeb0 comp rx=0 tx=0).stop 2024-04-25T00:30:42.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:42.079+0000 7fa061295640 1 -- 172.21.15.32:0/1861919035 shutdown_connections 2024-04-25T00:30:42.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:42.079+0000 7fa061295640 1 --2- 172.21.15.32:0/1861919035 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa0440a2c10 0x7fa0440a50d0 unknown :-1 s=CLOSED pgs=144 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:42.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:42.079+0000 7fa061295640 1 --2- 172.21.15.32:0/1861919035 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa05c103540 0x7fa05c101070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:42.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:42.079+0000 7fa061295640 1 --2- 172.21.15.32:0/1861919035 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa05c102b80 0x7fa05c100b30 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:42.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:42.079+0000 7fa061295640 1 --2- 172.21.15.32:0/1861919035 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa05c0ffb10 0x7fa05c1024f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:42.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:42.079+0000 7fa061295640 1 -- 172.21.15.32:0/1861919035 >> 172.21.15.32:0/1861919035 conn(0x7fa05c0fb420 msgr2=0x7fa05c105e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:42.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:42.079+0000 7fa061295640 1 -- 172.21.15.32:0/1861919035 shutdown_connections 2024-04-25T00:30:42.081 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:42.079+0000 7fa061295640 1 -- 172.21.15.32:0/1861919035 wait complete. 2024-04-25T00:30:42.087 INFO:journalctl@ceph.mds.b.smithi121.stdout:Apr 25 00:30:42 smithi121 podman[62567]: 2024-04-25 00:30:41.943559085 +0000 UTC m=+0.017930036 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901 2024-04-25T00:30:42.087 INFO:journalctl@ceph.mds.b.smithi121.stdout:Apr 25 00:30:42 smithi121 podman[62567]: 2024-04-25 00:30:42.065062093 +0000 UTC m=+0.139433085 container create 43cd35e25c64f90b2697f7d39f99e8a9c21dc6b8254d631ad4137b6cd092ada1 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=squid-d2c45d9, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-25T00:30:42.450 INFO:journalctl@ceph.mds.b.smithi121.stdout:Apr 25 00:30:42 smithi121 podman[62567]: 2024-04-25 00:30:42.301085313 +0000 UTC m=+0.375456261 container init 43cd35e25c64f90b2697f7d39f99e8a9c21dc6b8254d631ad4137b6cd092ada1 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-d2c45d9, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T00:30:42.451 INFO:journalctl@ceph.mds.b.smithi121.stdout:Apr 25 00:30:42 smithi121 podman[62567]: 2024-04-25 00:30:42.303612486 +0000 UTC m=+0.377983434 container start 43cd35e25c64f90b2697f7d39f99e8a9c21dc6b8254d631ad4137b6cd092ada1 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=squid-d2c45d9, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T00:30:42.451 INFO:journalctl@ceph.mds.b.smithi121.stdout:Apr 25 00:30:42 smithi121 ceph-mds[62586]: -- 172.21.15.121:0/1277032842 <== mon.2 v2:172.21.15.121:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564f59fe3a40 con 0x564f59fb6c00 2024-04-25T00:30:42.451 INFO:journalctl@ceph.mds.b.smithi121.stdout:Apr 25 00:30:42 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-b[62582]: starting mds.b at 2024-04-25T00:30:42.451 INFO:journalctl@ceph.mds.b.smithi121.stdout:Apr 25 00:30:42 smithi121 bash[62567]: 43cd35e25c64f90b2697f7d39f99e8a9c21dc6b8254d631ad4137b6cd092ada1 2024-04-25T00:30:42.451 INFO:journalctl@ceph.mds.b.smithi121.stdout:Apr 25 00:30:42 smithi121 systemd[1]: Started Ceph mds.b for cb997156-0299-11ef-bc93-c7b262605968. 2024-04-25T00:30:42.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:42 smithi032 ceph-mon[29357]: pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:42.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:42 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1861919035' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:42.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:42.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:42 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:42.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:42 smithi121 ceph-mon[32231]: pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:42.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:42 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1861919035' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:42.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:42 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:42.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:42 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:42.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:42 smithi130 ceph-mon[32525]: pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:42.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:42 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1861919035' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:42.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:42 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:42.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:42 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:43.731 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:43 smithi032 ceph-mon[29357]: mds.? [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] up:boot 2024-04-25T00:30:43.732 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:43 smithi032 ceph-mon[29357]: fsmap 12 up:standby 2024-04-25T00:30:43.732 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-04-25T00:30:43.732 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:43.732 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:43.732 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:43.732 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:30:43.732 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:43.759 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:43 smithi130 ceph-mon[32525]: mds.? [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] up:boot 2024-04-25T00:30:43.759 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:43 smithi130 ceph-mon[32525]: fsmap 12 up:standby 2024-04-25T00:30:43.759 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-04-25T00:30:43.759 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:43.759 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:43.759 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:43.759 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:30:43.759 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:43.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:43 smithi121 ceph-mon[32231]: mds.? [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] up:boot 2024-04-25T00:30:43.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:43 smithi121 ceph-mon[32231]: fsmap 12 up:standby 2024-04-25T00:30:43.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-04-25T00:30:43.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:43.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:43.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:43.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:30:43.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:44.092 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:30:44.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.202+0000 7f9409272640 1 -- 172.21.15.32:0/3370914882 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9404102960 msgr2=0x7f9404104dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:44.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.202+0000 7f9409272640 1 --2- 172.21.15.32:0/3370914882 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9404102960 0x7f9404104dd0 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f93ec009900 tx=0x7f93ec02a410 comp rx=0 tx=0).stop 2024-04-25T00:30:44.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.202+0000 7f9409272640 1 -- 172.21.15.32:0/3370914882 shutdown_connections 2024-04-25T00:30:44.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.202+0000 7f9409272640 1 --2- 172.21.15.32:0/3370914882 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9404105310 0x7f9404105790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.202+0000 7f9409272640 1 --2- 172.21.15.32:0/3370914882 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9404102960 0x7f9404104dd0 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.202+0000 7f9409272640 1 --2- 172.21.15.32:0/3370914882 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f94040ffff0 0x7f9404102420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.202+0000 7f9409272640 1 -- 172.21.15.32:0/3370914882 >> 172.21.15.32:0/3370914882 conn(0x7f94040fbbf0 msgr2=0x7f94040fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:44.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.202+0000 7f9409272640 1 -- 172.21.15.32:0/3370914882 shutdown_connections 2024-04-25T00:30:44.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.202+0000 7f9409272640 1 -- 172.21.15.32:0/3370914882 wait complete. 2024-04-25T00:30:44.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.203+0000 7f9409272640 1 Processor -- start 2024-04-25T00:30:44.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.203+0000 7f9409272640 1 -- start start 2024-04-25T00:30:44.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.203+0000 7f9409272640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f94040ffff0 0x7f940419e100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:44.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.203+0000 7f9409272640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9404102960 0x7f940419e640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:44.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.203+0000 7f9409272640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9404105310 0x7f94041a5610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:44.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.203+0000 7f9409272640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f94041139f0 con 0x7f94040ffff0 2024-04-25T00:30:44.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.203+0000 7f9409272640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f9404113870 con 0x7f9404105310 2024-04-25T00:30:44.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.203+0000 7f9409272640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f9404113b70 con 0x7f9404102960 2024-04-25T00:30:44.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.203+0000 7f9402ffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f94040ffff0 0x7f940419e100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:44.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.203+0000 7f9402ffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f94040ffff0 0x7f940419e100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:36604/0 (socket says 172.21.15.32:36604) 2024-04-25T00:30:44.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.203+0000 7f9402ffd640 1 -- 172.21.15.32:0/1991077908 learned_addr learned my addr 172.21.15.32:0/1991077908 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:44.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.203+0000 7f94037fe640 1 --2- 172.21.15.32:0/1991077908 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9404105310 0x7f94041a5610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:44.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.203+0000 7f93fbfff640 1 --2- 172.21.15.32:0/1991077908 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9404102960 0x7f940419e640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:44.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.204+0000 7f9402ffd640 1 -- 172.21.15.32:0/1991077908 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9404102960 msgr2=0x7f940419e640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:44.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.204+0000 7f9402ffd640 1 --2- 172.21.15.32:0/1991077908 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9404102960 0x7f940419e640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.204+0000 7f9402ffd640 1 -- 172.21.15.32:0/1991077908 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9404105310 msgr2=0x7f94041a5610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:44.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.204+0000 7f9402ffd640 1 --2- 172.21.15.32:0/1991077908 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9404105310 0x7f94041a5610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.204+0000 7f9402ffd640 1 -- 172.21.15.32:0/1991077908 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f94041a5c80 con 0x7f94040ffff0 2024-04-25T00:30:44.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.204+0000 7f94037fe640 1 --2- 172.21.15.32:0/1991077908 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9404105310 0x7f94041a5610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:30:44.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.204+0000 7f9402ffd640 1 --2- 172.21.15.32:0/1991077908 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f94040ffff0 0x7f940419e100 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f93f400e9f0 tx=0x7f93f400eeb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:44.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.204+0000 7f9400ff9640 1 -- 172.21.15.32:0/1991077908 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f93f400cd80 con 0x7f94040ffff0 2024-04-25T00:30:44.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.204+0000 7f9400ff9640 1 -- 172.21.15.32:0/1991077908 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f93f4010910 con 0x7f94040ffff0 2024-04-25T00:30:44.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.204+0000 7f9409272640 1 -- 172.21.15.32:0/1991077908 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f94041a5f70 con 0x7f94040ffff0 2024-04-25T00:30:44.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.204+0000 7f9400ff9640 1 -- 172.21.15.32:0/1991077908 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f93f4018990 con 0x7f94040ffff0 2024-04-25T00:30:44.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.204+0000 7f9409272640 1 -- 172.21.15.32:0/1991077908 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f94041a63d0 con 0x7f94040ffff0 2024-04-25T00:30:44.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.205+0000 7f9409272640 1 -- 172.21.15.32:0/1991077908 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f94040fd6e0 con 0x7f94040ffff0 2024-04-25T00:30:44.209 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.207+0000 7f9400ff9640 1 -- 172.21.15.32:0/1991077908 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f93f4010ac0 con 0x7f94040ffff0 2024-04-25T00:30:44.209 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.207+0000 7f9400ff9640 1 --2- 172.21.15.32:0/1991077908 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f93f00a2b90 0x7f93f00a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:44.209 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.207+0000 7f93fbfff640 1 --2- 172.21.15.32:0/1991077908 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f93f00a2b90 0x7f93f00a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:44.209 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.207+0000 7f9400ff9640 1 -- 172.21.15.32:0/1991077908 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7f93f40cf3f0 con 0x7f94040ffff0 2024-04-25T00:30:44.209 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.208+0000 7f93fbfff640 1 --2- 172.21.15.32:0/1991077908 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f93f00a2b90 0x7f93f00a5050 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f93ec000c00 tx=0x7f93ec0315d0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:44.213 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.213+0000 7f9400ff9640 1 -- 172.21.15.32:0/1991077908 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f93f409c7f0 con 0x7f94040ffff0 2024-04-25T00:30:44.475 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.475+0000 7f9409272640 1 -- 172.21.15.32:0/1991077908 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f940402d170 con 0x7f94040ffff0 2024-04-25T00:30:44.476 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.476+0000 7f9400ff9640 1 -- 172.21.15.32:0/1991077908 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 13 v13) ==== 94+0+10485 (secure 0 0 0) 0x7f93f409c0f0 con 0x7f94040ffff0 2024-04-25T00:30:44.476 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:44.476 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":13,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14643,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6839/187492674","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6838","nonce":187492674},{"type":"v1","addr":"172.21.15.32:6839","nonce":187492674}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14655,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6841/1948785545","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6840","nonce":1948785545},{"type":"v1","addr":"172.21.15.32:6841","nonce":1948785545}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24443,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6},{"gid":24445,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":7},{"gid":24457,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":8},{"gid":24476,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":9},{"gid":24481,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6839/4267125809","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":4267125809},{"type":"v1","addr":"172.21.15.130:6839","nonce":4267125809}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":10},{"gid":24506,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6839/258444853","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6838","nonce":258444853},{"type":"v1","addr":"172.21.15.121:6839","nonce":258444853}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":13}],"filesystems":[]} 2024-04-25T00:30:44.477 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 13 2024-04-25T00:30:44.479 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.478+0000 7f9409272640 1 -- 172.21.15.32:0/1991077908 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f93f00a2b90 msgr2=0x7f93f00a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:44.479 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.478+0000 7f9409272640 1 --2- 172.21.15.32:0/1991077908 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f93f00a2b90 0x7f93f00a5050 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f93ec000c00 tx=0x7f93ec0315d0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.479 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.478+0000 7f9409272640 1 -- 172.21.15.32:0/1991077908 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f94040ffff0 msgr2=0x7f940419e100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:44.479 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.478+0000 7f9409272640 1 --2- 172.21.15.32:0/1991077908 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f94040ffff0 0x7f940419e100 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f93f400e9f0 tx=0x7f93f400eeb0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.479 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.478+0000 7f9409272640 1 -- 172.21.15.32:0/1991077908 shutdown_connections 2024-04-25T00:30:44.479 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.478+0000 7f9409272640 1 --2- 172.21.15.32:0/1991077908 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f93f00a2b90 0x7f93f00a5050 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.479 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.478+0000 7f9409272640 1 --2- 172.21.15.32:0/1991077908 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9404105310 0x7f94041a5610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.479 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.478+0000 7f9409272640 1 --2- 172.21.15.32:0/1991077908 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9404102960 0x7f940419e640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.480 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.478+0000 7f9409272640 1 --2- 172.21.15.32:0/1991077908 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f94040ffff0 0x7f940419e100 unknown :-1 s=CLOSED pgs=191 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.480 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.478+0000 7f9409272640 1 -- 172.21.15.32:0/1991077908 >> 172.21.15.32:0/1991077908 conn(0x7f94040fbbf0 msgr2=0x7f9404100990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:44.480 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.478+0000 7f9409272640 1 -- 172.21.15.32:0/1991077908 shutdown_connections 2024-04-25T00:30:44.480 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.478+0000 7f9409272640 1 -- 172.21.15.32:0/1991077908 wait complete. 2024-04-25T00:30:44.492 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-25T00:30:44.597 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.595+0000 7fea10f42640 1 -- 172.21.15.32:0/3054744331 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea0c100680 msgr2=0x7fea0c100aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:44.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.595+0000 7fea10f42640 1 --2- 172.21.15.32:0/3054744331 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea0c100680 0x7fea0c100aa0 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7fe9fc009900 tx=0x7fe9fc02a410 comp rx=0 tx=0).stop 2024-04-25T00:30:44.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.595+0000 7fea10f42640 1 -- 172.21.15.32:0/3054744331 shutdown_connections 2024-04-25T00:30:44.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.595+0000 7fea10f42640 1 --2- 172.21.15.32:0/3054744331 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fea0c102230 0x7fea0c108ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.595+0000 7fea10f42640 1 --2- 172.21.15.32:0/3054744331 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea0c101870 0x7fea0c101cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.595+0000 7fea10f42640 1 --2- 172.21.15.32:0/3054744331 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea0c100680 0x7fea0c100aa0 unknown :-1 s=CLOSED pgs=192 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.595+0000 7fea10f42640 1 -- 172.21.15.32:0/3054744331 >> 172.21.15.32:0/3054744331 conn(0x7fea0c0fbbf0 msgr2=0x7fea0c0fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:44.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.595+0000 7fea10f42640 1 -- 172.21.15.32:0/3054744331 shutdown_connections 2024-04-25T00:30:44.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.595+0000 7fea10f42640 1 -- 172.21.15.32:0/3054744331 wait complete. 2024-04-25T00:30:44.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.596+0000 7fea10f42640 1 Processor -- start 2024-04-25T00:30:44.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.596+0000 7fea10f42640 1 -- start start 2024-04-25T00:30:44.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.596+0000 7fea10f42640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea0c100680 0x7fea0c069440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:44.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.596+0000 7fea10f42640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea0c101870 0x7fea0c19c790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:44.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.596+0000 7fea10f42640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fea0c102230 0x7fea0c19edb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:44.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.596+0000 7fea10f42640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fea0c10c2d0 con 0x7fea0c100680 2024-04-25T00:30:44.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.596+0000 7fea10f42640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fea0c10c150 con 0x7fea0c102230 2024-04-25T00:30:44.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.596+0000 7fea10f42640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fea0c10c450 con 0x7fea0c101870 2024-04-25T00:30:44.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.596+0000 7fea0a59c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea0c100680 0x7fea0c069440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:44.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.596+0000 7fea0a59c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea0c100680 0x7fea0c069440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:36620/0 (socket says 172.21.15.32:36620) 2024-04-25T00:30:44.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.596+0000 7fea0a59c640 1 -- 172.21.15.32:0/347507021 learned_addr learned my addr 172.21.15.32:0/347507021 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:44.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.596+0000 7fea03fff640 1 --2- 172.21.15.32:0/347507021 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea0c101870 0x7fea0c19c790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:44.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.596+0000 7fea0ad9d640 1 --2- 172.21.15.32:0/347507021 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fea0c102230 0x7fea0c19edb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:44.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.596+0000 7fea0a59c640 1 -- 172.21.15.32:0/347507021 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea0c101870 msgr2=0x7fea0c19c790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:44.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.596+0000 7fea0a59c640 1 --2- 172.21.15.32:0/347507021 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea0c101870 0x7fea0c19c790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.596+0000 7fea0a59c640 1 -- 172.21.15.32:0/347507021 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fea0c102230 msgr2=0x7fea0c19edb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:44.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.596+0000 7fea0a59c640 1 --2- 172.21.15.32:0/347507021 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fea0c102230 0x7fea0c19edb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.596+0000 7fea0a59c640 1 -- 172.21.15.32:0/347507021 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fea0c19f6c0 con 0x7fea0c100680 2024-04-25T00:30:44.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.597+0000 7fea0a59c640 1 --2- 172.21.15.32:0/347507021 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea0c100680 0x7fea0c069440 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7fe9fc031610 tx=0x7fe9fc004270 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:44.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.597+0000 7fea037fe640 1 -- 172.21.15.32:0/347507021 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe9fc03f030 con 0x7fea0c100680 2024-04-25T00:30:44.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.597+0000 7fea10f42640 1 -- 172.21.15.32:0/347507021 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fea0c19f8f0 con 0x7fea0c100680 2024-04-25T00:30:44.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.597+0000 7fea037fe640 1 -- 172.21.15.32:0/347507021 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe9fc003ad0 con 0x7fea0c100680 2024-04-25T00:30:44.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.597+0000 7fea037fe640 1 -- 172.21.15.32:0/347507021 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe9fc035070 con 0x7fea0c100680 2024-04-25T00:30:44.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.597+0000 7fea10f42640 1 -- 172.21.15.32:0/347507021 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fea0c19fde0 con 0x7fea0c100680 2024-04-25T00:30:44.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.598+0000 7fea10f42640 1 -- 172.21.15.32:0/347507021 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fea0c1079a0 con 0x7fea0c100680 2024-04-25T00:30:44.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.599+0000 7fea037fe640 1 -- 172.21.15.32:0/347507021 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7fe9fc042030 con 0x7fea0c100680 2024-04-25T00:30:44.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.599+0000 7fea037fe640 1 --2- 172.21.15.32:0/347507021 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe9f80a2b90 0x7fe9f80a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:44.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.600+0000 7fea03fff640 1 --2- 172.21.15.32:0/347507021 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe9f80a2b90 0x7fe9f80a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:44.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.600+0000 7fea037fe640 1 -- 172.21.15.32:0/347507021 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7fe9fc0ef220 con 0x7fea0c100680 2024-04-25T00:30:44.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.600+0000 7fea03fff640 1 --2- 172.21.15.32:0/347507021 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe9f80a2b90 0x7fe9f80a5050 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7fea0c0691b0 tx=0x7fe9f4005f90 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:44.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.604+0000 7fea037fe640 1 -- 172.21.15.32:0/347507021 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fe9fc0bbfd0 con 0x7fea0c100680 2024-04-25T00:30:44.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:44 smithi032 ceph-mon[29357]: pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:44.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:44.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:44.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:44.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:44.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:44.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:44.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:44.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:30:44.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:44.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:44 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1991077908' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:44.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.793+0000 7fea10f42640 1 -- 172.21.15.32:0/347507021 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fea0c02d3c0 con 0x7fea0c100680 2024-04-25T00:30:44.794 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.794+0000 7fea037fe640 1 -- 172.21.15.32:0/347507021 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v81) ==== 74+0+18456 (secure 0 0 0) 0x7fe9fc0c02d0 con 0x7fea0c100680 2024-04-25T00:30:44.795 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:44.795 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":81,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:29:59.700603+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:30:44.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.796+0000 7fea10f42640 1 -- 172.21.15.32:0/347507021 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe9f80a2b90 msgr2=0x7fe9f80a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:44.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.796+0000 7fea10f42640 1 --2- 172.21.15.32:0/347507021 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe9f80a2b90 0x7fe9f80a5050 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7fea0c0691b0 tx=0x7fe9f4005f90 comp rx=0 tx=0).stop 2024-04-25T00:30:44.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.796+0000 7fea10f42640 1 -- 172.21.15.32:0/347507021 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea0c100680 msgr2=0x7fea0c069440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:44.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.796+0000 7fea10f42640 1 --2- 172.21.15.32:0/347507021 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea0c100680 0x7fea0c069440 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7fe9fc031610 tx=0x7fe9fc004270 comp rx=0 tx=0).stop 2024-04-25T00:30:44.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.796+0000 7fea10f42640 1 -- 172.21.15.32:0/347507021 shutdown_connections 2024-04-25T00:30:44.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.796+0000 7fea10f42640 1 --2- 172.21.15.32:0/347507021 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe9f80a2b90 0x7fe9f80a5050 unknown :-1 s=CLOSED pgs=147 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.796+0000 7fea10f42640 1 --2- 172.21.15.32:0/347507021 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fea0c102230 0x7fea0c19edb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.798 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.796+0000 7fea10f42640 1 --2- 172.21.15.32:0/347507021 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea0c101870 0x7fea0c19c790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.798 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.796+0000 7fea10f42640 1 --2- 172.21.15.32:0/347507021 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea0c100680 0x7fea0c069440 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.798 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.796+0000 7fea10f42640 1 -- 172.21.15.32:0/347507021 >> 172.21.15.32:0/347507021 conn(0x7fea0c0fbbf0 msgr2=0x7fea0c104b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:44.798 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.797+0000 7fea10f42640 1 -- 172.21.15.32:0/347507021 shutdown_connections 2024-04-25T00:30:44.798 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.797+0000 7fea10f42640 1 -- 172.21.15.32:0/347507021 wait complete. 2024-04-25T00:30:44.809 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-25T00:26:07.306839+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-04-25T00:30:44.809 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-25T00:30:44.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:44 smithi121 ceph-mon[32231]: pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:44.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:44.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:44.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:44.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:44.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:44.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:44.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:44.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:30:44.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:44.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:44 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1991077908' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:44.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:44 smithi130 ceph-mon[32525]: pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:44.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:44.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:44.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:44.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:44.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:44.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:44.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:44.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:30:44.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:44.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:44 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1991077908' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:44.918 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.915+0000 7fcae1476640 1 -- 172.21.15.32:0/411247443 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcadc102a20 msgr2=0x7fcadc102ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:44.918 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.915+0000 7fcae1476640 1 --2- 172.21.15.32:0/411247443 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcadc102a20 0x7fcadc102ea0 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7fcad0009930 tx=0x7fcad002a420 comp rx=0 tx=0).stop 2024-04-25T00:30:44.918 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.915+0000 7fcae1476640 1 -- 172.21.15.32:0/411247443 shutdown_connections 2024-04-25T00:30:44.918 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.915+0000 7fcae1476640 1 --2- 172.21.15.32:0/411247443 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fcadc1033e0 0x7fcadc109c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.918 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.915+0000 7fcae1476640 1 --2- 172.21.15.32:0/411247443 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcadc102a20 0x7fcadc102ea0 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.915+0000 7fcae1476640 1 --2- 172.21.15.32:0/411247443 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcadc0ffa50 0x7fcadc0ffe70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.915+0000 7fcae1476640 1 -- 172.21.15.32:0/411247443 >> 172.21.15.32:0/411247443 conn(0x7fcadc0fb380 msgr2=0x7fcadc0fd7c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:44.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.915+0000 7fcae1476640 1 -- 172.21.15.32:0/411247443 shutdown_connections 2024-04-25T00:30:44.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.915+0000 7fcae1476640 1 -- 172.21.15.32:0/411247443 wait complete. 2024-04-25T00:30:44.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.916+0000 7fcae1476640 1 Processor -- start 2024-04-25T00:30:44.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.916+0000 7fcae1476640 1 -- start start 2024-04-25T00:30:44.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.916+0000 7fcae1476640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fcadc0ffa50 0x7fcadc102390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:44.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.916+0000 7fcae1476640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcadc102a20 0x7fcadc1009d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:44.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.916+0000 7fcae1476640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcadc1033e0 0x7fcadc100f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:44.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.916+0000 7fcae1476640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fcadc10d480 con 0x7fcadc1033e0 2024-04-25T00:30:44.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.916+0000 7fcae1476640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fcadc10d300 con 0x7fcadc0ffa50 2024-04-25T00:30:44.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.916+0000 7fcae1476640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fcadc10d600 con 0x7fcadc102a20 2024-04-25T00:30:44.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.916+0000 7fcadb7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcadc1033e0 0x7fcadc100f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:44.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.916+0000 7fcada7fc640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcadc102a20 0x7fcadc1009d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:44.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.916+0000 7fcadb7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcadc1033e0 0x7fcadc100f10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:36634/0 (socket says 172.21.15.32:36634) 2024-04-25T00:30:44.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.916+0000 7fcadb7fe640 1 -- 172.21.15.32:0/769314593 learned_addr learned my addr 172.21.15.32:0/769314593 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:44.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.916+0000 7fcadaffd640 1 --2- 172.21.15.32:0/769314593 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fcadc0ffa50 0x7fcadc102390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:44.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.917+0000 7fcada7fc640 1 -- 172.21.15.32:0/769314593 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fcadc0ffa50 msgr2=0x7fcadc102390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:44.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.917+0000 7fcada7fc640 1 --2- 172.21.15.32:0/769314593 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fcadc0ffa50 0x7fcadc102390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.917+0000 7fcada7fc640 1 -- 172.21.15.32:0/769314593 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcadc1033e0 msgr2=0x7fcadc100f10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:44.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.917+0000 7fcada7fc640 1 --2- 172.21.15.32:0/769314593 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcadc1033e0 0x7fcadc100f10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:44.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.917+0000 7fcada7fc640 1 -- 172.21.15.32:0/769314593 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fcadc1017a0 con 0x7fcadc102a20 2024-04-25T00:30:44.921 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.917+0000 7fcadb7fe640 1 --2- 172.21.15.32:0/769314593 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcadc1033e0 0x7fcadc100f10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:30:44.921 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.917+0000 7fcada7fc640 1 --2- 172.21.15.32:0/769314593 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcadc102a20 0x7fcadc1009d0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7fcad002a940 tx=0x7fcad0032c60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:44.921 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.917+0000 7fcac3fff640 1 -- 172.21.15.32:0/769314593 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcad003f030 con 0x7fcadc102a20 2024-04-25T00:30:44.921 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.917+0000 7fcae1476640 1 -- 172.21.15.32:0/769314593 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fcadc1a1be0 con 0x7fcadc102a20 2024-04-25T00:30:44.921 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.917+0000 7fcac3fff640 1 -- 172.21.15.32:0/769314593 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fcad0032e50 con 0x7fcadc102a20 2024-04-25T00:30:44.921 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.917+0000 7fcac3fff640 1 -- 172.21.15.32:0/769314593 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcad0035070 con 0x7fcadc102a20 2024-04-25T00:30:44.921 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.917+0000 7fcae1476640 1 -- 172.21.15.32:0/769314593 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fcadc1a2150 con 0x7fcadc102a20 2024-04-25T00:30:44.922 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.918+0000 7fcae1476640 1 -- 172.21.15.32:0/769314593 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fcadc108b50 con 0x7fcadc102a20 2024-04-25T00:30:44.922 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.919+0000 7fcac3fff640 1 -- 172.21.15.32:0/769314593 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7fcad0042030 con 0x7fcadc102a20 2024-04-25T00:30:44.922 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.920+0000 7fcac3fff640 1 --2- 172.21.15.32:0/769314593 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fcac40a2b90 0x7fcac40a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:44.922 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.920+0000 7fcadaffd640 1 --2- 172.21.15.32:0/769314593 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fcac40a2b90 0x7fcac40a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:44.922 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.920+0000 7fcac3fff640 1 -- 172.21.15.32:0/769314593 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7fcad00ef150 con 0x7fcadc102a20 2024-04-25T00:30:44.922 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.920+0000 7fcadaffd640 1 --2- 172.21.15.32:0/769314593 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fcac40a2b90 0x7fcac40a5050 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7fcadc069b60 tx=0x7fcacc005f90 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:44.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:44.923+0000 7fcac3fff640 1 -- 172.21.15.32:0/769314593 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fcad00bbf00 con 0x7fcadc102a20 2024-04-25T00:30:45.111 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.110+0000 7fcae1476640 1 -- 172.21.15.32:0/769314593 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7fcadc02d370 con 0x7fcadc102a20 2024-04-25T00:30:45.112 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.111+0000 7fcac3fff640 1 -- 172.21.15.32:0/769314593 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v81) ==== 93+0+10 (secure 0 0 0) 0x7fcad00c0200 con 0x7fcadc102a20 2024-04-25T00:30:45.112 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 1 2024-04-25T00:30:45.114 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.113+0000 7fcae1476640 1 -- 172.21.15.32:0/769314593 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fcac40a2b90 msgr2=0x7fcac40a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:45.114 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.113+0000 7fcae1476640 1 --2- 172.21.15.32:0/769314593 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fcac40a2b90 0x7fcac40a5050 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7fcadc069b60 tx=0x7fcacc005f90 comp rx=0 tx=0).stop 2024-04-25T00:30:45.114 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.113+0000 7fcae1476640 1 -- 172.21.15.32:0/769314593 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcadc102a20 msgr2=0x7fcadc1009d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:45.114 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.113+0000 7fcae1476640 1 --2- 172.21.15.32:0/769314593 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcadc102a20 0x7fcadc1009d0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7fcad002a940 tx=0x7fcad0032c60 comp rx=0 tx=0).stop 2024-04-25T00:30:45.114 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.113+0000 7fcae1476640 1 -- 172.21.15.32:0/769314593 shutdown_connections 2024-04-25T00:30:45.114 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.113+0000 7fcae1476640 1 --2- 172.21.15.32:0/769314593 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fcac40a2b90 0x7fcac40a5050 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:45.115 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.113+0000 7fcae1476640 1 --2- 172.21.15.32:0/769314593 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcadc1033e0 0x7fcadc100f10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:45.115 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.113+0000 7fcae1476640 1 --2- 172.21.15.32:0/769314593 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcadc102a20 0x7fcadc1009d0 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:45.115 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.113+0000 7fcae1476640 1 --2- 172.21.15.32:0/769314593 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fcadc0ffa50 0x7fcadc102390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:45.115 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.113+0000 7fcae1476640 1 -- 172.21.15.32:0/769314593 >> 172.21.15.32:0/769314593 conn(0x7fcadc0fb380 msgr2=0x7fcadc105d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:45.115 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.113+0000 7fcae1476640 1 -- 172.21.15.32:0/769314593 shutdown_connections 2024-04-25T00:30:45.115 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.113+0000 7fcae1476640 1 -- 172.21.15.32:0/769314593 wait complete. 2024-04-25T00:30:45.126 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2024-04-25T00:30:45.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.234+0000 7f6323829640 1 -- 172.21.15.32:0/2357517963 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f631c107460 msgr2=0x7f631c1098b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:45.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.234+0000 7f6323829640 1 --2- 172.21.15.32:0/2357517963 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f631c107460 0x7f631c1098b0 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f6314009ca0 tx=0x7f631402a8c0 comp rx=0 tx=0).stop 2024-04-25T00:30:45.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.234+0000 7f6323829640 1 -- 172.21.15.32:0/2357517963 shutdown_connections 2024-04-25T00:30:45.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.234+0000 7f6323829640 1 --2- 172.21.15.32:0/2357517963 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f631c107460 0x7f631c1098b0 unknown :-1 s=CLOSED pgs=195 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:45.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.234+0000 7f6323829640 1 --2- 172.21.15.32:0/2357517963 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f631c104b30 0x7f631c106f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:45.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.234+0000 7f6323829640 1 --2- 172.21.15.32:0/2357517963 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f631c102200 0x7f631c1045f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:45.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.234+0000 7f6323829640 1 -- 172.21.15.32:0/2357517963 >> 172.21.15.32:0/2357517963 conn(0x7f631c0fbbf0 msgr2=0x7f631c0fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:45.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.234+0000 7f6323829640 1 -- 172.21.15.32:0/2357517963 shutdown_connections 2024-04-25T00:30:45.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.234+0000 7f6323829640 1 -- 172.21.15.32:0/2357517963 wait complete. 2024-04-25T00:30:45.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.235+0000 7f6323829640 1 Processor -- start 2024-04-25T00:30:45.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.235+0000 7f6323829640 1 -- start start 2024-04-25T00:30:45.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.235+0000 7f6323829640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f631c102200 0x7f631c101b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:45.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.235+0000 7f6323829640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f631c104b30 0x7f631c1001b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:45.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.235+0000 7f6323829640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f631c107460 0x7f631c1006f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:45.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.235+0000 7f6323829640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f631c10d420 con 0x7f631c104b30 2024-04-25T00:30:45.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.235+0000 7f6323829640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f631c10d2a0 con 0x7f631c107460 2024-04-25T00:30:45.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.235+0000 7f6323829640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f631c10d5a0 con 0x7f631c102200 2024-04-25T00:30:45.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.235+0000 7f6320dc4640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f631c104b30 0x7f631c1001b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:45.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.235+0000 7f6320dc4640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f631c104b30 0x7f631c1001b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:36660/0 (socket says 172.21.15.32:36660) 2024-04-25T00:30:45.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.235+0000 7f6320dc4640 1 -- 172.21.15.32:0/1554684040 learned_addr learned my addr 172.21.15.32:0/1554684040 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:45.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.235+0000 7f63215c5640 1 --2- 172.21.15.32:0/1554684040 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f631c102200 0x7f631c101b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:45.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.235+0000 7f6321dc6640 1 --2- 172.21.15.32:0/1554684040 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f631c107460 0x7f631c1006f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:45.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.236+0000 7f6320dc4640 1 -- 172.21.15.32:0/1554684040 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f631c102200 msgr2=0x7f631c101b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:45.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.236+0000 7f6320dc4640 1 --2- 172.21.15.32:0/1554684040 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f631c102200 0x7f631c101b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:45.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.236+0000 7f6320dc4640 1 -- 172.21.15.32:0/1554684040 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f631c107460 msgr2=0x7f631c1006f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:45.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.236+0000 7f6320dc4640 1 --2- 172.21.15.32:0/1554684040 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f631c107460 0x7f631c1006f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:45.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.236+0000 7f6320dc4640 1 -- 172.21.15.32:0/1554684040 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f631c100c30 con 0x7f631c104b30 2024-04-25T00:30:45.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.236+0000 7f6321dc6640 1 --2- 172.21.15.32:0/1554684040 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f631c107460 0x7f631c1006f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:30:45.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.236+0000 7f63215c5640 1 --2- 172.21.15.32:0/1554684040 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f631c102200 0x7f631c101b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:30:45.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.236+0000 7f6320dc4640 1 --2- 172.21.15.32:0/1554684040 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f631c104b30 0x7f631c1001b0 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f631800d770 tx=0x7f631800dc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:45.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.236+0000 7f63127fc640 1 -- 172.21.15.32:0/1554684040 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6318017070 con 0x7f631c104b30 2024-04-25T00:30:45.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.236+0000 7f63127fc640 1 -- 172.21.15.32:0/1554684040 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6318009c30 con 0x7f631c104b30 2024-04-25T00:30:45.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.236+0000 7f6323829640 1 -- 172.21.15.32:0/1554684040 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f631c100f20 con 0x7f631c104b30 2024-04-25T00:30:45.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.236+0000 7f63127fc640 1 -- 172.21.15.32:0/1554684040 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f631800f620 con 0x7f631c104b30 2024-04-25T00:30:45.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.236+0000 7f6323829640 1 -- 172.21.15.32:0/1554684040 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f631c1a6550 con 0x7f631c104b30 2024-04-25T00:30:45.240 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.237+0000 7f6323829640 1 -- 172.21.15.32:0/1554684040 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f631c0fd6e0 con 0x7f631c104b30 2024-04-25T00:30:45.240 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.238+0000 7f63127fc640 1 -- 172.21.15.32:0/1554684040 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f631801a070 con 0x7f631c104b30 2024-04-25T00:30:45.240 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.239+0000 7f63127fc640 1 --2- 172.21.15.32:0/1554684040 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f63080a2b90 0x7f63080a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:45.240 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.239+0000 7f63127fc640 1 -- 172.21.15.32:0/1554684040 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7f63180ce710 con 0x7f631c104b30 2024-04-25T00:30:45.241 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.239+0000 7f63215c5640 1 --2- 172.21.15.32:0/1554684040 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f63080a2b90 0x7f63080a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:45.241 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.239+0000 7f63215c5640 1 --2- 172.21.15.32:0/1554684040 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f63080a2b90 0x7f63080a5050 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f630c009c20 tx=0x7f630c005f90 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:45.243 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.243+0000 7f63127fc640 1 -- 172.21.15.32:0/1554684040 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f6318009de0 con 0x7f631c104b30 2024-04-25T00:30:45.431 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.430+0000 7f6323829640 1 -- 172.21.15.32:0/1554684040 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) -- 0x7f631c02d340 con 0x7f631c104b30 2024-04-25T00:30:45.431 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.431+0000 7f63127fc640 1 -- 172.21.15.32:0/1554684040 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v81) ==== 84+0+65 (secure 0 0 0) 0x7f631809bb10 con 0x7f631c104b30 2024-04-25T00:30:45.431 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:45.431 INFO:teuthology.orchestra.run.smithi032.stdout:[ 2024-04-25T00:30:45.431 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:30:45.431 INFO:teuthology.orchestra.run.smithi032.stdout: "poolnum": 1, 2024-04-25T00:30:45.432 INFO:teuthology.orchestra.run.smithi032.stdout: "poolname": ".mgr" 2024-04-25T00:30:45.432 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:30:45.432 INFO:teuthology.orchestra.run.smithi032.stdout:] 2024-04-25T00:30:45.433 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.432+0000 7f6323829640 1 -- 172.21.15.32:0/1554684040 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f63080a2b90 msgr2=0x7f63080a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:45.433 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.432+0000 7f6323829640 1 --2- 172.21.15.32:0/1554684040 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f63080a2b90 0x7f63080a5050 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f630c009c20 tx=0x7f630c005f90 comp rx=0 tx=0).stop 2024-04-25T00:30:45.433 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.432+0000 7f6323829640 1 -- 172.21.15.32:0/1554684040 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f631c104b30 msgr2=0x7f631c1001b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:45.433 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.432+0000 7f6323829640 1 --2- 172.21.15.32:0/1554684040 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f631c104b30 0x7f631c1001b0 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f631800d770 tx=0x7f631800dc30 comp rx=0 tx=0).stop 2024-04-25T00:30:45.433 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.432+0000 7f6323829640 1 -- 172.21.15.32:0/1554684040 shutdown_connections 2024-04-25T00:30:45.434 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.432+0000 7f6323829640 1 --2- 172.21.15.32:0/1554684040 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f63080a2b90 0x7f63080a5050 unknown :-1 s=CLOSED pgs=149 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:45.434 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.432+0000 7f6323829640 1 --2- 172.21.15.32:0/1554684040 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f631c107460 0x7f631c1006f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:45.434 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.432+0000 7f6323829640 1 --2- 172.21.15.32:0/1554684040 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f631c104b30 0x7f631c1001b0 unknown :-1 s=CLOSED pgs=196 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:45.434 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.432+0000 7f6323829640 1 --2- 172.21.15.32:0/1554684040 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f631c102200 0x7f631c101b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:45.434 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.432+0000 7f6323829640 1 -- 172.21.15.32:0/1554684040 >> 172.21.15.32:0/1554684040 conn(0x7f631c0fbbf0 msgr2=0x7f631c102b40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:45.434 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.433+0000 7f6323829640 1 -- 172.21.15.32:0/1554684040 shutdown_connections 2024-04-25T00:30:45.434 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.433+0000 7f6323829640 1 -- 172.21.15.32:0/1554684040 wait complete. 2024-04-25T00:30:45.444 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-04-25T00:30:45.444 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2024-04-25T00:30:45.555 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.553+0000 7f8f27f5a640 1 -- 172.21.15.32:0/2392955823 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8f20101870 msgr2=0x7f8f20101cf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:45.556 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.553+0000 7f8f27f5a640 1 --2- 172.21.15.32:0/2392955823 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8f20101870 0x7f8f20101cf0 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7f8f1c009900 tx=0x7f8f1c02a410 comp rx=0 tx=0).stop 2024-04-25T00:30:45.556 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.553+0000 7f8f27f5a640 1 -- 172.21.15.32:0/2392955823 shutdown_connections 2024-04-25T00:30:45.556 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.553+0000 7f8f27f5a640 1 --2- 172.21.15.32:0/2392955823 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8f20102230 0x7f8f20108ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:45.556 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.553+0000 7f8f27f5a640 1 --2- 172.21.15.32:0/2392955823 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8f20101870 0x7f8f20101cf0 unknown :-1 s=CLOSED pgs=197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:45.556 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.553+0000 7f8f27f5a640 1 --2- 172.21.15.32:0/2392955823 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8f20100680 0x7f8f20100aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:45.556 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.553+0000 7f8f27f5a640 1 -- 172.21.15.32:0/2392955823 >> 172.21.15.32:0/2392955823 conn(0x7f8f200fbc30 msgr2=0x7f8f200fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:45.556 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.553+0000 7f8f27f5a640 1 -- 172.21.15.32:0/2392955823 shutdown_connections 2024-04-25T00:30:45.556 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.553+0000 7f8f27f5a640 1 -- 172.21.15.32:0/2392955823 wait complete. 2024-04-25T00:30:45.556 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.553+0000 7f8f27f5a640 1 Processor -- start 2024-04-25T00:30:45.556 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.553+0000 7f8f27f5a640 1 -- start start 2024-04-25T00:30:45.556 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.553+0000 7f8f27f5a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8f20100680 0x7f8f2006a6c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:45.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.554+0000 7f8f27f5a640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8f20101870 0x7f8f2019c7f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:45.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.554+0000 7f8f27f5a640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8f20102230 0x7f8f2019ee60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:45.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.554+0000 7f8f27f5a640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f8f2010c350 con 0x7f8f20100680 2024-04-25T00:30:45.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.554+0000 7f8f27f5a640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f8f2010c1d0 con 0x7f8f20102230 2024-04-25T00:30:45.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.554+0000 7f8f27f5a640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f8f2010c4d0 con 0x7f8f20101870 2024-04-25T00:30:45.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.554+0000 7f8f25cf6640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8f20100680 0x7f8f2006a6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:45.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.554+0000 7f8f25cf6640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8f20100680 0x7f8f2006a6c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:36676/0 (socket says 172.21.15.32:36676) 2024-04-25T00:30:45.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.554+0000 7f8f25cf6640 1 -- 172.21.15.32:0/793666879 learned_addr learned my addr 172.21.15.32:0/793666879 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:45.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.554+0000 7f8f254f5640 1 --2- 172.21.15.32:0/793666879 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8f20101870 0x7f8f2019c7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:45.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.554+0000 7f8f25cf6640 1 -- 172.21.15.32:0/793666879 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8f20101870 msgr2=0x7f8f2019c7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:45.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.554+0000 7f8f264f7640 1 --2- 172.21.15.32:0/793666879 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8f20102230 0x7f8f2019ee60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:45.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.554+0000 7f8f25cf6640 1 --2- 172.21.15.32:0/793666879 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8f20101870 0x7f8f2019c7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:45.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.554+0000 7f8f25cf6640 1 -- 172.21.15.32:0/793666879 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8f20102230 msgr2=0x7f8f2019ee60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:45.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.554+0000 7f8f25cf6640 1 --2- 172.21.15.32:0/793666879 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8f20102230 0x7f8f2019ee60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:45.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.554+0000 7f8f25cf6640 1 -- 172.21.15.32:0/793666879 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8f2019f720 con 0x7f8f20100680 2024-04-25T00:30:45.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.554+0000 7f8f254f5640 1 --2- 172.21.15.32:0/793666879 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8f20101870 0x7f8f2019c7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:30:45.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.554+0000 7f8f25cf6640 1 --2- 172.21.15.32:0/793666879 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8f20100680 0x7f8f2006a6c0 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f8f1000e9c0 tx=0x7f8f1000ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:45.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.554+0000 7f8f16ffd640 1 -- 172.21.15.32:0/793666879 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8f1000cd80 con 0x7f8f20100680 2024-04-25T00:30:45.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.554+0000 7f8f27f5a640 1 -- 172.21.15.32:0/793666879 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8f2019fa10 con 0x7f8f20100680 2024-04-25T00:30:45.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.554+0000 7f8f16ffd640 1 -- 172.21.15.32:0/793666879 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8f100035f0 con 0x7f8f20100680 2024-04-25T00:30:45.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.554+0000 7f8f16ffd640 1 -- 172.21.15.32:0/793666879 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8f10010690 con 0x7f8f20100680 2024-04-25T00:30:45.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.554+0000 7f8f27f5a640 1 -- 172.21.15.32:0/793666879 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8f2019fee0 con 0x7f8f20100680 2024-04-25T00:30:45.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.555+0000 7f8f27f5a640 1 -- 172.21.15.32:0/793666879 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8f201079a0 con 0x7f8f20100680 2024-04-25T00:30:45.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.557+0000 7f8f16ffd640 1 -- 172.21.15.32:0/793666879 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f8f100037a0 con 0x7f8f20100680 2024-04-25T00:30:45.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.557+0000 7f8f16ffd640 1 --2- 172.21.15.32:0/793666879 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8f0c0a2b90 0x7f8f0c0a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:45.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.557+0000 7f8f16ffd640 1 -- 172.21.15.32:0/793666879 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 7127+0+0 (secure 0 0 0) 0x7f8f100cf170 con 0x7f8f20100680 2024-04-25T00:30:45.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.557+0000 7f8f254f5640 1 --2- 172.21.15.32:0/793666879 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8f0c0a2b90 0x7f8f0c0a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:45.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.558+0000 7f8f254f5640 1 --2- 172.21.15.32:0/793666879 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8f0c0a2b90 0x7f8f0c0a5050 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f8f1c006010 tx=0x7f8f1c000bc0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:45.561 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.561+0000 7f8f16ffd640 1 -- 172.21.15.32:0/793666879 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f8f1009c570 con 0x7f8f20100680 2024-04-25T00:30:45.646 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:30:45.646 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:45 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/347507021' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:30:45.646 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:45 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/769314593' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:30:45.647 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:45 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1554684040' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-04-25T00:30:45.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:30:45.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:45 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/347507021' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:30:45.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:45 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/769314593' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:30:45.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:45 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1554684040' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-04-25T00:30:45.753 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:45.753+0000 7f8f27f5a640 1 -- 172.21.15.32:0/793666879 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) -- 0x7f8f2002d170 con 0x7f8f20100680 2024-04-25T00:30:45.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:30:45.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:45 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/347507021' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:30:45.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:45 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/769314593' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:30:45.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:45 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1554684040' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-04-25T00:30:46.497 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.497+0000 7f8f16ffd640 1 -- 172.21.15.32:0/793666879 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v82) ==== 138+0+0 (secure 0 0 0) 0x7f8f1009be70 con 0x7f8f20100680 2024-04-25T00:30:46.498 INFO:teuthology.orchestra.run.smithi032.stderr:pool 'cephfs_metadata' created 2024-04-25T00:30:46.500 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.499+0000 7f8f27f5a640 1 -- 172.21.15.32:0/793666879 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8f0c0a2b90 msgr2=0x7f8f0c0a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:46.500 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.499+0000 7f8f27f5a640 1 --2- 172.21.15.32:0/793666879 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8f0c0a2b90 0x7f8f0c0a5050 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f8f1c006010 tx=0x7f8f1c000bc0 comp rx=0 tx=0).stop 2024-04-25T00:30:46.500 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.499+0000 7f8f27f5a640 1 -- 172.21.15.32:0/793666879 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8f20100680 msgr2=0x7f8f2006a6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:46.500 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.499+0000 7f8f27f5a640 1 --2- 172.21.15.32:0/793666879 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8f20100680 0x7f8f2006a6c0 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f8f1000e9c0 tx=0x7f8f1000ee80 comp rx=0 tx=0).stop 2024-04-25T00:30:46.500 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.499+0000 7f8f27f5a640 1 -- 172.21.15.32:0/793666879 shutdown_connections 2024-04-25T00:30:46.500 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.499+0000 7f8f27f5a640 1 --2- 172.21.15.32:0/793666879 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8f0c0a2b90 0x7f8f0c0a5050 unknown :-1 s=CLOSED pgs=150 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:46.500 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.499+0000 7f8f27f5a640 1 --2- 172.21.15.32:0/793666879 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8f20102230 0x7f8f2019ee60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:46.500 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.499+0000 7f8f27f5a640 1 --2- 172.21.15.32:0/793666879 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8f20101870 0x7f8f2019c7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:46.500 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.499+0000 7f8f27f5a640 1 --2- 172.21.15.32:0/793666879 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8f20100680 0x7f8f2006a6c0 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:46.501 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.499+0000 7f8f27f5a640 1 -- 172.21.15.32:0/793666879 >> 172.21.15.32:0/793666879 conn(0x7f8f200fbc30 msgr2=0x7f8f20104b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:46.501 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.499+0000 7f8f27f5a640 1 -- 172.21.15.32:0/793666879 shutdown_connections 2024-04-25T00:30:46.501 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.499+0000 7f8f27f5a640 1 -- 172.21.15.32:0/793666879 wait complete. 2024-04-25T00:30:46.514 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.05 2024-04-25T00:30:46.625 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.622+0000 7f60a541c640 1 -- 172.21.15.32:0/55335579 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f60a0102200 msgr2=0x7f60a01045f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:46.625 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.622+0000 7f60a541c640 1 --2- 172.21.15.32:0/55335579 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f60a0102200 0x7f60a01045f0 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f6098009900 tx=0x7f609802a410 comp rx=0 tx=0).stop 2024-04-25T00:30:46.625 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.622+0000 7f60a541c640 1 -- 172.21.15.32:0/55335579 shutdown_connections 2024-04-25T00:30:46.625 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.622+0000 7f60a541c640 1 --2- 172.21.15.32:0/55335579 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f60a0107460 0x7f60a01098b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:46.625 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.622+0000 7f60a541c640 1 --2- 172.21.15.32:0/55335579 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f60a0104b30 0x7f60a0106f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:46.626 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.622+0000 7f60a541c640 1 --2- 172.21.15.32:0/55335579 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f60a0102200 0x7f60a01045f0 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:46.626 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.622+0000 7f60a541c640 1 -- 172.21.15.32:0/55335579 >> 172.21.15.32:0/55335579 conn(0x7f60a00fbbf0 msgr2=0x7f60a00fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:46.626 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.622+0000 7f60a541c640 1 -- 172.21.15.32:0/55335579 shutdown_connections 2024-04-25T00:30:46.626 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.622+0000 7f60a541c640 1 -- 172.21.15.32:0/55335579 wait complete. 2024-04-25T00:30:46.626 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.623+0000 7f60a541c640 1 Processor -- start 2024-04-25T00:30:46.626 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.623+0000 7f60a541c640 1 -- start start 2024-04-25T00:30:46.626 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.623+0000 7f60a541c640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f60a0102200 0x7f60a0197af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:46.626 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.623+0000 7f60a541c640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f60a0104b30 0x7f60a0198030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:46.626 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.623+0000 7f60a541c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f60a0107460 0x7f60a019f0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:46.626 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.623+0000 7f60a541c640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f60a010d420 con 0x7f60a0107460 2024-04-25T00:30:46.626 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.623+0000 7f60a541c640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f60a010d2a0 con 0x7f60a0102200 2024-04-25T00:30:46.626 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.623+0000 7f60a541c640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f60a010d5a0 con 0x7f60a0104b30 2024-04-25T00:30:46.626 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.623+0000 7f609f7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f60a0107460 0x7f60a019f0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:46.627 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.623+0000 7f609f7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f60a0107460 0x7f60a019f0a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:36690/0 (socket says 172.21.15.32:36690) 2024-04-25T00:30:46.627 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.623+0000 7f609f7fe640 1 -- 172.21.15.32:0/3720145513 learned_addr learned my addr 172.21.15.32:0/3720145513 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:46.627 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.623+0000 7f609e7fc640 1 --2- 172.21.15.32:0/3720145513 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f60a0104b30 0x7f60a0198030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:46.627 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.624+0000 7f609effd640 1 --2- 172.21.15.32:0/3720145513 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f60a0102200 0x7f60a0197af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:46.627 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.624+0000 7f609f7fe640 1 -- 172.21.15.32:0/3720145513 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f60a0104b30 msgr2=0x7f60a0198030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:46.627 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.624+0000 7f609f7fe640 1 --2- 172.21.15.32:0/3720145513 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f60a0104b30 0x7f60a0198030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:46.627 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.624+0000 7f609f7fe640 1 -- 172.21.15.32:0/3720145513 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f60a0102200 msgr2=0x7f60a0197af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:46.627 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.624+0000 7f609f7fe640 1 --2- 172.21.15.32:0/3720145513 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f60a0102200 0x7f60a0197af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:46.627 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.624+0000 7f609f7fe640 1 -- 172.21.15.32:0/3720145513 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f60a019f710 con 0x7f60a0107460 2024-04-25T00:30:46.627 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.624+0000 7f609effd640 1 --2- 172.21.15.32:0/3720145513 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f60a0102200 0x7f60a0197af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:30:46.627 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.624+0000 7f609f7fe640 1 --2- 172.21.15.32:0/3720145513 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f60a0107460 0x7f60a019f0a0 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f609000eaa0 tx=0x7f609000ef60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:46.627 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.624+0000 7f6097fff640 1 -- 172.21.15.32:0/3720145513 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f609000cdb0 con 0x7f60a0107460 2024-04-25T00:30:46.627 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.624+0000 7f6097fff640 1 -- 172.21.15.32:0/3720145513 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6090005020 con 0x7f60a0107460 2024-04-25T00:30:46.627 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.624+0000 7f60a541c640 1 -- 172.21.15.32:0/3720145513 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f60a019fa00 con 0x7f60a0107460 2024-04-25T00:30:46.628 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.624+0000 7f6097fff640 1 -- 172.21.15.32:0/3720145513 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f60900116b0 con 0x7f60a0107460 2024-04-25T00:30:46.628 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.624+0000 7f60a541c640 1 -- 172.21.15.32:0/3720145513 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f60a019ffa0 con 0x7f60a0107460 2024-04-25T00:30:46.628 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.625+0000 7f60a541c640 1 -- 172.21.15.32:0/3720145513 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f60a00fd6e0 con 0x7f60a0107460 2024-04-25T00:30:46.629 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.626+0000 7f6097fff640 1 -- 172.21.15.32:0/3720145513 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f6090005560 con 0x7f60a0107460 2024-04-25T00:30:46.629 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.627+0000 7f6097fff640 1 --2- 172.21.15.32:0/3720145513 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f608c0a2c10 0x7f608c0a50d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:46.629 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.627+0000 7f609effd640 1 --2- 172.21.15.32:0/3720145513 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f608c0a2c10 0x7f608c0a50d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:46.629 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.627+0000 7f6097fff640 1 -- 172.21.15.32:0/3720145513 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(82..82 src has 1..82) ==== 7497+0+0 (secure 0 0 0) 0x7f60900cf320 con 0x7f60a0107460 2024-04-25T00:30:46.629 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.628+0000 7f609effd640 1 --2- 172.21.15.32:0/3720145513 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f608c0a2c10 0x7f608c0a50d0 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f6098000c00 tx=0x7f60980095f0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:46.632 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.631+0000 7f6097fff640 1 -- 172.21.15.32:0/3720145513 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f6090016030 con 0x7f60a0107460 2024-04-25T00:30:46.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:46 smithi032 ceph-mon[29357]: pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:46.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:46 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/793666879' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-04-25T00:30:46.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:46 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:46.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:46 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:46.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:46 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:46.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:46 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:46.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:46 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:46.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:46 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:46.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:46 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:46.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:46 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:30:46.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:46 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:46.820 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:46.819+0000 7f60a541c640 1 -- 172.21.15.32:0/3720145513 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05} v 0) -- 0x7f60a002d170 con 0x7f60a0107460 2024-04-25T00:30:46.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:46 smithi121 ceph-mon[32231]: pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:46.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:46 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/793666879' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-04-25T00:30:46.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:46 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:46.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:46 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:46.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:46 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:46.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:46 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:46.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:46 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:46.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:46 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:46.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:46 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:46.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:46 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:30:46.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:46 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:46.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:46 smithi130 ceph-mon[32525]: pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:46.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:46 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/793666879' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-04-25T00:30:46.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:46 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:46.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:46 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:46.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:46 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:46.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:46 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:46.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:46 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:46.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:46 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:46.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:46 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:30:46.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:46 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:30:46.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:46 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:30:47.500 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.499+0000 7f6097fff640 1 -- 172.21.15.32:0/3720145513 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05}]=0 pool 'cephfs_data' created v83) ==== 171+0+0 (secure 0 0 0) 0x7f609009c590 con 0x7f60a0107460 2024-04-25T00:30:47.500 INFO:teuthology.orchestra.run.smithi032.stderr:pool 'cephfs_data' created 2024-04-25T00:30:47.503 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.502+0000 7f60a541c640 1 -- 172.21.15.32:0/3720145513 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f608c0a2c10 msgr2=0x7f608c0a50d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:47.503 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.502+0000 7f60a541c640 1 --2- 172.21.15.32:0/3720145513 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f608c0a2c10 0x7f608c0a50d0 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f6098000c00 tx=0x7f60980095f0 comp rx=0 tx=0).stop 2024-04-25T00:30:47.503 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.502+0000 7f60a541c640 1 -- 172.21.15.32:0/3720145513 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f60a0107460 msgr2=0x7f60a019f0a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:47.503 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.502+0000 7f60a541c640 1 --2- 172.21.15.32:0/3720145513 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f60a0107460 0x7f60a019f0a0 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f609000eaa0 tx=0x7f609000ef60 comp rx=0 tx=0).stop 2024-04-25T00:30:47.503 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.502+0000 7f60a541c640 1 -- 172.21.15.32:0/3720145513 shutdown_connections 2024-04-25T00:30:47.503 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.502+0000 7f60a541c640 1 --2- 172.21.15.32:0/3720145513 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f608c0a2c10 0x7f608c0a50d0 unknown :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:47.504 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.502+0000 7f60a541c640 1 --2- 172.21.15.32:0/3720145513 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f60a0107460 0x7f60a019f0a0 unknown :-1 s=CLOSED pgs=200 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:47.504 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.502+0000 7f60a541c640 1 --2- 172.21.15.32:0/3720145513 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f60a0104b30 0x7f60a0198030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:47.504 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.502+0000 7f60a541c640 1 --2- 172.21.15.32:0/3720145513 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f60a0102200 0x7f60a0197af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:47.504 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.502+0000 7f60a541c640 1 -- 172.21.15.32:0/3720145513 >> 172.21.15.32:0/3720145513 conn(0x7f60a00fbbf0 msgr2=0x7f60a0102ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:47.504 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.503+0000 7f60a541c640 1 -- 172.21.15.32:0/3720145513 shutdown_connections 2024-04-25T00:30:47.504 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.503+0000 7f60a541c640 1 -- 172.21.15.32:0/3720145513 wait complete. 2024-04-25T00:30:47.523 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2024-04-25T00:30:47.633 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.630+0000 7f052a853640 1 -- 172.21.15.32:0/3203397290 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0524102230 msgr2=0x7f0524108ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:47.633 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.630+0000 7f052a853640 1 --2- 172.21.15.32:0/3203397290 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0524102230 0x7f0524108ac0 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f051c009ca0 tx=0x7f051c02a8c0 comp rx=0 tx=0).stop 2024-04-25T00:30:47.633 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.630+0000 7f052a853640 1 -- 172.21.15.32:0/3203397290 shutdown_connections 2024-04-25T00:30:47.633 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.630+0000 7f052a853640 1 --2- 172.21.15.32:0/3203397290 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0524102230 0x7f0524108ac0 unknown :-1 s=CLOSED pgs=201 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:47.633 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.630+0000 7f052a853640 1 --2- 172.21.15.32:0/3203397290 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0524101870 0x7f0524101cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:47.633 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.630+0000 7f052a853640 1 --2- 172.21.15.32:0/3203397290 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0524100680 0x7f0524100aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:47.633 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.630+0000 7f052a853640 1 -- 172.21.15.32:0/3203397290 >> 172.21.15.32:0/3203397290 conn(0x7f05240fbbf0 msgr2=0x7f05240fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:47.634 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.630+0000 7f052a853640 1 -- 172.21.15.32:0/3203397290 shutdown_connections 2024-04-25T00:30:47.634 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.630+0000 7f052a853640 1 -- 172.21.15.32:0/3203397290 wait complete. 2024-04-25T00:30:47.634 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.630+0000 7f052a853640 1 Processor -- start 2024-04-25T00:30:47.634 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.631+0000 7f052a853640 1 -- start start 2024-04-25T00:30:47.634 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.631+0000 7f052a853640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0524100680 0x7f0524104610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:47.634 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.631+0000 7f052a853640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0524101870 0x7f0524102c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:47.634 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.631+0000 7f052a853640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0524102230 0x7f0524103190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:47.634 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.631+0000 7f052a853640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f052410c440 con 0x7f0524100680 2024-04-25T00:30:47.634 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.631+0000 7f052a853640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f052410c2c0 con 0x7f0524101870 2024-04-25T00:30:47.634 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.631+0000 7f052a853640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f052410c5c0 con 0x7f0524102230 2024-04-25T00:30:47.635 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.631+0000 7f0523fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0524100680 0x7f0524104610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:47.635 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.631+0000 7f0523fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0524100680 0x7f0524104610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:36706/0 (socket says 172.21.15.32:36706) 2024-04-25T00:30:47.635 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.631+0000 7f0523fff640 1 -- 172.21.15.32:0/252303026 learned_addr learned my addr 172.21.15.32:0/252303026 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:47.635 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.631+0000 7f05237fe640 1 --2- 172.21.15.32:0/252303026 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0524101870 0x7f0524102c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:47.637 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.631+0000 7f0528df0640 1 --2- 172.21.15.32:0/252303026 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0524102230 0x7f0524103190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:47.637 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.631+0000 7f0523fff640 1 -- 172.21.15.32:0/252303026 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0524102230 msgr2=0x7f0524103190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:47.637 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.631+0000 7f0523fff640 1 --2- 172.21.15.32:0/252303026 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0524102230 0x7f0524103190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:47.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.631+0000 7f0523fff640 1 -- 172.21.15.32:0/252303026 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0524101870 msgr2=0x7f0524102c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:47.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.631+0000 7f0523fff640 1 --2- 172.21.15.32:0/252303026 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0524101870 0x7f0524102c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:47.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.631+0000 7f0523fff640 1 -- 172.21.15.32:0/252303026 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0524103a20 con 0x7f0524100680 2024-04-25T00:30:47.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.632+0000 7f0523fff640 1 --2- 172.21.15.32:0/252303026 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0524100680 0x7f0524104610 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f051400b450 tx=0x7f051400b850 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:47.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.632+0000 7f05217fa640 1 -- 172.21.15.32:0/252303026 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0514015020 con 0x7f0524100680 2024-04-25T00:30:47.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.632+0000 7f05217fa640 1 -- 172.21.15.32:0/252303026 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0514011d10 con 0x7f0524100680 2024-04-25T00:30:47.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.632+0000 7f052a853640 1 -- 172.21.15.32:0/252303026 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f05241a1cc0 con 0x7f0524100680 2024-04-25T00:30:47.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.632+0000 7f05217fa640 1 -- 172.21.15.32:0/252303026 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f051400f2e0 con 0x7f0524100680 2024-04-25T00:30:47.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.632+0000 7f052a853640 1 -- 172.21.15.32:0/252303026 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f05241a21b0 con 0x7f0524100680 2024-04-25T00:30:47.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.633+0000 7f052a853640 1 -- 172.21.15.32:0/252303026 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f05241a2440 con 0x7f0524100680 2024-04-25T00:30:47.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.634+0000 7f05217fa640 1 -- 172.21.15.32:0/252303026 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f0514018020 con 0x7f0524100680 2024-04-25T00:30:47.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.635+0000 7f05217fa640 1 --2- 172.21.15.32:0/252303026 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f050c0a2c10 0x7f050c0a50d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:47.639 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.635+0000 7f05237fe640 1 --2- 172.21.15.32:0/252303026 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f050c0a2c10 0x7f050c0a50d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:47.639 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.635+0000 7f05217fa640 1 -- 172.21.15.32:0/252303026 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7879+0+0 (secure 0 0 0) 0x7f05140cec60 con 0x7f0524100680 2024-04-25T00:30:47.639 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.635+0000 7f05237fe640 1 --2- 172.21.15.32:0/252303026 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f050c0a2c10 0x7f050c0a50d0 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7f0524104380 tx=0x7f0518009110 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:47.639 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.638+0000 7f05217fa640 1 -- 172.21.15.32:0/252303026 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f051401f020 con 0x7f0524100680 2024-04-25T00:30:47.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:47 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/793666879' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-04-25T00:30:47.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:47 smithi121 ceph-mon[32231]: osdmap e82: 12 total, 12 up, 12 in 2024-04-25T00:30:47.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:47 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3720145513' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05}]: dispatch 2024-04-25T00:30:47.865 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:47.865+0000 7f052a853640 1 -- 172.21.15.32:0/252303026 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) -- 0x7f05241a2610 con 0x7f0524100680 2024-04-25T00:30:47.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:47 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/793666879' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-04-25T00:30:47.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:47 smithi130 ceph-mon[32525]: osdmap e82: 12 total, 12 up, 12 in 2024-04-25T00:30:47.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:47 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3720145513' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05}]: dispatch 2024-04-25T00:30:47.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:47 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/793666879' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-04-25T00:30:47.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:47 smithi032 ceph-mon[29357]: osdmap e82: 12 total, 12 up, 12 in 2024-04-25T00:30:47.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:47 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3720145513' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05}]: dispatch 2024-04-25T00:30:48.734 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.734+0000 7f05217fa640 1 -- 172.21.15.32:0/252303026 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 2 and data pool 3 v14) ==== 172+0+0 (secure 0 0 0) 0x7f051409bd70 con 0x7f0524100680 2024-04-25T00:30:48.734 INFO:teuthology.orchestra.run.smithi032.stderr:new fs with metadata pool 2 and data pool 3 2024-04-25T00:30:48.737 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.736+0000 7f052a853640 1 -- 172.21.15.32:0/252303026 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f050c0a2c10 msgr2=0x7f050c0a50d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:48.737 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.736+0000 7f052a853640 1 --2- 172.21.15.32:0/252303026 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f050c0a2c10 0x7f050c0a50d0 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7f0524104380 tx=0x7f0518009110 comp rx=0 tx=0).stop 2024-04-25T00:30:48.737 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.736+0000 7f052a853640 1 -- 172.21.15.32:0/252303026 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0524100680 msgr2=0x7f0524104610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:48.737 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.736+0000 7f052a853640 1 --2- 172.21.15.32:0/252303026 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0524100680 0x7f0524104610 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f051400b450 tx=0x7f051400b850 comp rx=0 tx=0).stop 2024-04-25T00:30:48.737 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.737+0000 7f052a853640 1 -- 172.21.15.32:0/252303026 shutdown_connections 2024-04-25T00:30:48.738 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.737+0000 7f052a853640 1 --2- 172.21.15.32:0/252303026 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f050c0a2c10 0x7f050c0a50d0 unknown :-1 s=CLOSED pgs=152 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:48.738 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.737+0000 7f052a853640 1 --2- 172.21.15.32:0/252303026 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0524102230 0x7f0524103190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:48.738 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.737+0000 7f052a853640 1 --2- 172.21.15.32:0/252303026 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0524101870 0x7f0524102c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:48.738 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.737+0000 7f052a853640 1 --2- 172.21.15.32:0/252303026 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0524100680 0x7f0524104610 unknown :-1 s=CLOSED pgs=202 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:48.738 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.737+0000 7f052a853640 1 -- 172.21.15.32:0/252303026 >> 172.21.15.32:0/252303026 conn(0x7f05240fbbf0 msgr2=0x7f05240fd690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:48.738 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.737+0000 7f052a853640 1 -- 172.21.15.32:0/252303026 shutdown_connections 2024-04-25T00:30:48.738 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.737+0000 7f052a853640 1 -- 172.21.15.32:0/252303026 wait complete. 2024-04-25T00:30:48.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:48 smithi032 ceph-mon[29357]: pgmap v284: 33 pgs: 3 creating+peering, 1 active+clean, 29 unknown; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:48.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:48 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3720145513' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05}]': finished 2024-04-25T00:30:48.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:48 smithi032 ceph-mon[29357]: osdmap e83: 12 total, 12 up, 12 in 2024-04-25T00:30:48.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:48 smithi032 ceph-mon[29357]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-25T00:30:48.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:48 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/252303026' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-04-25T00:30:48.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:48 smithi032 ceph-mon[29357]: daemon mds.b assigned to filesystem cephfs as rank 0 2024-04-25T00:30:48.753 DEBUG:tasks.cephfs.filesystem:EC profile is ['m=2', 'k=2', 'crush-failure-domain=osd'] 2024-04-25T00:30:48.754 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd erasure-code-profile set cephfs_data_ec m=2 k=2 crush-failure-domain=osd 2024-04-25T00:30:48.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:48 smithi121 ceph-mon[32231]: pgmap v284: 33 pgs: 3 creating+peering, 1 active+clean, 29 unknown; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:48.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:48 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3720145513' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05}]': finished 2024-04-25T00:30:48.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:48 smithi121 ceph-mon[32231]: osdmap e83: 12 total, 12 up, 12 in 2024-04-25T00:30:48.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:48 smithi121 ceph-mon[32231]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-25T00:30:48.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:48 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/252303026' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-04-25T00:30:48.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:48 smithi121 ceph-mon[32231]: daemon mds.b assigned to filesystem cephfs as rank 0 2024-04-25T00:30:48.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.866+0000 7f74515ed640 1 -- 172.21.15.32:0/3185408339 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f744c101870 msgr2=0x7f744c101cf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:48.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.866+0000 7f74515ed640 1 --2- 172.21.15.32:0/3185408339 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f744c101870 0x7f744c101cf0 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f7440009900 tx=0x7f744002a410 comp rx=0 tx=0).stop 2024-04-25T00:30:48.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.866+0000 7f74515ed640 1 -- 172.21.15.32:0/3185408339 shutdown_connections 2024-04-25T00:30:48.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.866+0000 7f74515ed640 1 --2- 172.21.15.32:0/3185408339 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f744c102230 0x7f744c108ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:48.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.866+0000 7f74515ed640 1 --2- 172.21.15.32:0/3185408339 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f744c101870 0x7f744c101cf0 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:48.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.866+0000 7f74515ed640 1 --2- 172.21.15.32:0/3185408339 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f744c100680 0x7f744c100aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:48.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.866+0000 7f74515ed640 1 -- 172.21.15.32:0/3185408339 >> 172.21.15.32:0/3185408339 conn(0x7f744c0fbbf0 msgr2=0x7f744c0fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:48.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.866+0000 7f74515ed640 1 -- 172.21.15.32:0/3185408339 shutdown_connections 2024-04-25T00:30:48.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.866+0000 7f74515ed640 1 -- 172.21.15.32:0/3185408339 wait complete. 2024-04-25T00:30:48.870 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.867+0000 7f74515ed640 1 Processor -- start 2024-04-25T00:30:48.870 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.867+0000 7f74515ed640 1 -- start start 2024-04-25T00:30:48.870 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.867+0000 7f74515ed640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f744c100680 0x7f744c197ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:48.870 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.867+0000 7f74515ed640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f744c101870 0x7f744c198010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:48.870 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.867+0000 7f74515ed640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f744c102230 0x7f744c19f080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:48.870 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.867+0000 7f74515ed640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f744c10c2e0 con 0x7f744c101870 2024-04-25T00:30:48.870 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.867+0000 7f74515ed640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f744c10c160 con 0x7f744c102230 2024-04-25T00:30:48.870 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.867+0000 7f74515ed640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f744c10c460 con 0x7f744c100680 2024-04-25T00:30:48.870 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.867+0000 7f744a7fc640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f744c101870 0x7f744c198010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:48.870 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.867+0000 7f744a7fc640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f744c101870 0x7f744c198010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:36730/0 (socket says 172.21.15.32:36730) 2024-04-25T00:30:48.870 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.867+0000 7f744a7fc640 1 -- 172.21.15.32:0/870825602 learned_addr learned my addr 172.21.15.32:0/870825602 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:48.870 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.867+0000 7f744affd640 1 --2- 172.21.15.32:0/870825602 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f744c100680 0x7f744c197ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:48.870 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.867+0000 7f744b7fe640 1 --2- 172.21.15.32:0/870825602 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f744c102230 0x7f744c19f080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:48.870 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.867+0000 7f744a7fc640 1 -- 172.21.15.32:0/870825602 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f744c100680 msgr2=0x7f744c197ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:48.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.867+0000 7f744a7fc640 1 --2- 172.21.15.32:0/870825602 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f744c100680 0x7f744c197ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:48.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.867+0000 7f744a7fc640 1 -- 172.21.15.32:0/870825602 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f744c102230 msgr2=0x7f744c19f080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:48.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.867+0000 7f744a7fc640 1 --2- 172.21.15.32:0/870825602 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f744c102230 0x7f744c19f080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:48.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.867+0000 7f744a7fc640 1 -- 172.21.15.32:0/870825602 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f744c19f6f0 con 0x7f744c101870 2024-04-25T00:30:48.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.868+0000 7f744b7fe640 1 --2- 172.21.15.32:0/870825602 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f744c102230 0x7f744c19f080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:30:48.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.868+0000 7f744a7fc640 1 --2- 172.21.15.32:0/870825602 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f744c101870 0x7f744c198010 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f7440009230 tx=0x7f74400042f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:48.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.868+0000 7f7433fff640 1 -- 172.21.15.32:0/870825602 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f744003f030 con 0x7f744c101870 2024-04-25T00:30:48.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.868+0000 7f74515ed640 1 -- 172.21.15.32:0/870825602 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f744c19f980 con 0x7f744c101870 2024-04-25T00:30:48.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.868+0000 7f7433fff640 1 -- 172.21.15.32:0/870825602 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7440003ce0 con 0x7f744c101870 2024-04-25T00:30:48.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.868+0000 7f7433fff640 1 -- 172.21.15.32:0/870825602 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7440035070 con 0x7f744c101870 2024-04-25T00:30:48.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.868+0000 7f74515ed640 1 -- 172.21.15.32:0/870825602 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f744c19fe70 con 0x7f744c101870 2024-04-25T00:30:48.872 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.869+0000 7f74515ed640 1 -- 172.21.15.32:0/870825602 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f744c1a00a0 con 0x7f744c101870 2024-04-25T00:30:48.872 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.870+0000 7f7433fff640 1 -- 172.21.15.32:0/870825602 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f7440042030 con 0x7f744c101870 2024-04-25T00:30:48.872 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.871+0000 7f7433fff640 1 --2- 172.21.15.32:0/870825602 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f74340a2b90 0x7f74340a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:48.872 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.871+0000 7f744affd640 1 --2- 172.21.15.32:0/870825602 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f74340a2b90 0x7f74340a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:48.872 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.871+0000 7f7433fff640 1 -- 172.21.15.32:0/870825602 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(84..84 src has 1..84) ==== 7979+0+0 (secure 0 0 0) 0x7f74400ef890 con 0x7f744c101870 2024-04-25T00:30:48.872 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.871+0000 7f744affd640 1 --2- 172.21.15.32:0/870825602 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f74340a2b90 0x7f74340a5050 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f744c1016d0 tx=0x7f743c005f90 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:48.874 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:48.874+0000 7f7433fff640 1 -- 172.21.15.32:0/870825602 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f74400bc940 con 0x7f744c101870 2024-04-25T00:30:48.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:48 smithi130 ceph-mon[32525]: pgmap v284: 33 pgs: 3 creating+peering, 1 active+clean, 29 unknown; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:30:48.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:48 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3720145513' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05}]': finished 2024-04-25T00:30:48.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:48 smithi130 ceph-mon[32525]: osdmap e83: 12 total, 12 up, 12 in 2024-04-25T00:30:48.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:48 smithi130 ceph-mon[32525]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-25T00:30:48.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:48 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/252303026' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-04-25T00:30:48.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:48 smithi130 ceph-mon[32525]: daemon mds.b assigned to filesystem cephfs as rank 0 2024-04-25T00:30:49.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.070+0000 7f74515ed640 1 -- 172.21.15.32:0/870825602 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]} v 0) -- 0x7f744c1a0270 con 0x7f744c101870 2024-04-25T00:30:49.520 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.519+0000 7f7433fff640 1 -- 172.21.15.32:0/870825602 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]}]=0 v85) ==== 157+0+0 (secure 0 0 0) 0x7f74400bc240 con 0x7f744c101870 2024-04-25T00:30:49.523 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.523+0000 7f74515ed640 1 -- 172.21.15.32:0/870825602 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f74340a2b90 msgr2=0x7f74340a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:49.524 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.523+0000 7f74515ed640 1 --2- 172.21.15.32:0/870825602 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f74340a2b90 0x7f74340a5050 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f744c1016d0 tx=0x7f743c005f90 comp rx=0 tx=0).stop 2024-04-25T00:30:49.524 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.523+0000 7f74515ed640 1 -- 172.21.15.32:0/870825602 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f744c101870 msgr2=0x7f744c198010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:49.524 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.523+0000 7f74515ed640 1 --2- 172.21.15.32:0/870825602 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f744c101870 0x7f744c198010 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f7440009230 tx=0x7f74400042f0 comp rx=0 tx=0).stop 2024-04-25T00:30:49.524 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.523+0000 7f74515ed640 1 -- 172.21.15.32:0/870825602 shutdown_connections 2024-04-25T00:30:49.524 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.523+0000 7f74515ed640 1 --2- 172.21.15.32:0/870825602 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f74340a2b90 0x7f74340a5050 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:49.524 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.523+0000 7f74515ed640 1 --2- 172.21.15.32:0/870825602 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f744c102230 0x7f744c19f080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:49.524 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.523+0000 7f74515ed640 1 --2- 172.21.15.32:0/870825602 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f744c101870 0x7f744c198010 unknown :-1 s=CLOSED pgs=204 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:49.525 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.523+0000 7f74515ed640 1 --2- 172.21.15.32:0/870825602 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f744c100680 0x7f744c197ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:49.525 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.523+0000 7f74515ed640 1 -- 172.21.15.32:0/870825602 >> 172.21.15.32:0/870825602 conn(0x7f744c0fbbf0 msgr2=0x7f744c0fd6e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:49.525 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.523+0000 7f74515ed640 1 -- 172.21.15.32:0/870825602 shutdown_connections 2024-04-25T00:30:49.525 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.523+0000 7f74515ed640 1 -- 172.21.15.32:0/870825602 wait complete. 2024-04-25T00:30:49.536 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data_ec erasure cephfs_data_ec --pg_num_min 64 --target_size_ratio 0.9 2024-04-25T00:30:49.651 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.649+0000 7f4a4eb29640 1 -- 172.21.15.32:0/3355084140 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4a48101890 msgr2=0x7f4a48101d10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:49.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.649+0000 7f4a4eb29640 1 --2- 172.21.15.32:0/3355084140 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4a48101890 0x7f4a48101d10 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f4a34009900 tx=0x7f4a3402a410 comp rx=0 tx=0).stop 2024-04-25T00:30:49.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.649+0000 7f4a4eb29640 1 -- 172.21.15.32:0/3355084140 shutdown_connections 2024-04-25T00:30:49.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.649+0000 7f4a4eb29640 1 --2- 172.21.15.32:0/3355084140 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4a48102250 0x7f4a48108ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:49.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.649+0000 7f4a4eb29640 1 --2- 172.21.15.32:0/3355084140 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4a48101890 0x7f4a48101d10 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:49.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.649+0000 7f4a4eb29640 1 --2- 172.21.15.32:0/3355084140 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4a481006a0 0x7f4a48100ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:49.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.649+0000 7f4a4eb29640 1 -- 172.21.15.32:0/3355084140 >> 172.21.15.32:0/3355084140 conn(0x7f4a480fbc30 msgr2=0x7f4a480fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:49.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.649+0000 7f4a4eb29640 1 -- 172.21.15.32:0/3355084140 shutdown_connections 2024-04-25T00:30:49.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.649+0000 7f4a4eb29640 1 -- 172.21.15.32:0/3355084140 wait complete. 2024-04-25T00:30:49.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.650+0000 7f4a4eb29640 1 Processor -- start 2024-04-25T00:30:49.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.650+0000 7f4a4eb29640 1 -- start start 2024-04-25T00:30:49.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.650+0000 7f4a4eb29640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4a481006a0 0x7f4a4806a910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:49.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.650+0000 7f4a4eb29640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4a48101890 0x7f4a4819c830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:49.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.650+0000 7f4a4eb29640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4a48102250 0x7f4a4819ee50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:49.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.650+0000 7f4a4eb29640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f4a4810c370 con 0x7f4a48102250 2024-04-25T00:30:49.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.650+0000 7f4a4eb29640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f4a4810c1f0 con 0x7f4a48101890 2024-04-25T00:30:49.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.650+0000 7f4a4eb29640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f4a4810c4f0 con 0x7f4a481006a0 2024-04-25T00:30:49.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.650+0000 7f4a4d0c6640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4a48102250 0x7f4a4819ee50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:49.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.650+0000 7f4a4d0c6640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4a48102250 0x7f4a4819ee50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:36744/0 (socket says 172.21.15.32:36744) 2024-04-25T00:30:49.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.650+0000 7f4a4d0c6640 1 -- 172.21.15.32:0/2913233743 learned_addr learned my addr 172.21.15.32:0/2913233743 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:49.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.650+0000 7f4a47fff640 1 --2- 172.21.15.32:0/2913233743 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4a48101890 0x7f4a4819c830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:49.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.650+0000 7f4a4c8c5640 1 --2- 172.21.15.32:0/2913233743 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4a481006a0 0x7f4a4806a910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:49.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.650+0000 7f4a4d0c6640 1 -- 172.21.15.32:0/2913233743 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4a481006a0 msgr2=0x7f4a4806a910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:49.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.650+0000 7f4a4d0c6640 1 --2- 172.21.15.32:0/2913233743 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4a481006a0 0x7f4a4806a910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:49.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.650+0000 7f4a4d0c6640 1 -- 172.21.15.32:0/2913233743 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4a48101890 msgr2=0x7f4a4819c830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:49.654 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.650+0000 7f4a4d0c6640 1 --2- 172.21.15.32:0/2913233743 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4a48101890 0x7f4a4819c830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:49.654 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.650+0000 7f4a4d0c6640 1 -- 172.21.15.32:0/2913233743 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4a4819f760 con 0x7f4a48102250 2024-04-25T00:30:49.654 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.650+0000 7f4a4c8c5640 1 --2- 172.21.15.32:0/2913233743 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4a481006a0 0x7f4a4806a910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:30:49.654 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.650+0000 7f4a4d0c6640 1 --2- 172.21.15.32:0/2913233743 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4a48102250 0x7f4a4819ee50 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f4a4000eaa0 tx=0x7f4a4000ef60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:49.654 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.650+0000 7f4a45ffb640 1 -- 172.21.15.32:0/2913233743 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a4000cdb0 con 0x7f4a48102250 2024-04-25T00:30:49.654 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.651+0000 7f4a45ffb640 1 -- 172.21.15.32:0/2913233743 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4a40005020 con 0x7f4a48102250 2024-04-25T00:30:49.654 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.651+0000 7f4a4eb29640 1 -- 172.21.15.32:0/2913233743 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4a4819f9f0 con 0x7f4a48102250 2024-04-25T00:30:49.654 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.651+0000 7f4a45ffb640 1 -- 172.21.15.32:0/2913233743 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a400116b0 con 0x7f4a48102250 2024-04-25T00:30:49.654 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.651+0000 7f4a4eb29640 1 -- 172.21.15.32:0/2913233743 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4a4819fec0 con 0x7f4a48102250 2024-04-25T00:30:49.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.652+0000 7f4a4eb29640 1 -- 172.21.15.32:0/2913233743 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4a48107a30 con 0x7f4a48102250 2024-04-25T00:30:49.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.653+0000 7f4a45ffb640 1 -- 172.21.15.32:0/2913233743 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f4a40005560 con 0x7f4a48102250 2024-04-25T00:30:49.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.653+0000 7f4a45ffb640 1 --2- 172.21.15.32:0/2913233743 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4a380a2b90 0x7f4a380a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:49.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.653+0000 7f4a4c8c5640 1 --2- 172.21.15.32:0/2913233743 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4a380a2b90 0x7f4a380a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:49.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.653+0000 7f4a45ffb640 1 -- 172.21.15.32:0/2913233743 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(85..85 src has 1..85) ==== 8277+0+0 (secure 0 0 0) 0x7f4a400cf2c0 con 0x7f4a48102250 2024-04-25T00:30:49.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.654+0000 7f4a4c8c5640 1 --2- 172.21.15.32:0/2913233743 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4a380a2b90 0x7f4a380a5050 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f4a481016f0 tx=0x7f4a3c005e70 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:49.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.658+0000 7f4a45ffb640 1 -- 172.21.15.32:0/2913233743 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f4a40016030 con 0x7f4a48102250 2024-04-25T00:30:49.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:49 smithi121 ceph-mon[32231]: osdmap e84: 12 total, 12 up, 12 in 2024-04-25T00:30:49.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:49 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/252303026' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-04-25T00:30:49.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:49 smithi121 ceph-mon[32231]: fsmap cephfs:1 {0=b=up:creating} 11 up:standby 2024-04-25T00:30:49.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:49 smithi121 ceph-mon[32231]: daemon mds.b is now active in filesystem cephfs as rank 0 2024-04-25T00:30:49.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:49 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/870825602' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]}]: dispatch 2024-04-25T00:30:49.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:49.849+0000 7f4a4eb29640 1 -- 172.21.15.32:0/2913233743 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9} v 0) -- 0x7f4a4802d170 con 0x7f4a48102250 2024-04-25T00:30:49.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:49 smithi130 ceph-mon[32525]: osdmap e84: 12 total, 12 up, 12 in 2024-04-25T00:30:49.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:49 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/252303026' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-04-25T00:30:49.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:49 smithi130 ceph-mon[32525]: fsmap cephfs:1 {0=b=up:creating} 11 up:standby 2024-04-25T00:30:49.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:49 smithi130 ceph-mon[32525]: daemon mds.b is now active in filesystem cephfs as rank 0 2024-04-25T00:30:49.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:49 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/870825602' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]}]: dispatch 2024-04-25T00:30:49.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:49 smithi032 ceph-mon[29357]: osdmap e84: 12 total, 12 up, 12 in 2024-04-25T00:30:49.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:49 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/252303026' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-04-25T00:30:49.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:49 smithi032 ceph-mon[29357]: fsmap cephfs:1 {0=b=up:creating} 11 up:standby 2024-04-25T00:30:49.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:49 smithi032 ceph-mon[29357]: daemon mds.b is now active in filesystem cephfs as rank 0 2024-04-25T00:30:49.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:49 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/870825602' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]}]: dispatch 2024-04-25T00:30:50.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:50 smithi121 ceph-mon[32231]: pgmap v287: 97 pgs: 20 active+clean, 40 creating+peering, 37 unknown; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail; 0 B/s wr, 0 op/s 2024-04-25T00:30:50.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:50 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/870825602' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]}]': finished 2024-04-25T00:30:50.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:50 smithi121 ceph-mon[32231]: osdmap e85: 12 total, 12 up, 12 in 2024-04-25T00:30:50.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:50 smithi121 ceph-mon[32231]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-25T00:30:50.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:50 smithi121 ceph-mon[32231]: Cluster is now healthy 2024-04-25T00:30:50.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:50 smithi121 ceph-mon[32231]: mds.? [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] up:active 2024-04-25T00:30:50.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:50 smithi121 ceph-mon[32231]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-25T00:30:50.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:50 smithi121 ceph-mon[32231]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-25T00:30:50.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:50 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2913233743' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-04-25T00:30:50.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:50 smithi130 ceph-mon[32525]: pgmap v287: 97 pgs: 20 active+clean, 40 creating+peering, 37 unknown; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail; 0 B/s wr, 0 op/s 2024-04-25T00:30:50.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:50 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/870825602' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]}]': finished 2024-04-25T00:30:50.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:50 smithi130 ceph-mon[32525]: osdmap e85: 12 total, 12 up, 12 in 2024-04-25T00:30:50.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:50 smithi130 ceph-mon[32525]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-25T00:30:50.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:50 smithi130 ceph-mon[32525]: Cluster is now healthy 2024-04-25T00:30:50.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:50 smithi130 ceph-mon[32525]: mds.? [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] up:active 2024-04-25T00:30:50.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:50 smithi130 ceph-mon[32525]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-25T00:30:50.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:50 smithi130 ceph-mon[32525]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-25T00:30:50.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:50 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2913233743' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-04-25T00:30:50.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:50 smithi032 ceph-mon[29357]: pgmap v287: 97 pgs: 20 active+clean, 40 creating+peering, 37 unknown; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail; 0 B/s wr, 0 op/s 2024-04-25T00:30:50.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:50 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/870825602' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]}]': finished 2024-04-25T00:30:50.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:50 smithi032 ceph-mon[29357]: osdmap e85: 12 total, 12 up, 12 in 2024-04-25T00:30:50.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:50 smithi032 ceph-mon[29357]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-25T00:30:50.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:50 smithi032 ceph-mon[29357]: Cluster is now healthy 2024-04-25T00:30:50.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:50 smithi032 ceph-mon[29357]: mds.? [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] up:active 2024-04-25T00:30:50.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:50 smithi032 ceph-mon[29357]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-25T00:30:50.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:50 smithi032 ceph-mon[29357]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-25T00:30:50.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:50 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2913233743' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-04-25T00:30:51.754 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.754+0000 7f4a45ffb640 1 -- 172.21.15.32:0/2913233743 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data_ec' created v87) ==== 228+0+0 (secure 0 0 0) 0x7f4a4009c1f0 con 0x7f4a48102250 2024-04-25T00:30:51.755 INFO:teuthology.orchestra.run.smithi032.stderr:pool 'cephfs_data_ec' created 2024-04-25T00:30:51.758 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.757+0000 7f4a4eb29640 1 -- 172.21.15.32:0/2913233743 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4a380a2b90 msgr2=0x7f4a380a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:51.758 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.757+0000 7f4a4eb29640 1 --2- 172.21.15.32:0/2913233743 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4a380a2b90 0x7f4a380a5050 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f4a481016f0 tx=0x7f4a3c005e70 comp rx=0 tx=0).stop 2024-04-25T00:30:51.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.757+0000 7f4a4eb29640 1 -- 172.21.15.32:0/2913233743 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4a48102250 msgr2=0x7f4a4819ee50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:51.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.757+0000 7f4a4eb29640 1 --2- 172.21.15.32:0/2913233743 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4a48102250 0x7f4a4819ee50 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f4a4000eaa0 tx=0x7f4a4000ef60 comp rx=0 tx=0).stop 2024-04-25T00:30:51.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.757+0000 7f4a4eb29640 1 -- 172.21.15.32:0/2913233743 shutdown_connections 2024-04-25T00:30:51.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.757+0000 7f4a4eb29640 1 --2- 172.21.15.32:0/2913233743 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4a380a2b90 0x7f4a380a5050 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:51.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.757+0000 7f4a4eb29640 1 --2- 172.21.15.32:0/2913233743 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4a48102250 0x7f4a4819ee50 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:51.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.757+0000 7f4a4eb29640 1 --2- 172.21.15.32:0/2913233743 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4a48101890 0x7f4a4819c830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:51.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.757+0000 7f4a4eb29640 1 --2- 172.21.15.32:0/2913233743 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4a481006a0 0x7f4a4806a910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:51.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.757+0000 7f4a4eb29640 1 -- 172.21.15.32:0/2913233743 >> 172.21.15.32:0/2913233743 conn(0x7f4a480fbc30 msgr2=0x7f4a48104c00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:51.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.757+0000 7f4a4eb29640 1 -- 172.21.15.32:0/2913233743 shutdown_connections 2024-04-25T00:30:51.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.757+0000 7f4a4eb29640 1 -- 172.21.15.32:0/2913233743 wait complete. 2024-04-25T00:30:51.777 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool set cephfs_data_ec allow_ec_overwrites true 2024-04-25T00:30:51.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.888+0000 7f460c015640 1 -- 172.21.15.32:0/3206098447 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4604104b30 msgr2=0x7f4604106f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:51.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.888+0000 7f460c015640 1 --2- 172.21.15.32:0/3206098447 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4604104b30 0x7f4604106f20 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f45f4009900 tx=0x7f45f402a410 comp rx=0 tx=0).stop 2024-04-25T00:30:51.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.888+0000 7f460c015640 1 -- 172.21.15.32:0/3206098447 shutdown_connections 2024-04-25T00:30:51.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.888+0000 7f460c015640 1 --2- 172.21.15.32:0/3206098447 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4604107460 0x7f46041098b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:51.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.888+0000 7f460c015640 1 --2- 172.21.15.32:0/3206098447 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4604104b30 0x7f4604106f20 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:51.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.888+0000 7f460c015640 1 --2- 172.21.15.32:0/3206098447 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4604102200 0x7f46041045f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:51.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.888+0000 7f460c015640 1 -- 172.21.15.32:0/3206098447 >> 172.21.15.32:0/3206098447 conn(0x7f46040fbbf0 msgr2=0x7f46040fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:51.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.889+0000 7f460c015640 1 -- 172.21.15.32:0/3206098447 shutdown_connections 2024-04-25T00:30:51.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.889+0000 7f460c015640 1 -- 172.21.15.32:0/3206098447 wait complete. 2024-04-25T00:30:51.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.889+0000 7f460c015640 1 Processor -- start 2024-04-25T00:30:51.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.889+0000 7f460c015640 1 -- start start 2024-04-25T00:30:51.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.889+0000 7f460c015640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4604102200 0x7f4604101b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:51.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.889+0000 7f460c015640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4604104b30 0x7f46041001b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:51.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.889+0000 7f460c015640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4604107460 0x7f46041006f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:51.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.889+0000 7f460c015640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f460410d3f0 con 0x7f4604104b30 2024-04-25T00:30:51.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.889+0000 7f460c015640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f460410d270 con 0x7f4604102200 2024-04-25T00:30:51.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.889+0000 7f460c015640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f460410d570 con 0x7f4604107460 2024-04-25T00:30:51.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.889+0000 7f46095b0640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4604104b30 0x7f46041001b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:51.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.889+0000 7f46095b0640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4604104b30 0x7f46041001b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46598/0 (socket says 172.21.15.32:46598) 2024-04-25T00:30:51.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.889+0000 7f46095b0640 1 -- 172.21.15.32:0/2125124241 learned_addr learned my addr 172.21.15.32:0/2125124241 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:51.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.890+0000 7f460a5b2640 1 --2- 172.21.15.32:0/2125124241 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4604107460 0x7f46041006f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:51.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.890+0000 7f4609db1640 1 --2- 172.21.15.32:0/2125124241 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4604102200 0x7f4604101b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:51.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.890+0000 7f46095b0640 1 -- 172.21.15.32:0/2125124241 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4604107460 msgr2=0x7f46041006f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:51.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.890+0000 7f46095b0640 1 --2- 172.21.15.32:0/2125124241 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4604107460 0x7f46041006f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:51.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.890+0000 7f46095b0640 1 -- 172.21.15.32:0/2125124241 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4604102200 msgr2=0x7f4604101b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:51.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.890+0000 7f46095b0640 1 --2- 172.21.15.32:0/2125124241 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4604102200 0x7f4604101b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:51.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.890+0000 7f46095b0640 1 -- 172.21.15.32:0/2125124241 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4604100fb0 con 0x7f4604104b30 2024-04-25T00:30:51.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.890+0000 7f460a5b2640 1 --2- 172.21.15.32:0/2125124241 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4604107460 0x7f46041006f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:30:51.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.890+0000 7f4609db1640 1 --2- 172.21.15.32:0/2125124241 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4604102200 0x7f4604101b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:30:51.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.890+0000 7f46095b0640 1 --2- 172.21.15.32:0/2125124241 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4604104b30 0x7f46041001b0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f45f4031610 tx=0x7f45f40043b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:51.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.890+0000 7f45faffd640 1 -- 172.21.15.32:0/2125124241 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f45f403f030 con 0x7f4604104b30 2024-04-25T00:30:51.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.890+0000 7f45faffd640 1 -- 172.21.15.32:0/2125124241 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f45f4003ce0 con 0x7f4604104b30 2024-04-25T00:30:51.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.890+0000 7f460c015640 1 -- 172.21.15.32:0/2125124241 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f46041a1c30 con 0x7f4604104b30 2024-04-25T00:30:51.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.890+0000 7f45faffd640 1 -- 172.21.15.32:0/2125124241 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f45f4035070 con 0x7f4604104b30 2024-04-25T00:30:51.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.890+0000 7f460c015640 1 -- 172.21.15.32:0/2125124241 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f46041a2140 con 0x7f4604104b30 2024-04-25T00:30:51.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.891+0000 7f460c015640 1 -- 172.21.15.32:0/2125124241 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f46040fd6e0 con 0x7f4604104b30 2024-04-25T00:30:51.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.893+0000 7f45faffd640 1 -- 172.21.15.32:0/2125124241 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f45f4042030 con 0x7f4604104b30 2024-04-25T00:30:51.896 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.893+0000 7f45faffd640 1 --2- 172.21.15.32:0/2125124241 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f45f00a2c10 0x7f45f00a50d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:51.896 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.893+0000 7f4609db1640 1 --2- 172.21.15.32:0/2125124241 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f45f00a2c10 0x7f45f00a50d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:51.896 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.894+0000 7f45faffd640 1 -- 172.21.15.32:0/2125124241 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 8770+0+0 (secure 0 0 0) 0x7f45f40efa20 con 0x7f4604104b30 2024-04-25T00:30:51.896 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.894+0000 7f4609db1640 1 --2- 172.21.15.32:0/2125124241 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f45f00a2c10 0x7f45f00a50d0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f4600009b40 tx=0x7f4600005e50 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:51.897 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:51.897+0000 7f45faffd640 1 -- 172.21.15.32:0/2125124241 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f45f40bc760 con 0x7f4604104b30 2024-04-25T00:30:52.081 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.080+0000 7f460c015640 1 -- 172.21.15.32:0/2125124241 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool set", "pool": "cephfs_data_ec", "var": "allow_ec_overwrites", "val": "true"} v 0) -- 0x7f46040623e0 con 0x7f4604104b30 2024-04-25T00:30:52.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:51 smithi121 ceph-mon[32231]: osdmap e86: 12 total, 12 up, 12 in 2024-04-25T00:30:52.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:51 smithi130 ceph-mon[32525]: osdmap e86: 12 total, 12 up, 12 in 2024-04-25T00:30:52.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:51 smithi032 ceph-mon[29357]: osdmap e86: 12 total, 12 up, 12 in 2024-04-25T00:30:52.756 INFO:journalctl@ceph.mds.b.smithi121.stdout:Apr 25 00:30:52 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-b[62582]: 2024-04-25T00:30:52.450+0000 7f6e5c388700 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-04-25T00:30:52.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.762+0000 7f45faffd640 1 -- 172.21.15.32:0/2125124241 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool set", "pool": "cephfs_data_ec", "var": "allow_ec_overwrites", "val": "true"}]=0 set pool 4 allow_ec_overwrites to true v88) ==== 169+0+0 (secure 0 0 0) 0x7f45f40bc060 con 0x7f4604104b30 2024-04-25T00:30:52.762 INFO:teuthology.orchestra.run.smithi032.stderr:set pool 4 allow_ec_overwrites to true 2024-04-25T00:30:52.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.765+0000 7f460c015640 1 -- 172.21.15.32:0/2125124241 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f45f00a2c10 msgr2=0x7f45f00a50d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:52.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.765+0000 7f460c015640 1 --2- 172.21.15.32:0/2125124241 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f45f00a2c10 0x7f45f00a50d0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f4600009b40 tx=0x7f4600005e50 comp rx=0 tx=0).stop 2024-04-25T00:30:52.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.765+0000 7f460c015640 1 -- 172.21.15.32:0/2125124241 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4604104b30 msgr2=0x7f46041001b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:52.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.765+0000 7f460c015640 1 --2- 172.21.15.32:0/2125124241 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4604104b30 0x7f46041001b0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f45f4031610 tx=0x7f45f40043b0 comp rx=0 tx=0).stop 2024-04-25T00:30:52.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.766+0000 7f460c015640 1 -- 172.21.15.32:0/2125124241 shutdown_connections 2024-04-25T00:30:52.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.766+0000 7f460c015640 1 --2- 172.21.15.32:0/2125124241 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f45f00a2c10 0x7f45f00a50d0 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:52.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.766+0000 7f460c015640 1 --2- 172.21.15.32:0/2125124241 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4604107460 0x7f46041006f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:52.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.766+0000 7f460c015640 1 --2- 172.21.15.32:0/2125124241 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4604104b30 0x7f46041001b0 unknown :-1 s=CLOSED pgs=208 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:52.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.766+0000 7f460c015640 1 --2- 172.21.15.32:0/2125124241 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4604102200 0x7f4604101b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:52.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.766+0000 7f460c015640 1 -- 172.21.15.32:0/2125124241 >> 172.21.15.32:0/2125124241 conn(0x7f46040fbbf0 msgr2=0x7f4604102b40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:52.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.766+0000 7f460c015640 1 -- 172.21.15.32:0/2125124241 shutdown_connections 2024-04-25T00:30:52.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.767+0000 7f460c015640 1 -- 172.21.15.32:0/2125124241 wait complete. 2024-04-25T00:30:52.789 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs add_data_pool cephfs cephfs_data_ec 2024-04-25T00:30:52.903 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.900+0000 7f7e752b3640 1 -- 172.21.15.32:0/2297831184 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7e70100680 msgr2=0x7f7e70100aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:52.904 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.900+0000 7f7e752b3640 1 --2- 172.21.15.32:0/2297831184 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7e70100680 0x7f7e70100aa0 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f7e60009900 tx=0x7f7e6002a410 comp rx=0 tx=0).stop 2024-04-25T00:30:52.904 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.900+0000 7f7e752b3640 1 -- 172.21.15.32:0/2297831184 shutdown_connections 2024-04-25T00:30:52.904 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.900+0000 7f7e752b3640 1 --2- 172.21.15.32:0/2297831184 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7e70102230 0x7f7e70108ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:52.904 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.900+0000 7f7e752b3640 1 --2- 172.21.15.32:0/2297831184 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7e70101870 0x7f7e70101cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:52.904 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.900+0000 7f7e752b3640 1 --2- 172.21.15.32:0/2297831184 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7e70100680 0x7f7e70100aa0 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:52.904 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.900+0000 7f7e752b3640 1 -- 172.21.15.32:0/2297831184 >> 172.21.15.32:0/2297831184 conn(0x7f7e700fbbf0 msgr2=0x7f7e700fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:52.904 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.901+0000 7f7e752b3640 1 -- 172.21.15.32:0/2297831184 shutdown_connections 2024-04-25T00:30:52.904 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.901+0000 7f7e752b3640 1 -- 172.21.15.32:0/2297831184 wait complete. 2024-04-25T00:30:52.904 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.901+0000 7f7e752b3640 1 Processor -- start 2024-04-25T00:30:52.904 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.901+0000 7f7e752b3640 1 -- start start 2024-04-25T00:30:52.904 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.901+0000 7f7e752b3640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7e70100680 0x7f7e701045a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:52.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.901+0000 7f7e752b3640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7e70101870 0x7f7e70102c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:52.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.901+0000 7f7e752b3640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7e70102230 0x7f7e70103190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:52.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.901+0000 7f7e752b3640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f7e7010c2d0 con 0x7f7e70102230 2024-04-25T00:30:52.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.902+0000 7f7e752b3640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f7e7010c150 con 0x7f7e70101870 2024-04-25T00:30:52.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.902+0000 7f7e752b3640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f7e7010c450 con 0x7f7e70100680 2024-04-25T00:30:52.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.902+0000 7f7e6f7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7e70102230 0x7f7e70103190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:52.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.902+0000 7f7e6f7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7e70102230 0x7f7e70103190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46618/0 (socket says 172.21.15.32:46618) 2024-04-25T00:30:52.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.902+0000 7f7e6f7fe640 1 -- 172.21.15.32:0/1293821860 learned_addr learned my addr 172.21.15.32:0/1293821860 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:52.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.902+0000 7f7e6effd640 1 --2- 172.21.15.32:0/1293821860 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7e70100680 0x7f7e701045a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:52.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.902+0000 7f7e6e7fc640 1 --2- 172.21.15.32:0/1293821860 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7e70101870 0x7f7e70102c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:52.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.902+0000 7f7e6f7fe640 1 -- 172.21.15.32:0/1293821860 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7e70100680 msgr2=0x7f7e701045a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:52.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.902+0000 7f7e6f7fe640 1 --2- 172.21.15.32:0/1293821860 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7e70100680 0x7f7e701045a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:52.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.902+0000 7f7e6f7fe640 1 -- 172.21.15.32:0/1293821860 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7e70101870 msgr2=0x7f7e70102c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:52.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.902+0000 7f7e6f7fe640 1 --2- 172.21.15.32:0/1293821860 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7e70101870 0x7f7e70102c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:52.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.902+0000 7f7e6f7fe640 1 -- 172.21.15.32:0/1293821860 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7e701039b0 con 0x7f7e70102230 2024-04-25T00:30:52.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.902+0000 7f7e6f7fe640 1 --2- 172.21.15.32:0/1293821860 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7e70102230 0x7f7e70103190 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f7e6800eaa0 tx=0x7f7e6800eea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:52.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.902+0000 7f7e57fff640 1 -- 172.21.15.32:0/1293821860 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7e68004d20 con 0x7f7e70102230 2024-04-25T00:30:52.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.902+0000 7f7e57fff640 1 -- 172.21.15.32:0/1293821860 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7e68005020 con 0x7f7e70102230 2024-04-25T00:30:52.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.902+0000 7f7e752b3640 1 -- 172.21.15.32:0/1293821860 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7e701a1be0 con 0x7f7e70102230 2024-04-25T00:30:52.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.902+0000 7f7e57fff640 1 -- 172.21.15.32:0/1293821860 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7e680116d0 con 0x7f7e70102230 2024-04-25T00:30:52.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.902+0000 7f7e752b3640 1 -- 172.21.15.32:0/1293821860 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7e701a20e0 con 0x7f7e70102230 2024-04-25T00:30:52.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.903+0000 7f7e752b3640 1 -- 172.21.15.32:0/1293821860 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7e701a2310 con 0x7f7e70102230 2024-04-25T00:30:52.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.905+0000 7f7e57fff640 1 -- 172.21.15.32:0/1293821860 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f7e680051d0 con 0x7f7e70102230 2024-04-25T00:30:52.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.905+0000 7f7e57fff640 1 --2- 172.21.15.32:0/1293821860 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f7e580a2b90 0x7f7e580a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:52.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.905+0000 7f7e57fff640 1 -- 172.21.15.32:0/1293821860 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(88..88 src has 1..88) ==== 8770+0+0 (secure 0 0 0) 0x7f7e680cf7b0 con 0x7f7e70102230 2024-04-25T00:30:52.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.905+0000 7f7e6effd640 1 --2- 172.21.15.32:0/1293821860 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f7e580a2b90 0x7f7e580a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:52.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.906+0000 7f7e6effd640 1 --2- 172.21.15.32:0/1293821860 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f7e580a2b90 0x7f7e580a5050 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f7e60005ab0 tx=0x7f7e60000bc0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:52.909 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:52.909+0000 7f7e57fff640 1 -- 172.21.15.32:0/1293821860 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f7e68016030 con 0x7f7e70102230 2024-04-25T00:30:53.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:52 smithi130 ceph-mon[32525]: pgmap v290: 97 pgs: 38 active+clean, 40 creating+peering, 19 unknown; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail; 255 B/s wr, 5 op/s 2024-04-25T00:30:53.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:52 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2913233743' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-04-25T00:30:53.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:52 smithi130 ceph-mon[32525]: osdmap e87: 12 total, 12 up, 12 in 2024-04-25T00:30:53.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:52 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2125124241' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "cephfs_data_ec", "var": "allow_ec_overwrites", "val": "true"}]: dispatch 2024-04-25T00:30:53.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:52 smithi121 ceph-mon[32231]: pgmap v290: 97 pgs: 38 active+clean, 40 creating+peering, 19 unknown; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail; 255 B/s wr, 5 op/s 2024-04-25T00:30:53.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:52 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2913233743' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-04-25T00:30:53.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:52 smithi121 ceph-mon[32231]: osdmap e87: 12 total, 12 up, 12 in 2024-04-25T00:30:53.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:52 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2125124241' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "cephfs_data_ec", "var": "allow_ec_overwrites", "val": "true"}]: dispatch 2024-04-25T00:30:53.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.133+0000 7f7e752b3640 1 -- 172.21.15.32:0/1293821860 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs add_data_pool", "fs_name": "cephfs", "pool": "cephfs_data_ec"} v 0) -- 0x7f7e701a24e0 con 0x7f7e70102230 2024-04-25T00:30:53.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:52 smithi032 ceph-mon[29357]: pgmap v290: 97 pgs: 38 active+clean, 40 creating+peering, 19 unknown; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail; 255 B/s wr, 5 op/s 2024-04-25T00:30:53.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:52 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2913233743' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-04-25T00:30:53.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:52 smithi032 ceph-mon[29357]: osdmap e87: 12 total, 12 up, 12 in 2024-04-25T00:30:53.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:52 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2125124241' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "cephfs_data_ec", "var": "allow_ec_overwrites", "val": "true"}]: dispatch 2024-04-25T00:30:53.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.771+0000 7f7e57fff640 1 -- 172.21.15.32:0/1293821860 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs add_data_pool", "fs_name": "cephfs", "pool": "cephfs_data_ec"}]=0 added data pool 4 to fsmap v17) ==== 137+0+0 (secure 0 0 0) 0x7f7e6809c4f0 con 0x7f7e70102230 2024-04-25T00:30:53.772 INFO:teuthology.orchestra.run.smithi032.stderr:added data pool 4 to fsmap 2024-04-25T00:30:53.774 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.774+0000 7f7e752b3640 1 -- 172.21.15.32:0/1293821860 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f7e580a2b90 msgr2=0x7f7e580a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:53.775 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.774+0000 7f7e752b3640 1 --2- 172.21.15.32:0/1293821860 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f7e580a2b90 0x7f7e580a5050 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f7e60005ab0 tx=0x7f7e60000bc0 comp rx=0 tx=0).stop 2024-04-25T00:30:53.775 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.774+0000 7f7e752b3640 1 -- 172.21.15.32:0/1293821860 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7e70102230 msgr2=0x7f7e70103190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:53.775 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.774+0000 7f7e752b3640 1 --2- 172.21.15.32:0/1293821860 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7e70102230 0x7f7e70103190 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f7e6800eaa0 tx=0x7f7e6800eea0 comp rx=0 tx=0).stop 2024-04-25T00:30:53.775 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.774+0000 7f7e752b3640 1 -- 172.21.15.32:0/1293821860 shutdown_connections 2024-04-25T00:30:53.775 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.774+0000 7f7e752b3640 1 --2- 172.21.15.32:0/1293821860 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f7e580a2b90 0x7f7e580a5050 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:53.775 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.774+0000 7f7e752b3640 1 --2- 172.21.15.32:0/1293821860 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7e70102230 0x7f7e70103190 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:53.775 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.774+0000 7f7e752b3640 1 --2- 172.21.15.32:0/1293821860 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7e70101870 0x7f7e70102c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:53.775 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.774+0000 7f7e752b3640 1 --2- 172.21.15.32:0/1293821860 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7e70100680 0x7f7e701045a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:53.775 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.774+0000 7f7e752b3640 1 -- 172.21.15.32:0/1293821860 >> 172.21.15.32:0/1293821860 conn(0x7f7e700fbbf0 msgr2=0x7f7e700fd6c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:53.775 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.774+0000 7f7e752b3640 1 -- 172.21.15.32:0/1293821860 shutdown_connections 2024-04-25T00:30:53.775 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.774+0000 7f7e752b3640 1 -- 172.21.15.32:0/1293821860 wait complete. 2024-04-25T00:30:53.789 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:30:53.899 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.896+0000 7f547e8ba640 1 -- 172.21.15.32:0/580370613 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5478102250 msgr2=0x7f5478108ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:53.899 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.896+0000 7f547e8ba640 1 --2- 172.21.15.32:0/580370613 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5478102250 0x7f5478108ae0 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7f5470009ca0 tx=0x7f547002a8c0 comp rx=0 tx=0).stop 2024-04-25T00:30:53.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.897+0000 7f547e8ba640 1 -- 172.21.15.32:0/580370613 shutdown_connections 2024-04-25T00:30:53.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.897+0000 7f547e8ba640 1 --2- 172.21.15.32:0/580370613 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5478102250 0x7f5478108ae0 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:53.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.897+0000 7f547e8ba640 1 --2- 172.21.15.32:0/580370613 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5478101890 0x7f5478101d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:53.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.897+0000 7f547e8ba640 1 --2- 172.21.15.32:0/580370613 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f54781006a0 0x7f5478100ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:53.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.897+0000 7f547e8ba640 1 -- 172.21.15.32:0/580370613 >> 172.21.15.32:0/580370613 conn(0x7f54780fbc30 msgr2=0x7f54780fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:53.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.897+0000 7f547e8ba640 1 -- 172.21.15.32:0/580370613 shutdown_connections 2024-04-25T00:30:53.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.897+0000 7f547e8ba640 1 -- 172.21.15.32:0/580370613 wait complete. 2024-04-25T00:30:53.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.897+0000 7f547e8ba640 1 Processor -- start 2024-04-25T00:30:53.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.897+0000 7f547e8ba640 1 -- start start 2024-04-25T00:30:53.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.897+0000 7f547e8ba640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f54781006a0 0x7f5478197ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:53.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.897+0000 7f547e8ba640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5478101890 0x7f5478198000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:53.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.897+0000 7f547e8ba640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5478102250 0x7f547819f070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:53.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.898+0000 7f547e8ba640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f547810d4a0 con 0x7f54781006a0 2024-04-25T00:30:53.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.898+0000 7f547e8ba640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f547810d320 con 0x7f5478101890 2024-04-25T00:30:53.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.898+0000 7f547e8ba640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f547810d620 con 0x7f5478102250 2024-04-25T00:30:53.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.898+0000 7f5477fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f54781006a0 0x7f5478197ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:53.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.898+0000 7f5477fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f54781006a0 0x7f5478197ac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46636/0 (socket says 172.21.15.32:46636) 2024-04-25T00:30:53.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.898+0000 7f5477fff640 1 -- 172.21.15.32:0/3734810551 learned_addr learned my addr 172.21.15.32:0/3734810551 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:53.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.898+0000 7f547ce57640 1 --2- 172.21.15.32:0/3734810551 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5478102250 0x7f547819f070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:53.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.898+0000 7f54777fe640 1 --2- 172.21.15.32:0/3734810551 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5478101890 0x7f5478198000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:53.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.898+0000 7f5477fff640 1 -- 172.21.15.32:0/3734810551 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5478102250 msgr2=0x7f547819f070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:53.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.898+0000 7f5477fff640 1 --2- 172.21.15.32:0/3734810551 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5478102250 0x7f547819f070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:53.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.898+0000 7f5477fff640 1 -- 172.21.15.32:0/3734810551 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5478101890 msgr2=0x7f5478198000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:53.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.898+0000 7f5477fff640 1 --2- 172.21.15.32:0/3734810551 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5478101890 0x7f5478198000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:53.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.898+0000 7f5477fff640 1 -- 172.21.15.32:0/3734810551 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f547819f6e0 con 0x7f54781006a0 2024-04-25T00:30:53.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.898+0000 7f547ce57640 1 --2- 172.21.15.32:0/3734810551 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5478102250 0x7f547819f070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:30:53.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.898+0000 7f5477fff640 1 --2- 172.21.15.32:0/3734810551 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f54781006a0 0x7f5478197ac0 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7f546000e9c0 tx=0x7f546000ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:53.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.898+0000 7f54757fa640 1 -- 172.21.15.32:0/3734810551 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f546000cd80 con 0x7f54781006a0 2024-04-25T00:30:53.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.898+0000 7f54757fa640 1 -- 172.21.15.32:0/3734810551 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f54600035f0 con 0x7f54781006a0 2024-04-25T00:30:53.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.898+0000 7f547e8ba640 1 -- 172.21.15.32:0/3734810551 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f547819f9d0 con 0x7f54781006a0 2024-04-25T00:30:53.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.898+0000 7f54757fa640 1 -- 172.21.15.32:0/3734810551 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5460010690 con 0x7f54781006a0 2024-04-25T00:30:53.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.898+0000 7f547e8ba640 1 -- 172.21.15.32:0/3734810551 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f547819fed0 con 0x7f54781006a0 2024-04-25T00:30:53.903 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.900+0000 7f547e8ba640 1 -- 172.21.15.32:0/3734810551 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5478107960 con 0x7f54781006a0 2024-04-25T00:30:53.903 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.901+0000 7f54757fa640 1 -- 172.21.15.32:0/3734810551 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f54600037a0 con 0x7f54781006a0 2024-04-25T00:30:53.903 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.901+0000 7f54757fa640 1 --2- 172.21.15.32:0/3734810551 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f54640a2b90 0x7f54640a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:53.903 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.902+0000 7f54777fe640 1 --2- 172.21.15.32:0/3734810551 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f54640a2b90 0x7f54640a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:53.903 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.902+0000 7f54757fa640 1 -- 172.21.15.32:0/3734810551 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f54600cf680 con 0x7f54781006a0 2024-04-25T00:30:53.903 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.902+0000 7f54777fe640 1 --2- 172.21.15.32:0/3734810551 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f54640a2b90 0x7f54640a5050 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f5478198fd0 tx=0x7f546800b590 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:53.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:53.906+0000 7f54757fa640 1 -- 172.21.15.32:0/3734810551 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f546009c3a0 con 0x7f54781006a0 2024-04-25T00:30:54.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:53 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2125124241' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "cephfs_data_ec", "var": "allow_ec_overwrites", "val": "true"}]': finished 2024-04-25T00:30:54.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:53 smithi121 ceph-mon[32231]: osdmap e88: 12 total, 12 up, 12 in 2024-04-25T00:30:54.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:53 smithi121 ceph-mon[32231]: pgmap v293: 129 pgs: 82 active+clean, 33 creating+peering, 14 unknown; 579 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.5 KiB/s wr, 10 op/s 2024-04-25T00:30:54.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:53 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1293821860' entity='client.admin' cmd=[{"prefix": "fs add_data_pool", "fs_name": "cephfs", "pool": "cephfs_data_ec"}]: dispatch 2024-04-25T00:30:54.127 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.127+0000 7f547e8ba640 1 -- 172.21.15.32:0/3734810551 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f547802d170 con 0x7f54781006a0 2024-04-25T00:30:54.128 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.128+0000 7f54757fa640 1 -- 172.21.15.32:0/3734810551 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 17 v17) ==== 94+0+11803 (secure 0 0 0) 0x7f546009bca0 con 0x7f54781006a0 2024-04-25T00:30:54.128 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:54.129 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":17,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14643,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6839/187492674","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6838","nonce":187492674},{"type":"v1","addr":"172.21.15.32:6839","nonce":187492674}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14655,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6841/1948785545","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6840","nonce":1948785545},{"type":"v1","addr":"172.21.15.32:6841","nonce":1948785545}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24443,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6},{"gid":24445,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":7},{"gid":24457,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":8},{"gid":24476,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":9},{"gid":24481,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6839/4267125809","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":4267125809},{"type":"v1","addr":"172.21.15.130:6839","nonce":4267125809}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":10}],"filesystems":[{"mdsmap":{"epoch":17,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-04-25T00:30:47.866875+0000","modified":"2024-04-25T00:30:53.134841+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":1,"in":[0],"up":{"mds_0":24506},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24506":{"gid":24506,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.121:6839/258444853","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6838","nonce":258444853},{"type":"v1","addr":"172.21.15.121:6839","nonce":258444853}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":1,"qdb_leader":24506,"qdb_cluster":[24506]},"id":1}]} 2024-04-25T00:30:54.129 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 17 2024-04-25T00:30:54.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.131+0000 7f547e8ba640 1 -- 172.21.15.32:0/3734810551 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f54640a2b90 msgr2=0x7f54640a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:54.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.131+0000 7f547e8ba640 1 --2- 172.21.15.32:0/3734810551 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f54640a2b90 0x7f54640a5050 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f5478198fd0 tx=0x7f546800b590 comp rx=0 tx=0).stop 2024-04-25T00:30:54.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.131+0000 7f547e8ba640 1 -- 172.21.15.32:0/3734810551 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f54781006a0 msgr2=0x7f5478197ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:54.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.131+0000 7f547e8ba640 1 --2- 172.21.15.32:0/3734810551 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f54781006a0 0x7f5478197ac0 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7f546000e9c0 tx=0x7f546000ee80 comp rx=0 tx=0).stop 2024-04-25T00:30:54.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.131+0000 7f547e8ba640 1 -- 172.21.15.32:0/3734810551 shutdown_connections 2024-04-25T00:30:54.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.131+0000 7f547e8ba640 1 --2- 172.21.15.32:0/3734810551 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f54640a2b90 0x7f54640a5050 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.131+0000 7f547e8ba640 1 --2- 172.21.15.32:0/3734810551 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5478102250 0x7f547819f070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.131+0000 7f547e8ba640 1 --2- 172.21.15.32:0/3734810551 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5478101890 0x7f5478198000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.131+0000 7f547e8ba640 1 --2- 172.21.15.32:0/3734810551 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f54781006a0 0x7f5478197ac0 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.131+0000 7f547e8ba640 1 -- 172.21.15.32:0/3734810551 >> 172.21.15.32:0/3734810551 conn(0x7f54780fbc30 msgr2=0x7f5478104b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:54.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.131+0000 7f547e8ba640 1 -- 172.21.15.32:0/3734810551 shutdown_connections 2024-04-25T00:30:54.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.131+0000 7f547e8ba640 1 -- 172.21.15.32:0/3734810551 wait complete. 2024-04-25T00:30:54.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:53 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2125124241' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "cephfs_data_ec", "var": "allow_ec_overwrites", "val": "true"}]': finished 2024-04-25T00:30:54.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:53 smithi130 ceph-mon[32525]: osdmap e88: 12 total, 12 up, 12 in 2024-04-25T00:30:54.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:53 smithi130 ceph-mon[32525]: pgmap v293: 129 pgs: 82 active+clean, 33 creating+peering, 14 unknown; 579 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.5 KiB/s wr, 10 op/s 2024-04-25T00:30:54.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:53 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1293821860' entity='client.admin' cmd=[{"prefix": "fs add_data_pool", "fs_name": "cephfs", "pool": "cephfs_data_ec"}]: dispatch 2024-04-25T00:30:54.146 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:53 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2125124241' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "cephfs_data_ec", "var": "allow_ec_overwrites", "val": "true"}]': finished 2024-04-25T00:30:54.146 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:53 smithi032 ceph-mon[29357]: osdmap e88: 12 total, 12 up, 12 in 2024-04-25T00:30:54.146 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:53 smithi032 ceph-mon[29357]: pgmap v293: 129 pgs: 82 active+clean, 33 creating+peering, 14 unknown; 579 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.5 KiB/s wr, 10 op/s 2024-04-25T00:30:54.146 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:53 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1293821860' entity='client.admin' cmd=[{"prefix": "fs add_data_pool", "fs_name": "cephfs", "pool": "cephfs_data_ec"}]: dispatch 2024-04-25T00:30:54.147 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-25T00:30:54.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.248+0000 7f09cb5d5640 1 -- 172.21.15.32:0/3042710993 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f09c4100680 msgr2=0x7f09c4100aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:54.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.248+0000 7f09cb5d5640 1 --2- 172.21.15.32:0/3042710993 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f09c4100680 0x7f09c4100aa0 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f09b4009900 tx=0x7f09b402a410 comp rx=0 tx=0).stop 2024-04-25T00:30:54.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.248+0000 7f09cb5d5640 1 -- 172.21.15.32:0/3042710993 shutdown_connections 2024-04-25T00:30:54.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.248+0000 7f09cb5d5640 1 --2- 172.21.15.32:0/3042710993 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f09c4102230 0x7f09c4108ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.249+0000 7f09cb5d5640 1 --2- 172.21.15.32:0/3042710993 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f09c4101870 0x7f09c4101cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.249+0000 7f09cb5d5640 1 --2- 172.21.15.32:0/3042710993 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f09c4100680 0x7f09c4100aa0 unknown :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.249+0000 7f09cb5d5640 1 -- 172.21.15.32:0/3042710993 >> 172.21.15.32:0/3042710993 conn(0x7f09c40fbbf0 msgr2=0x7f09c40fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:54.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.249+0000 7f09cb5d5640 1 -- 172.21.15.32:0/3042710993 shutdown_connections 2024-04-25T00:30:54.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.249+0000 7f09cb5d5640 1 -- 172.21.15.32:0/3042710993 wait complete. 2024-04-25T00:30:54.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.249+0000 7f09cb5d5640 1 Processor -- start 2024-04-25T00:30:54.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.249+0000 7f09cb5d5640 1 -- start start 2024-04-25T00:30:54.253 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.249+0000 7f09cb5d5640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f09c4100680 0x7f09c4104610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:54.253 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.249+0000 7f09cb5d5640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f09c4101870 0x7f09c4102c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:54.253 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.249+0000 7f09cb5d5640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f09c4102230 0x7f09c4103190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:54.253 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.249+0000 7f09cb5d5640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f09c410c3a0 con 0x7f09c4101870 2024-04-25T00:30:54.253 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.249+0000 7f09cb5d5640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f09c410c220 con 0x7f09c4102230 2024-04-25T00:30:54.253 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.249+0000 7f09cb5d5640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f09c410c520 con 0x7f09c4100680 2024-04-25T00:30:54.253 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.250+0000 7f09c8b70640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f09c4101870 0x7f09c4102c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:54.253 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.250+0000 7f09c8b70640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f09c4101870 0x7f09c4102c50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46666/0 (socket says 172.21.15.32:46666) 2024-04-25T00:30:54.253 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.250+0000 7f09c8b70640 1 -- 172.21.15.32:0/265876846 learned_addr learned my addr 172.21.15.32:0/265876846 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:54.253 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.250+0000 7f09c9371640 1 --2- 172.21.15.32:0/265876846 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f09c4100680 0x7f09c4104610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:54.253 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.250+0000 7f09c9b72640 1 --2- 172.21.15.32:0/265876846 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f09c4102230 0x7f09c4103190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:54.253 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.250+0000 7f09c8b70640 1 -- 172.21.15.32:0/265876846 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f09c4100680 msgr2=0x7f09c4104610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:54.253 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.250+0000 7f09c8b70640 1 --2- 172.21.15.32:0/265876846 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f09c4100680 0x7f09c4104610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.253 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.250+0000 7f09c8b70640 1 -- 172.21.15.32:0/265876846 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f09c4102230 msgr2=0x7f09c4103190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:54.254 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.250+0000 7f09c8b70640 1 --2- 172.21.15.32:0/265876846 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f09c4102230 0x7f09c4103190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.254 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.250+0000 7f09c8b70640 1 -- 172.21.15.32:0/265876846 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f09c4103a20 con 0x7f09c4101870 2024-04-25T00:30:54.254 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.250+0000 7f09c9b72640 1 --2- 172.21.15.32:0/265876846 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f09c4102230 0x7f09c4103190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:30:54.254 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.250+0000 7f09c8b70640 1 --2- 172.21.15.32:0/265876846 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f09c4101870 0x7f09c4102c50 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7f09c000d950 tx=0x7f09c000de10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:54.254 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.250+0000 7f09ba7fc640 1 -- 172.21.15.32:0/265876846 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f09c0017070 con 0x7f09c4101870 2024-04-25T00:30:54.254 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.250+0000 7f09cb5d5640 1 -- 172.21.15.32:0/265876846 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f09c41a1c50 con 0x7f09c4101870 2024-04-25T00:30:54.254 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.250+0000 7f09ba7fc640 1 -- 172.21.15.32:0/265876846 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f09c0009c30 con 0x7f09c4101870 2024-04-25T00:30:54.254 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.250+0000 7f09ba7fc640 1 -- 172.21.15.32:0/265876846 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f09c000f6b0 con 0x7f09c4101870 2024-04-25T00:30:54.254 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.250+0000 7f09cb5d5640 1 -- 172.21.15.32:0/265876846 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f09c41a21a0 con 0x7f09c4101870 2024-04-25T00:30:54.254 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.251+0000 7f09cb5d5640 1 -- 172.21.15.32:0/265876846 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f09c41a23d0 con 0x7f09c4101870 2024-04-25T00:30:54.255 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.253+0000 7f09ba7fc640 1 -- 172.21.15.32:0/265876846 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f09c00109d0 con 0x7f09c4101870 2024-04-25T00:30:54.255 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.253+0000 7f09ba7fc640 1 --2- 172.21.15.32:0/265876846 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f09b00a2b90 0x7f09b00a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:54.255 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.253+0000 7f09c9371640 1 --2- 172.21.15.32:0/265876846 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f09b00a2b90 0x7f09b00a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:54.255 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.254+0000 7f09ba7fc640 1 -- 172.21.15.32:0/265876846 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f09c00ceea0 con 0x7f09c4101870 2024-04-25T00:30:54.255 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.254+0000 7f09c9371640 1 --2- 172.21.15.32:0/265876846 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f09b00a2b90 0x7f09b00a5050 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f09b402aa00 tx=0x7f09b40315b0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:54.257 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.257+0000 7f09ba7fc640 1 -- 172.21.15.32:0/265876846 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f09c0009de0 con 0x7f09c4101870 2024-04-25T00:30:54.444 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.443+0000 7f09cb5d5640 1 -- 172.21.15.32:0/265876846 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f09c41a26c0 con 0x7f09c4101870 2024-04-25T00:30:54.445 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.445+0000 7f09ba7fc640 1 -- 172.21.15.32:0/265876846 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v89) ==== 74+0+24128 (secure 0 0 0) 0x7f09c009bbc0 con 0x7f09c4101870 2024-04-25T00:30:54.446 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:54.446 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":89,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:30:53.134605+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-25T00:30:46.239556+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-25T00:30:47.257182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2024-04-25T00:30:51.193975+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":83,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-num-failure-domains":"0","crush-osds-per-failure-domain":"0","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:30:54.448 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.447+0000 7f09cb5d5640 1 -- 172.21.15.32:0/265876846 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f09b00a2b90 msgr2=0x7f09b00a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:54.448 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.447+0000 7f09cb5d5640 1 --2- 172.21.15.32:0/265876846 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f09b00a2b90 0x7f09b00a5050 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f09b402aa00 tx=0x7f09b40315b0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.448 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.447+0000 7f09cb5d5640 1 -- 172.21.15.32:0/265876846 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f09c4101870 msgr2=0x7f09c4102c50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:54.448 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.447+0000 7f09cb5d5640 1 --2- 172.21.15.32:0/265876846 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f09c4101870 0x7f09c4102c50 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7f09c000d950 tx=0x7f09c000de10 comp rx=0 tx=0).stop 2024-04-25T00:30:54.448 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.447+0000 7f09cb5d5640 1 -- 172.21.15.32:0/265876846 shutdown_connections 2024-04-25T00:30:54.448 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.447+0000 7f09cb5d5640 1 --2- 172.21.15.32:0/265876846 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f09b00a2b90 0x7f09b00a5050 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.448 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.447+0000 7f09cb5d5640 1 --2- 172.21.15.32:0/265876846 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f09c4102230 0x7f09c4103190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.447+0000 7f09cb5d5640 1 --2- 172.21.15.32:0/265876846 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f09c4101870 0x7f09c4102c50 unknown :-1 s=CLOSED pgs=214 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.447+0000 7f09cb5d5640 1 --2- 172.21.15.32:0/265876846 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f09c4100680 0x7f09c4104610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.447+0000 7f09cb5d5640 1 -- 172.21.15.32:0/265876846 >> 172.21.15.32:0/265876846 conn(0x7f09c40fbbf0 msgr2=0x7f09c40fd670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:54.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.447+0000 7f09cb5d5640 1 -- 172.21.15.32:0/265876846 shutdown_connections 2024-04-25T00:30:54.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.447+0000 7f09cb5d5640 1 -- 172.21.15.32:0/265876846 wait complete. 2024-04-25T00:30:54.459 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-25T00:30:54.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.560+0000 7f8ebf4a5640 1 -- 172.21.15.32:0/461120590 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8eb81006a0 msgr2=0x7f8eb8100ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:54.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.560+0000 7f8ebf4a5640 1 --2- 172.21.15.32:0/461120590 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8eb81006a0 0x7f8eb8100ac0 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f8eb4009900 tx=0x7f8eb402a410 comp rx=0 tx=0).stop 2024-04-25T00:30:54.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.561+0000 7f8ebf4a5640 1 -- 172.21.15.32:0/461120590 shutdown_connections 2024-04-25T00:30:54.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.561+0000 7f8ebf4a5640 1 --2- 172.21.15.32:0/461120590 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8eb8102250 0x7f8eb8108ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.561+0000 7f8ebf4a5640 1 --2- 172.21.15.32:0/461120590 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8eb8101890 0x7f8eb8101d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.561+0000 7f8ebf4a5640 1 --2- 172.21.15.32:0/461120590 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8eb81006a0 0x7f8eb8100ac0 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.561+0000 7f8ebf4a5640 1 -- 172.21.15.32:0/461120590 >> 172.21.15.32:0/461120590 conn(0x7f8eb80fbc30 msgr2=0x7f8eb80fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:54.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.561+0000 7f8ebf4a5640 1 -- 172.21.15.32:0/461120590 shutdown_connections 2024-04-25T00:30:54.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.561+0000 7f8ebf4a5640 1 -- 172.21.15.32:0/461120590 wait complete. 2024-04-25T00:30:54.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.561+0000 7f8ebf4a5640 1 Processor -- start 2024-04-25T00:30:54.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.561+0000 7f8ebf4a5640 1 -- start start 2024-04-25T00:30:54.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.561+0000 7f8ebf4a5640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8eb81006a0 0x7f8eb8104630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:54.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.561+0000 7f8ebf4a5640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8eb8101890 0x7f8eb8102c70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:54.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.561+0000 7f8ebf4a5640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8eb8102250 0x7f8eb81031b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:54.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.561+0000 7f8ebf4a5640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f8eb810c320 con 0x7f8eb8101890 2024-04-25T00:30:54.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.561+0000 7f8ebf4a5640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f8eb810c1a0 con 0x7f8eb8102250 2024-04-25T00:30:54.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.561+0000 7f8ebf4a5640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f8eb810c4a0 con 0x7f8eb81006a0 2024-04-25T00:30:54.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.562+0000 7f8ebca40640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8eb8101890 0x7f8eb8102c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:54.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.562+0000 7f8ebca40640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8eb8101890 0x7f8eb8102c70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46690/0 (socket says 172.21.15.32:46690) 2024-04-25T00:30:54.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.562+0000 7f8ebca40640 1 -- 172.21.15.32:0/3101142236 learned_addr learned my addr 172.21.15.32:0/3101142236 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:54.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.562+0000 7f8ebda42640 1 --2- 172.21.15.32:0/3101142236 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8eb8102250 0x7f8eb81031b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:54.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.562+0000 7f8ebd241640 1 --2- 172.21.15.32:0/3101142236 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8eb81006a0 0x7f8eb8104630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:54.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.562+0000 7f8ebca40640 1 -- 172.21.15.32:0/3101142236 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8eb81006a0 msgr2=0x7f8eb8104630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:54.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.562+0000 7f8ebca40640 1 --2- 172.21.15.32:0/3101142236 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8eb81006a0 0x7f8eb8104630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.562+0000 7f8ebca40640 1 -- 172.21.15.32:0/3101142236 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8eb8102250 msgr2=0x7f8eb81031b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:54.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.562+0000 7f8ebca40640 1 --2- 172.21.15.32:0/3101142236 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8eb8102250 0x7f8eb81031b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.562+0000 7f8ebca40640 1 -- 172.21.15.32:0/3101142236 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8eb8103a40 con 0x7f8eb8101890 2024-04-25T00:30:54.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.562+0000 7f8ebd241640 1 --2- 172.21.15.32:0/3101142236 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8eb81006a0 0x7f8eb8104630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:30:54.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.562+0000 7f8ebca40640 1 --2- 172.21.15.32:0/3101142236 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8eb8101890 0x7f8eb8102c70 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7f8ea800d950 tx=0x7f8ea800de10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:54.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.562+0000 7f8eae7fc640 1 -- 172.21.15.32:0/3101142236 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8ea8017030 con 0x7f8eb8101890 2024-04-25T00:30:54.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.562+0000 7f8eae7fc640 1 -- 172.21.15.32:0/3101142236 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8ea8004020 con 0x7f8eb8101890 2024-04-25T00:30:54.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.562+0000 7f8eae7fc640 1 -- 172.21.15.32:0/3101142236 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8ea80047a0 con 0x7f8eb8101890 2024-04-25T00:30:54.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.562+0000 7f8ebf4a5640 1 -- 172.21.15.32:0/3101142236 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8eb81a1bb0 con 0x7f8eb8101890 2024-04-25T00:30:54.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.562+0000 7f8ebf4a5640 1 -- 172.21.15.32:0/3101142236 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8eb81a2100 con 0x7f8eb8101890 2024-04-25T00:30:54.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.563+0000 7f8ebf4a5640 1 -- 172.21.15.32:0/3101142236 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8eb81a2330 con 0x7f8eb8101890 2024-04-25T00:30:54.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.564+0000 7f8eae7fc640 1 -- 172.21.15.32:0/3101142236 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f8ea80041d0 con 0x7f8eb8101890 2024-04-25T00:30:54.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.565+0000 7f8eae7fc640 1 --2- 172.21.15.32:0/3101142236 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8ea40a2b90 0x7f8ea40a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:54.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.565+0000 7f8ebd241640 1 --2- 172.21.15.32:0/3101142236 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8ea40a2b90 0x7f8ea40a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:54.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.566+0000 7f8eae7fc640 1 -- 172.21.15.32:0/3101142236 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f8ea80cede0 con 0x7f8eb8101890 2024-04-25T00:30:54.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.566+0000 7f8ebd241640 1 --2- 172.21.15.32:0/3101142236 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8ea40a2b90 0x7f8ea40a5050 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f8eb4000c00 tx=0x7f8eb4031400 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:54.570 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.570+0000 7f8eae7fc640 1 -- 172.21.15.32:0/3101142236 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f8ea809bb00 con 0x7f8eb8101890 2024-04-25T00:30:54.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.759+0000 7f8ebf4a5640 1 -- 172.21.15.32:0/3101142236 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f8eb81a26c0 con 0x7f8eb8101890 2024-04-25T00:30:54.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.761+0000 7f8eae7fc640 1 -- 172.21.15.32:0/3101142236 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v89) ==== 74+0+24128 (secure 0 0 0) 0x7f8ea809b400 con 0x7f8eb8101890 2024-04-25T00:30:54.762 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:54.762 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":89,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:30:53.134605+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-25T00:30:46.239556+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-25T00:30:47.257182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2024-04-25T00:30:51.193975+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":83,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-num-failure-domains":"0","crush-osds-per-failure-domain":"0","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:30:54.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.763+0000 7f8ebf4a5640 1 -- 172.21.15.32:0/3101142236 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8ea40a2b90 msgr2=0x7f8ea40a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:54.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.763+0000 7f8ebf4a5640 1 --2- 172.21.15.32:0/3101142236 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8ea40a2b90 0x7f8ea40a5050 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f8eb4000c00 tx=0x7f8eb4031400 comp rx=0 tx=0).stop 2024-04-25T00:30:54.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.763+0000 7f8ebf4a5640 1 -- 172.21.15.32:0/3101142236 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8eb8101890 msgr2=0x7f8eb8102c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:54.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.763+0000 7f8ebf4a5640 1 --2- 172.21.15.32:0/3101142236 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8eb8101890 0x7f8eb8102c70 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7f8ea800d950 tx=0x7f8ea800de10 comp rx=0 tx=0).stop 2024-04-25T00:30:54.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.763+0000 7f8ebf4a5640 1 -- 172.21.15.32:0/3101142236 shutdown_connections 2024-04-25T00:30:54.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.763+0000 7f8ebf4a5640 1 --2- 172.21.15.32:0/3101142236 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8ea40a2b90 0x7f8ea40a5050 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.763+0000 7f8ebf4a5640 1 --2- 172.21.15.32:0/3101142236 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8eb8102250 0x7f8eb81031b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.763+0000 7f8ebf4a5640 1 --2- 172.21.15.32:0/3101142236 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8eb8101890 0x7f8eb8102c70 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.763+0000 7f8ebf4a5640 1 --2- 172.21.15.32:0/3101142236 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8eb81006a0 0x7f8eb8104630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.763+0000 7f8ebf4a5640 1 -- 172.21.15.32:0/3101142236 >> 172.21.15.32:0/3101142236 conn(0x7f8eb80fbc30 msgr2=0x7f8eb80fd6b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:54.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.763+0000 7f8ebf4a5640 1 -- 172.21.15.32:0/3101142236 shutdown_connections 2024-04-25T00:30:54.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.763+0000 7f8ebf4a5640 1 -- 172.21.15.32:0/3101142236 wait complete. 2024-04-25T00:30:54.795 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:30:54.899 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.895+0000 7f4ba2a15640 1 -- 172.21.15.32:0/3268447489 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4b9c102250 msgr2=0x7f4b9c108ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:54.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.895+0000 7f4ba2a15640 1 --2- 172.21.15.32:0/3268447489 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4b9c102250 0x7f4b9c108ae0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f4b94009ca0 tx=0x7f4b9402a8c0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.895+0000 7f4ba2a15640 1 -- 172.21.15.32:0/3268447489 shutdown_connections 2024-04-25T00:30:54.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.895+0000 7f4ba2a15640 1 --2- 172.21.15.32:0/3268447489 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4b9c102250 0x7f4b9c108ae0 unknown :-1 s=CLOSED pgs=217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.895+0000 7f4ba2a15640 1 --2- 172.21.15.32:0/3268447489 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b9c101890 0x7f4b9c101d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.895+0000 7f4ba2a15640 1 --2- 172.21.15.32:0/3268447489 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4b9c1006a0 0x7f4b9c100ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.896+0000 7f4ba2a15640 1 -- 172.21.15.32:0/3268447489 >> 172.21.15.32:0/3268447489 conn(0x7f4b9c0fbc30 msgr2=0x7f4b9c0fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:54.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.896+0000 7f4ba2a15640 1 -- 172.21.15.32:0/3268447489 shutdown_connections 2024-04-25T00:30:54.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.896+0000 7f4ba2a15640 1 -- 172.21.15.32:0/3268447489 wait complete. 2024-04-25T00:30:54.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.896+0000 7f4ba2a15640 1 Processor -- start 2024-04-25T00:30:54.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.896+0000 7f4ba2a15640 1 -- start start 2024-04-25T00:30:54.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.896+0000 7f4ba2a15640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4b9c1006a0 0x7f4b9c104630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:54.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.896+0000 7f4ba2a15640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4b9c101890 0x7f4b9c102c70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:54.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.896+0000 7f4ba2a15640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b9c102250 0x7f4b9c1031b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:54.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.896+0000 7f4ba2a15640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f4b9c10c300 con 0x7f4b9c101890 2024-04-25T00:30:54.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.896+0000 7f4ba2a15640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f4b9c10c180 con 0x7f4b9c1006a0 2024-04-25T00:30:54.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.897+0000 7f4ba2a15640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f4b9c10c480 con 0x7f4b9c102250 2024-04-25T00:30:54.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.897+0000 7f4b9b7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4b9c101890 0x7f4b9c102c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:54.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.897+0000 7f4b9b7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4b9c101890 0x7f4b9c102c70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46710/0 (socket says 172.21.15.32:46710) 2024-04-25T00:30:54.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.897+0000 7f4b9b7fe640 1 -- 172.21.15.32:0/1533636337 learned_addr learned my addr 172.21.15.32:0/1533636337 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:54.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.897+0000 7f4b9bfff640 1 --2- 172.21.15.32:0/1533636337 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4b9c1006a0 0x7f4b9c104630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:54.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.897+0000 7f4ba0fb2640 1 --2- 172.21.15.32:0/1533636337 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b9c102250 0x7f4b9c1031b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:54.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.897+0000 7f4b9b7fe640 1 -- 172.21.15.32:0/1533636337 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b9c102250 msgr2=0x7f4b9c1031b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:54.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.897+0000 7f4b9b7fe640 1 --2- 172.21.15.32:0/1533636337 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b9c102250 0x7f4b9c1031b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.897+0000 7f4b9b7fe640 1 -- 172.21.15.32:0/1533636337 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4b9c1006a0 msgr2=0x7f4b9c104630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:54.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.897+0000 7f4b9b7fe640 1 --2- 172.21.15.32:0/1533636337 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4b9c1006a0 0x7f4b9c104630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:54.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.897+0000 7f4b9b7fe640 1 -- 172.21.15.32:0/1533636337 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4b9c103a40 con 0x7f4b9c101890 2024-04-25T00:30:54.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.897+0000 7f4b9b7fe640 1 --2- 172.21.15.32:0/1533636337 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4b9c101890 0x7f4b9c102c70 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f4b9000d950 tx=0x7f4b9000de10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:54.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.897+0000 7f4b997fa640 1 -- 172.21.15.32:0/1533636337 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4b90017030 con 0x7f4b9c101890 2024-04-25T00:30:54.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.897+0000 7f4ba2a15640 1 -- 172.21.15.32:0/1533636337 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4b9c1a1c40 con 0x7f4b9c101890 2024-04-25T00:30:54.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.897+0000 7f4b997fa640 1 -- 172.21.15.32:0/1533636337 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4b90004020 con 0x7f4b9c101890 2024-04-25T00:30:54.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.897+0000 7f4b997fa640 1 -- 172.21.15.32:0/1533636337 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4b900047a0 con 0x7f4b9c101890 2024-04-25T00:30:54.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.897+0000 7f4ba2a15640 1 -- 172.21.15.32:0/1533636337 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4b9c1a2190 con 0x7f4b9c101890 2024-04-25T00:30:54.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.898+0000 7f4ba2a15640 1 -- 172.21.15.32:0/1533636337 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4b9c1a23c0 con 0x7f4b9c101890 2024-04-25T00:30:54.903 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.899+0000 7f4b997fa640 1 -- 172.21.15.32:0/1533636337 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f4b900041d0 con 0x7f4b9c101890 2024-04-25T00:30:54.903 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.900+0000 7f4b997fa640 1 --2- 172.21.15.32:0/1533636337 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4b840a2b90 0x7f4b840a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:54.903 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.900+0000 7f4b9bfff640 1 --2- 172.21.15.32:0/1533636337 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4b840a2b90 0x7f4b840a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:54.903 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.900+0000 7f4b997fa640 1 -- 172.21.15.32:0/1533636337 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f4b900cec00 con 0x7f4b9c101890 2024-04-25T00:30:54.903 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.900+0000 7f4b9bfff640 1 --2- 172.21.15.32:0/1533636337 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4b840a2b90 0x7f4b840a5050 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f4b9c1016f0 tx=0x7f4b8c00b390 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:54.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:54.904+0000 7f4b997fa640 1 -- 172.21.15.32:0/1533636337 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f4b900d5080 con 0x7f4b9c101890 2024-04-25T00:30:55.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:54 smithi121 ceph-mon[32231]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-25T00:30:55.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:54 smithi121 ceph-mon[32231]: osdmap e89: 12 total, 12 up, 12 in 2024-04-25T00:30:55.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:54 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1293821860' entity='client.admin' cmd='[{"prefix": "fs add_data_pool", "fs_name": "cephfs", "pool": "cephfs_data_ec"}]': finished 2024-04-25T00:30:55.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:54 smithi121 ceph-mon[32231]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-25T00:30:55.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:54 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3734810551' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:55.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:54 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/265876846' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:30:55.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:54 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3101142236' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:30:55.129 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.128+0000 7f4ba2a15640 1 -- 172.21.15.32:0/1533636337 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f4b9c1a2750 con 0x7f4b9c101890 2024-04-25T00:30:55.130 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.128+0000 7f4b997fa640 1 -- 172.21.15.32:0/1533636337 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 17 v17) ==== 94+0+11803 (secure 0 0 0) 0x7f4b9009b920 con 0x7f4b9c101890 2024-04-25T00:30:55.130 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:55.130 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":17,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14643,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6839/187492674","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6838","nonce":187492674},{"type":"v1","addr":"172.21.15.32:6839","nonce":187492674}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14655,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6841/1948785545","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6840","nonce":1948785545},{"type":"v1","addr":"172.21.15.32:6841","nonce":1948785545}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24443,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6},{"gid":24445,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":7},{"gid":24457,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":8},{"gid":24476,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":9},{"gid":24481,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6839/4267125809","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":4267125809},{"type":"v1","addr":"172.21.15.130:6839","nonce":4267125809}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":10}],"filesystems":[{"mdsmap":{"epoch":17,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-04-25T00:30:47.866875+0000","modified":"2024-04-25T00:30:53.134841+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":1,"in":[0],"up":{"mds_0":24506},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24506":{"gid":24506,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.121:6839/258444853","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6838","nonce":258444853},{"type":"v1","addr":"172.21.15.121:6839","nonce":258444853}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":1,"qdb_leader":24506,"qdb_cluster":[24506]},"id":1}]} 2024-04-25T00:30:55.130 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 17 2024-04-25T00:30:55.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.130+0000 7f4ba2a15640 1 -- 172.21.15.32:0/1533636337 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4b840a2b90 msgr2=0x7f4b840a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:55.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.130+0000 7f4ba2a15640 1 --2- 172.21.15.32:0/1533636337 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4b840a2b90 0x7f4b840a5050 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f4b9c1016f0 tx=0x7f4b8c00b390 comp rx=0 tx=0).stop 2024-04-25T00:30:55.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.130+0000 7f4ba2a15640 1 -- 172.21.15.32:0/1533636337 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4b9c101890 msgr2=0x7f4b9c102c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:55.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.130+0000 7f4ba2a15640 1 --2- 172.21.15.32:0/1533636337 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4b9c101890 0x7f4b9c102c70 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f4b9000d950 tx=0x7f4b9000de10 comp rx=0 tx=0).stop 2024-04-25T00:30:55.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.130+0000 7f4ba2a15640 1 -- 172.21.15.32:0/1533636337 shutdown_connections 2024-04-25T00:30:55.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.130+0000 7f4ba2a15640 1 --2- 172.21.15.32:0/1533636337 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4b840a2b90 0x7f4b840a5050 unknown :-1 s=CLOSED pgs=160 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.130+0000 7f4ba2a15640 1 --2- 172.21.15.32:0/1533636337 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b9c102250 0x7f4b9c1031b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.130+0000 7f4ba2a15640 1 --2- 172.21.15.32:0/1533636337 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4b9c101890 0x7f4b9c102c70 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.130+0000 7f4ba2a15640 1 --2- 172.21.15.32:0/1533636337 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4b9c1006a0 0x7f4b9c104630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.130+0000 7f4ba2a15640 1 -- 172.21.15.32:0/1533636337 >> 172.21.15.32:0/1533636337 conn(0x7f4b9c0fbc30 msgr2=0x7f4b9c0fd6b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:55.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.130+0000 7f4ba2a15640 1 -- 172.21.15.32:0/1533636337 shutdown_connections 2024-04-25T00:30:55.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.130+0000 7f4ba2a15640 1 -- 172.21.15.32:0/1533636337 wait complete. 2024-04-25T00:30:55.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:54 smithi130 ceph-mon[32525]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-25T00:30:55.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:54 smithi130 ceph-mon[32525]: osdmap e89: 12 total, 12 up, 12 in 2024-04-25T00:30:55.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:54 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1293821860' entity='client.admin' cmd='[{"prefix": "fs add_data_pool", "fs_name": "cephfs", "pool": "cephfs_data_ec"}]': finished 2024-04-25T00:30:55.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:54 smithi130 ceph-mon[32525]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-25T00:30:55.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:54 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3734810551' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:55.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:54 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/265876846' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:30:55.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:54 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3101142236' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:30:55.143 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:54 smithi032 ceph-mon[29357]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-25T00:30:55.143 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:54 smithi032 ceph-mon[29357]: osdmap e89: 12 total, 12 up, 12 in 2024-04-25T00:30:55.143 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:54 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1293821860' entity='client.admin' cmd='[{"prefix": "fs add_data_pool", "fs_name": "cephfs", "pool": "cephfs_data_ec"}]': finished 2024-04-25T00:30:55.143 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:54 smithi032 ceph-mon[29357]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-25T00:30:55.143 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:54 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3734810551' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:55.143 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:54 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/265876846' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:30:55.143 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:54 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3101142236' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:30:55.144 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-25T00:30:55.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.244+0000 7fde972d6640 1 -- 172.21.15.32:0/433257119 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fde90102230 msgr2=0x7fde90108ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:55.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.244+0000 7fde972d6640 1 --2- 172.21.15.32:0/433257119 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fde90102230 0x7fde90108ac0 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7fde88009ca0 tx=0x7fde8802a8c0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.244+0000 7fde972d6640 1 -- 172.21.15.32:0/433257119 shutdown_connections 2024-04-25T00:30:55.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.244+0000 7fde972d6640 1 --2- 172.21.15.32:0/433257119 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fde90102230 0x7fde90108ac0 unknown :-1 s=CLOSED pgs=219 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.244+0000 7fde972d6640 1 --2- 172.21.15.32:0/433257119 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fde90101870 0x7fde90101cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.244+0000 7fde972d6640 1 --2- 172.21.15.32:0/433257119 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fde90100680 0x7fde90100aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.244+0000 7fde972d6640 1 -- 172.21.15.32:0/433257119 >> 172.21.15.32:0/433257119 conn(0x7fde900fbbf0 msgr2=0x7fde900fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:55.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.245+0000 7fde972d6640 1 -- 172.21.15.32:0/433257119 shutdown_connections 2024-04-25T00:30:55.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.245+0000 7fde972d6640 1 -- 172.21.15.32:0/433257119 wait complete. 2024-04-25T00:30:55.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.245+0000 7fde972d6640 1 Processor -- start 2024-04-25T00:30:55.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.245+0000 7fde972d6640 1 -- start start 2024-04-25T00:30:55.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.245+0000 7fde972d6640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fde90100680 0x7fde90197aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:55.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.245+0000 7fde972d6640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fde90101870 0x7fde90197fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:55.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.245+0000 7fde972d6640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fde90102230 0x7fde9019f050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:55.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.245+0000 7fde972d6640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fde9010d460 con 0x7fde90101870 2024-04-25T00:30:55.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.245+0000 7fde972d6640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fde9010d2e0 con 0x7fde90102230 2024-04-25T00:30:55.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.245+0000 7fde972d6640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fde9010d5e0 con 0x7fde90100680 2024-04-25T00:30:55.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.245+0000 7fde94871640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fde90101870 0x7fde90197fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:55.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.245+0000 7fde94871640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fde90101870 0x7fde90197fe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46722/0 (socket says 172.21.15.32:46722) 2024-04-25T00:30:55.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.245+0000 7fde94871640 1 -- 172.21.15.32:0/1039545753 learned_addr learned my addr 172.21.15.32:0/1039545753 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:55.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.245+0000 7fde95873640 1 --2- 172.21.15.32:0/1039545753 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fde90102230 0x7fde9019f050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:55.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.245+0000 7fde95072640 1 --2- 172.21.15.32:0/1039545753 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fde90100680 0x7fde90197aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:55.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.246+0000 7fde94871640 1 -- 172.21.15.32:0/1039545753 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fde90100680 msgr2=0x7fde90197aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:55.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.246+0000 7fde94871640 1 --2- 172.21.15.32:0/1039545753 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fde90100680 0x7fde90197aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.246+0000 7fde94871640 1 -- 172.21.15.32:0/1039545753 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fde90102230 msgr2=0x7fde9019f050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:55.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.246+0000 7fde94871640 1 --2- 172.21.15.32:0/1039545753 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fde90102230 0x7fde9019f050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.246+0000 7fde94871640 1 -- 172.21.15.32:0/1039545753 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fde9019f6c0 con 0x7fde90101870 2024-04-25T00:30:55.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.246+0000 7fde94871640 1 --2- 172.21.15.32:0/1039545753 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fde90101870 0x7fde90197fe0 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7fde8000b4a0 tx=0x7fde8000b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:55.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.246+0000 7fde867fc640 1 -- 172.21.15.32:0/1039545753 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fde80014070 con 0x7fde90101870 2024-04-25T00:30:55.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.246+0000 7fde867fc640 1 -- 172.21.15.32:0/1039545753 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fde80010af0 con 0x7fde90101870 2024-04-25T00:30:55.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.246+0000 7fde972d6640 1 -- 172.21.15.32:0/1039545753 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fde9019f9b0 con 0x7fde90101870 2024-04-25T00:30:55.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.246+0000 7fde867fc640 1 -- 172.21.15.32:0/1039545753 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fde80017d10 con 0x7fde90101870 2024-04-25T00:30:55.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.246+0000 7fde972d6640 1 -- 172.21.15.32:0/1039545753 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fde9019ff30 con 0x7fde90101870 2024-04-25T00:30:55.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.247+0000 7fde972d6640 1 -- 172.21.15.32:0/1039545753 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fde901078d0 con 0x7fde90101870 2024-04-25T00:30:55.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.248+0000 7fde867fc640 1 -- 172.21.15.32:0/1039545753 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7fde80017070 con 0x7fde90101870 2024-04-25T00:30:55.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.249+0000 7fde867fc640 1 --2- 172.21.15.32:0/1039545753 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fde7c0a2b90 0x7fde7c0a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:55.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.249+0000 7fde95072640 1 --2- 172.21.15.32:0/1039545753 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fde7c0a2b90 0x7fde7c0a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:55.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.249+0000 7fde867fc640 1 -- 172.21.15.32:0/1039545753 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7fde800ce9a0 con 0x7fde90101870 2024-04-25T00:30:55.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.249+0000 7fde95072640 1 --2- 172.21.15.32:0/1039545753 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fde7c0a2b90 0x7fde7c0a5050 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7fde901016d0 tx=0x7fde8c00b390 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:55.253 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.252+0000 7fde867fc640 1 -- 172.21.15.32:0/1039545753 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fde8009b010 con 0x7fde90101870 2024-04-25T00:30:55.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.441+0000 7fde972d6640 1 -- 172.21.15.32:0/1039545753 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fde9002d170 con 0x7fde90101870 2024-04-25T00:30:55.444 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.442+0000 7fde867fc640 1 -- 172.21.15.32:0/1039545753 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v89) ==== 74+0+24128 (secure 0 0 0) 0x7fde8009f310 con 0x7fde90101870 2024-04-25T00:30:55.444 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:30:55.444 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":89,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:30:53.134605+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-25T00:30:46.239556+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-25T00:30:47.257182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2024-04-25T00:30:51.193975+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":83,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-num-failure-domains":"0","crush-osds-per-failure-domain":"0","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:30:55.446 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.444+0000 7fde972d6640 1 -- 172.21.15.32:0/1039545753 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fde7c0a2b90 msgr2=0x7fde7c0a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:55.446 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.444+0000 7fde972d6640 1 --2- 172.21.15.32:0/1039545753 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fde7c0a2b90 0x7fde7c0a5050 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7fde901016d0 tx=0x7fde8c00b390 comp rx=0 tx=0).stop 2024-04-25T00:30:55.446 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.444+0000 7fde972d6640 1 -- 172.21.15.32:0/1039545753 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fde90101870 msgr2=0x7fde90197fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:55.446 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.444+0000 7fde972d6640 1 --2- 172.21.15.32:0/1039545753 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fde90101870 0x7fde90197fe0 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7fde8000b4a0 tx=0x7fde8000b960 comp rx=0 tx=0).stop 2024-04-25T00:30:55.446 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.444+0000 7fde972d6640 1 -- 172.21.15.32:0/1039545753 shutdown_connections 2024-04-25T00:30:55.446 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.444+0000 7fde972d6640 1 --2- 172.21.15.32:0/1039545753 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fde7c0a2b90 0x7fde7c0a5050 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.446 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.444+0000 7fde972d6640 1 --2- 172.21.15.32:0/1039545753 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fde90102230 0x7fde9019f050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.444+0000 7fde972d6640 1 --2- 172.21.15.32:0/1039545753 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fde90101870 0x7fde90197fe0 unknown :-1 s=CLOSED pgs=220 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.444+0000 7fde972d6640 1 --2- 172.21.15.32:0/1039545753 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fde90100680 0x7fde90197aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.444+0000 7fde972d6640 1 -- 172.21.15.32:0/1039545753 >> 172.21.15.32:0/1039545753 conn(0x7fde900fbbf0 msgr2=0x7fde90104aa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:55.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.444+0000 7fde972d6640 1 -- 172.21.15.32:0/1039545753 shutdown_connections 2024-04-25T00:30:55.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.444+0000 7fde972d6640 1 -- 172.21.15.32:0/1039545753 wait complete. 2024-04-25T00:30:55.457 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 17, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-04-25T00:30:47.866875+0000', 'modified': '2024-04-25T00:30:53.134841+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 24506}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_24506': {'gid': 24506, 'name': 'b', 'rank': 0, 'incarnation': 14, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.121:6839/258444853', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.121:6838', 'nonce': 258444853}, {'type': 'v1', 'addr': '172.21.15.121:6839', 'nonce': 258444853}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}}, 'data_pools': [3, 4], 'metadata_pool': 2, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 1, 'qdb_leader': 24506, 'qdb_cluster': [24506]} 2024-04-25T00:30:55.458 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-04-25T00:30:55.458 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.b status 2024-04-25T00:30:55.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.561+0000 7fa621b12640 1 -- 172.21.15.32:0/872073881 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa61c1022e0 msgr2=0x7fa61c1046d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:55.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.561+0000 7fa621b12640 1 --2- 172.21.15.32:0/872073881 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa61c1022e0 0x7fa61c1046d0 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7fa60c009900 tx=0x7fa60c02a410 comp rx=0 tx=0).stop 2024-04-25T00:30:55.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.561+0000 7fa621b12640 1 -- 172.21.15.32:0/872073881 shutdown_connections 2024-04-25T00:30:55.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.561+0000 7fa621b12640 1 --2- 172.21.15.32:0/872073881 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa61c107540 0x7fa61c109930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.561+0000 7fa621b12640 1 --2- 172.21.15.32:0/872073881 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa61c104c10 0x7fa61c107000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.561+0000 7fa621b12640 1 --2- 172.21.15.32:0/872073881 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa61c1022e0 0x7fa61c1046d0 unknown :-1 s=CLOSED pgs=221 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.561+0000 7fa621b12640 1 -- 172.21.15.32:0/872073881 >> 172.21.15.32:0/872073881 conn(0x7fa61c0fbc30 msgr2=0x7fa61c0fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:55.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.561+0000 7fa621b12640 1 -- 172.21.15.32:0/872073881 shutdown_connections 2024-04-25T00:30:55.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.561+0000 7fa621b12640 1 -- 172.21.15.32:0/872073881 wait complete. 2024-04-25T00:30:55.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.561+0000 7fa621b12640 1 Processor -- start 2024-04-25T00:30:55.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.562+0000 7fa621b12640 1 -- start start 2024-04-25T00:30:55.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.562+0000 7fa621b12640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa61c1022e0 0x7fa61c101c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:55.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.562+0000 7fa621b12640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa61c104c10 0x7fa61c100290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:55.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.562+0000 7fa621b12640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa61c107540 0x7fa61c1007d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:55.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.562+0000 7fa621b12640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fa61c10d420 con 0x7fa61c107540 2024-04-25T00:30:55.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.562+0000 7fa621b12640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fa61c10d2a0 con 0x7fa61c104c10 2024-04-25T00:30:55.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.562+0000 7fa621b12640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fa61c10d5a0 con 0x7fa61c1022e0 2024-04-25T00:30:55.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.562+0000 7fa61bfff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa61c107540 0x7fa61c1007d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:55.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.562+0000 7fa61b7fe640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa61c1022e0 0x7fa61c101c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:55.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.562+0000 7fa61affd640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa61c104c10 0x7fa61c100290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:55.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.562+0000 7fa61bfff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa61c107540 0x7fa61c1007d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46736/0 (socket says 172.21.15.32:46736) 2024-04-25T00:30:55.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.562+0000 7fa61bfff640 1 -- 172.21.15.32:0/2833964221 learned_addr learned my addr 172.21.15.32:0/2833964221 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:55.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.562+0000 7fa61bfff640 1 -- 172.21.15.32:0/2833964221 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa61c1022e0 msgr2=0x7fa61c101c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:55.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.562+0000 7fa61bfff640 1 --2- 172.21.15.32:0/2833964221 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa61c1022e0 0x7fa61c101c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.562+0000 7fa61bfff640 1 -- 172.21.15.32:0/2833964221 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa61c104c10 msgr2=0x7fa61c100290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:55.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.562+0000 7fa61bfff640 1 --2- 172.21.15.32:0/2833964221 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa61c104c10 0x7fa61c100290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.562+0000 7fa61bfff640 1 -- 172.21.15.32:0/2833964221 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa61c101060 con 0x7fa61c107540 2024-04-25T00:30:55.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.562+0000 7fa61affd640 1 --2- 172.21.15.32:0/2833964221 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa61c104c10 0x7fa61c100290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:30:55.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.562+0000 7fa61bfff640 1 --2- 172.21.15.32:0/2833964221 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa61c107540 0x7fa61c1007d0 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7fa614009fd0 tx=0x7fa61400b260 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:55.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.563+0000 7fa618ff9640 1 -- 172.21.15.32:0/2833964221 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa614017070 con 0x7fa61c107540 2024-04-25T00:30:55.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.563+0000 7fa618ff9640 1 -- 172.21.15.32:0/2833964221 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa614003e50 con 0x7fa61c107540 2024-04-25T00:30:55.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.563+0000 7fa618ff9640 1 -- 172.21.15.32:0/2833964221 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa6140106c0 con 0x7fa61c107540 2024-04-25T00:30:55.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.563+0000 7fa621b12640 1 -- 172.21.15.32:0/2833964221 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa61c1a1c50 con 0x7fa61c107540 2024-04-25T00:30:55.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.563+0000 7fa621b12640 1 -- 172.21.15.32:0/2833964221 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa61c1a2150 con 0x7fa61c107540 2024-04-25T00:30:55.574 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.565+0000 7fa618ff9640 1 -- 172.21.15.32:0/2833964221 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7fa614003970 con 0x7fa61c107540 2024-04-25T00:30:55.574 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.565+0000 7fa624816740 1 Processor -- start 2024-04-25T00:30:55.574 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.565+0000 7fa624816740 1 -- start start 2024-04-25T00:30:55.575 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.565+0000 7fa624816740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5606dc102440 0x5606dc102860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:55.575 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.565+0000 7fa624816740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102e30 0x5606dc107930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:55.575 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.565+0000 7fa624816740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5606dc107fa0 0x5606dc10a430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:55.575 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.565+0000 7fa624816740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5606dc065f00 con 0x5606dc102e30 2024-04-25T00:30:55.575 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.565+0000 7fa624816740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5606dc00e750 con 0x5606dc102440 2024-04-25T00:30:55.575 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.565+0000 7fa624816740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5606dc061ec0 con 0x5606dc107fa0 2024-04-25T00:30:55.575 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.565+0000 7fa618ff9640 1 --2- 172.21.15.32:0/2833964221 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa6040a2b90 0x7fa6040a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:55.575 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.566+0000 7fa61b7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102e30 0x5606dc107930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:55.575 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.566+0000 7fa61affd640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5606dc102440 0x5606dc102860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:55.575 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.566+0000 7fa61b7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102e30 0x5606dc107930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46750/0 (socket says 172.21.15.32:46750) 2024-04-25T00:30:55.575 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.566+0000 7fa61b7fe640 1 -- 172.21.15.32:0/20927622 learned_addr learned my addr 172.21.15.32:0/20927622 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:55.575 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.566+0000 7fa61bfff640 1 --2- 172.21.15.32:0/2833964221 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa6040a2b90 0x7fa6040a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:55.575 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.566+0000 7fa618ff9640 1 -- 172.21.15.32:0/2833964221 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7fa6140ceee0 con 0x7fa61c107540 2024-04-25T00:30:55.576 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.566+0000 7fa61affd640 1 --2- 172.21.15.32:0/20927622 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5606dc107fa0 0x5606dc10a430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:55.576 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.566+0000 7fa61b7fe640 1 -- 172.21.15.32:0/20927622 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5606dc107fa0 msgr2=0x5606dc10a430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:30:55.576 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.566+0000 7fa61b7fe640 1 --2- 172.21.15.32:0/20927622 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5606dc107fa0 0x5606dc10a430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.576 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.566+0000 7fa61b7fe640 1 -- 172.21.15.32:0/20927622 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5606dc102440 msgr2=0x5606dc102860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:30:55.576 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.566+0000 7fa61b7fe640 1 --2- 172.21.15.32:0/20927622 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5606dc102440 0x5606dc102860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.576 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.566+0000 7fa61b7fe640 1 -- 172.21.15.32:0/20927622 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5606dbd7a2d0 con 0x5606dc102e30 2024-04-25T00:30:55.576 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.566+0000 7fa61affd640 1 --2- 172.21.15.32:0/20927622 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5606dc102440 0x5606dc102860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:30:55.576 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.566+0000 7fa61b7fe640 1 --2- 172.21.15.32:0/20927622 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102e30 0x5606dc107930 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7fa60c02a930 tx=0x7fa60c032c30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=aff08e237448026c server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:55.576 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.566+0000 7fa61bfff640 1 --2- 172.21.15.32:0/2833964221 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa6040a2b90 0x7fa6040a5050 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7fa6140cc7f0 tx=0x7fa614086040 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:55.576 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.566+0000 7fa601ffb640 1 -- 172.21.15.32:0/20927622 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa60c040070 con 0x5606dc102e30 2024-04-25T00:30:55.576 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.566+0000 7fa601ffb640 1 -- 172.21.15.32:0/20927622 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa60c032e20 con 0x5606dc102e30 2024-04-25T00:30:55.576 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.566+0000 7fa601ffb640 1 -- 172.21.15.32:0/20927622 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa60c035040 con 0x5606dc102e30 2024-04-25T00:30:55.576 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.567+0000 7fa624816740 1 -- 172.21.15.32:0/20927622 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102e30 msgr2=0x5606dc107930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:55.576 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.567+0000 7fa624816740 1 --2- 172.21.15.32:0/20927622 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102e30 0x5606dc107930 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7fa60c02a930 tx=0x7fa60c032c30 comp rx=0 tx=0).stop 2024-04-25T00:30:55.576 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.567+0000 7fa624816740 1 -- 172.21.15.32:0/20927622 shutdown_connections 2024-04-25T00:30:55.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.567+0000 7fa624816740 1 --2- 172.21.15.32:0/20927622 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5606dc107fa0 0x5606dc10a430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.567+0000 7fa624816740 1 --2- 172.21.15.32:0/20927622 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102e30 0x5606dc107930 unknown :-1 s=CLOSED pgs=223 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.567+0000 7fa624816740 1 --2- 172.21.15.32:0/20927622 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5606dc102440 0x5606dc102860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.567+0000 7fa624816740 1 -- 172.21.15.32:0/20927622 >> 172.21.15.32:0/20927622 conn(0x5606dc0fdab0 msgr2=0x5606dc0fdeb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:55.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.567+0000 7fa624816740 1 -- 172.21.15.32:0/20927622 shutdown_connections 2024-04-25T00:30:55.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.567+0000 7fa624816740 1 -- 172.21.15.32:0/20927622 wait complete. 2024-04-25T00:30:55.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.567+0000 7fa624816740 1 -- start start 2024-04-25T00:30:55.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.568+0000 7fa624816740 1 Processor -- start 2024-04-25T00:30:55.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.568+0000 7fa624816740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5606dc102440 0x5606dc10c830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:55.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.568+0000 7fa624816740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102e30 0x5606dc10cd70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:55.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.568+0000 7fa624816740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5606dc107fa0 0x5606dc113db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:55.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.568+0000 7fa624816740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5606dc10aef0 con 0x5606dc102e30 2024-04-25T00:30:55.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.568+0000 7fa624816740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5606dc021cf0 con 0x5606dc107fa0 2024-04-25T00:30:55.578 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.568+0000 7fa624816740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5606dc10b070 con 0x5606dc102440 2024-04-25T00:30:55.578 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.568+0000 7fa624816740 10 client.0 _ll_register_callbacks cb 0x5606dc0216e0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:30:55.578 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.568+0000 7fa61b7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102e30 0x5606dc10cd70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:55.578 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.568+0000 7fa61b7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102e30 0x5606dc10cd70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46762/0 (socket says 172.21.15.32:46762) 2024-04-25T00:30:55.578 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.568+0000 7fa61b7fe640 1 -- 172.21.15.32:0/2116584277 learned_addr learned my addr 172.21.15.32:0/2116584277 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:55.578 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.568+0000 7fa61affd640 1 --2- 172.21.15.32:0/2116584277 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5606dc107fa0 0x5606dc113db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:55.578 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.568+0000 7fa61affd640 1 --2- 172.21.15.32:0/2116584277 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5606dc102440 0x5606dc10c830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:55.578 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.568+0000 7fa61b7fe640 1 -- 172.21.15.32:0/2116584277 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5606dc102440 msgr2=0x5606dc10c830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:30:55.578 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.568+0000 7fa61b7fe640 1 --2- 172.21.15.32:0/2116584277 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5606dc102440 0x5606dc10c830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.578 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.568+0000 7fa61b7fe640 1 -- 172.21.15.32:0/2116584277 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5606dc107fa0 msgr2=0x5606dc113db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:30:55.578 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.568+0000 7fa61b7fe640 1 --2- 172.21.15.32:0/2116584277 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5606dc107fa0 0x5606dc113db0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.578 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.568+0000 7fa61b7fe640 1 -- 172.21.15.32:0/2116584277 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5606dbe446b0 con 0x5606dc102e30 2024-04-25T00:30:55.578 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.568+0000 7fa61affd640 1 --2- 172.21.15.32:0/2116584277 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5606dc102440 0x5606dc10c830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:30:55.579 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.568+0000 7fa61b7fe640 1 --2- 172.21.15.32:0/2116584277 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102e30 0x5606dc10cd70 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7fa60c000c00 tx=0x7fa60c005770 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9a6fbcef51fd95b0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:55.579 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.568+0000 7fa5faffd640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:30:55.579 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.568+0000 7fa5faffd640 1 -- 172.21.15.32:0/2116584277 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa60c040050 con 0x5606dc102e30 2024-04-25T00:30:55.579 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.568+0000 7fa5faffd640 1 -- 172.21.15.32:0/2116584277 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7fa60c031730 con 0x5606dc102e30 2024-04-25T00:30:55.579 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.568+0000 7fa624816740 1 -- 172.21.15.32:0/2116584277 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa60c032e20 con 0x5606dc102e30 2024-04-25T00:30:55.579 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.569+0000 7fa5faffd640 1 -- 172.21.15.32:0/2116584277 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=17) ==== 24+0+0 (secure 0 0 0) 0x7fa60c035040 con 0x5606dc102e30 2024-04-25T00:30:55.579 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.569+0000 7fa624816740 10 client.14781 fetch_fsmap learned FSMap version 17 2024-04-25T00:30:55.579 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.569+0000 7fa624816740 1 -- 172.21.15.32:0/2116584277 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=17,monmap=4+}) -- 0x5606dc116d90 con 0x5606dc102e30 2024-04-25T00:30:55.579 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.569+0000 7fa5faffd640 1 -- 172.21.15.32:0/2116584277 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa60c032e20 con 0x5606dc102e30 2024-04-25T00:30:55.579 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.569+0000 7fa5faffd640 1 -- 172.21.15.32:0/2116584277 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 17) ==== 8342+0+0 (secure 0 0 0) 0x7fa60c003c50 con 0x5606dc102e30 2024-04-25T00:30:55.579 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.569+0000 7fa624816740 10 client.14781 fetch_fsmap finished waiting for FSMap version 17 2024-04-25T00:30:55.579 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.569+0000 7fa624816740 10 client.14781 resolve_mds: resolved name 'b' to daemon mds.b 2024-04-25T00:30:55.579 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.569+0000 7fa624816740 20 client.14781 populate_metadata read hostname 'smithi032' 2024-04-25T00:30:55.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.569+0000 7fa624816740 1 --2- 172.21.15.32:0/2116584277 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5606dc119020 0x5606dc11a490 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:55.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.569+0000 7fa624816740 4 client.14781 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:30:55.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.569+0000 7fa624816740 1 -- 172.21.15.32:0/2116584277 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5606dc073440 con 0x5606dc119020 2024-04-25T00:30:55.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.569+0000 7fa61affd640 1 --2- 172.21.15.32:0/2116584277 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5606dc119020 0x5606dc11a490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:55.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.570+0000 7fa5faffd640 10 client.14781 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:30:55.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.570+0000 7fa61affd640 1 --2- 172.21.15.32:0/2116584277 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5606dc119020 0x5606dc11a490 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=89a9cfa2b0588539 server_cookie=b660559183a4f8d2 in_seq=0 out_seq=0 2024-04-25T00:30:55.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.570+0000 7fa5faffd640 1 -- 172.21.15.32:0/2116584277 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x5606dc073440 con 0x5606dc119020 2024-04-25T00:30:55.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.570+0000 7fa5faffd640 10 client.14781 handle_command_reply: tid=0 2024-04-25T00:30:55.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.570+0000 7fa624816740 1 client.14781 shutdown 2024-04-25T00:30:55.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.571+0000 7fa624816740 1 -- 172.21.15.32:0/2116584277 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102e30 msgr2=0x5606dc10cd70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:55.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.571+0000 7fa624816740 1 --2- 172.21.15.32:0/2116584277 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102e30 0x5606dc10cd70 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7fa60c000c00 tx=0x7fa60c005770 comp rx=0 tx=0).stop 2024-04-25T00:30:55.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.571+0000 7fa624816740 1 -- 172.21.15.32:0/2116584277 shutdown_connections 2024-04-25T00:30:55.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.571+0000 7fa624816740 1 --2- 172.21.15.32:0/2116584277 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5606dc119020 0x5606dc11a490 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.571+0000 7fa624816740 1 --2- 172.21.15.32:0/2116584277 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5606dc107fa0 0x5606dc113db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.571+0000 7fa624816740 1 --2- 172.21.15.32:0/2116584277 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102e30 0x5606dc10cd70 unknown :-1 s=CLOSED pgs=224 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.581 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.571+0000 7fa624816740 1 --2- 172.21.15.32:0/2116584277 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5606dc102440 0x5606dc10c830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.581 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.571+0000 7fa5faffd640 0 client.14781 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:30:55.581 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.571+0000 7fa624816740 1 -- 172.21.15.32:0/2116584277 >> 172.21.15.32:0/2116584277 conn(0x5606dc0fdab0 msgr2=0x5606dc0fcae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:55.581 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.571+0000 7fa624816740 1 -- 172.21.15.32:0/2116584277 shutdown_connections 2024-04-25T00:30:55.581 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.571+0000 7fa624816740 1 -- 172.21.15.32:0/2116584277 wait complete. 2024-04-25T00:30:55.581 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.571+0000 7fa624816740 20 client.14781 trim_cache size 0 max 16384 2024-04-25T00:30:55.596 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.589+0000 7fa624816740 1 Processor -- start 2024-04-25T00:30:55.596 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.589+0000 7fa624816740 1 -- start start 2024-04-25T00:30:55.596 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.589+0000 7fa624816740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102440 0x5606dc10a390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:55.596 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.589+0000 7fa624816740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5606dc102e30 0x5606dc113d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:55.596 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.589+0000 7fa624816740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5606dc107fa0 0x5606dc1041f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:55.596 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.589+0000 7fa624816740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5606dc1187d0 con 0x5606dc102440 2024-04-25T00:30:55.597 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.589+0000 7fa624816740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5606dc118ea0 con 0x5606dc102e30 2024-04-25T00:30:55.597 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.589+0000 7fa624816740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5606dc119570 con 0x5606dc107fa0 2024-04-25T00:30:55.597 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.589+0000 7fa61affd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102440 0x5606dc10a390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:55.597 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.589+0000 7fa61affd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102440 0x5606dc10a390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46764/0 (socket says 172.21.15.32:46764) 2024-04-25T00:30:55.597 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.589+0000 7fa61affd640 1 -- 172.21.15.32:0/2189302101 learned_addr learned my addr 172.21.15.32:0/2189302101 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:55.597 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.589+0000 7fa61b7fe640 1 --2- 172.21.15.32:0/2189302101 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5606dc102e30 0x5606dc113d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:55.597 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.590+0000 7fa61affd640 1 -- 172.21.15.32:0/2189302101 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5606dc107fa0 msgr2=0x5606dc1041f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:30:55.597 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.590+0000 7fa61affd640 1 --2- 172.21.15.32:0/2189302101 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5606dc107fa0 0x5606dc1041f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.597 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.590+0000 7fa61affd640 1 -- 172.21.15.32:0/2189302101 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5606dc102e30 msgr2=0x5606dc113d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:30:55.597 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.590+0000 7fa61affd640 1 --2- 172.21.15.32:0/2189302101 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5606dc102e30 0x5606dc113d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.597 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.590+0000 7fa61affd640 1 -- 172.21.15.32:0/2189302101 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5606dc107c80 con 0x5606dc102440 2024-04-25T00:30:55.597 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.590+0000 7fa61b7fe640 1 --2- 172.21.15.32:0/2189302101 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5606dc102e30 0x5606dc113d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:30:55.597 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.590+0000 7fa61affd640 1 --2- 172.21.15.32:0/2189302101 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102440 0x5606dc10a390 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7fa61c06a7e0 tx=0x7fa610005f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cb861a57f44206b1 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:55.597 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.590+0000 7fa6017fa640 1 -- 172.21.15.32:0/2189302101 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa610020c20 con 0x5606dc102440 2024-04-25T00:30:55.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.590+0000 7fa6017fa640 1 -- 172.21.15.32:0/2189302101 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa610020250 con 0x5606dc102440 2024-04-25T00:30:55.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.590+0000 7fa6017fa640 1 -- 172.21.15.32:0/2189302101 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa6100047a0 con 0x5606dc102440 2024-04-25T00:30:55.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.590+0000 7fa624816740 1 -- 172.21.15.32:0/2189302101 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102440 msgr2=0x5606dc10a390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:55.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.590+0000 7fa624816740 1 --2- 172.21.15.32:0/2189302101 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102440 0x5606dc10a390 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7fa61c06a7e0 tx=0x7fa610005f90 comp rx=0 tx=0).stop 2024-04-25T00:30:55.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.591+0000 7fa624816740 1 -- 172.21.15.32:0/2189302101 shutdown_connections 2024-04-25T00:30:55.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.591+0000 7fa624816740 1 --2- 172.21.15.32:0/2189302101 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5606dc107fa0 0x5606dc1041f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.591+0000 7fa624816740 1 --2- 172.21.15.32:0/2189302101 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5606dc102e30 0x5606dc113d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.591+0000 7fa624816740 1 --2- 172.21.15.32:0/2189302101 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102440 0x5606dc10a390 unknown :-1 s=CLOSED pgs=225 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.591+0000 7fa624816740 1 -- 172.21.15.32:0/2189302101 >> 172.21.15.32:0/2189302101 conn(0x5606dc0fdab0 msgr2=0x5606dc0fde90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:55.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.591+0000 7fa624816740 1 -- 172.21.15.32:0/2189302101 shutdown_connections 2024-04-25T00:30:55.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.591+0000 7fa624816740 1 -- 172.21.15.32:0/2189302101 wait complete. 2024-04-25T00:30:55.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.591+0000 7fa624816740 1 -- start start 2024-04-25T00:30:55.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.591+0000 7fa624816740 1 Processor -- start 2024-04-25T00:30:55.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.591+0000 7fa624816740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5606dc102440 0x5606dc111fb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:55.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.591+0000 7fa624816740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102e30 0x5606dc1124f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:55.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.591+0000 7fa624816740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5606dc107fa0 0x5606dc112a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:55.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.591+0000 7fa624816740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5606dc11a310 con 0x5606dc102e30 2024-04-25T00:30:55.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.591+0000 7fa624816740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5606dc119c40 con 0x5606dc102440 2024-04-25T00:30:55.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.591+0000 7fa624816740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5606dc11a9e0 con 0x5606dc107fa0 2024-04-25T00:30:55.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.591+0000 7fa624816740 10 client.0 _ll_register_callbacks cb 0x7fa60c02ab10 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:30:55.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.591+0000 7fa61b7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102e30 0x5606dc1124f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:55.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.591+0000 7fa61b7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102e30 0x5606dc1124f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46772/0 (socket says 172.21.15.32:46772) 2024-04-25T00:30:55.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.591+0000 7fa61b7fe640 1 -- 172.21.15.32:0/1926631139 learned_addr learned my addr 172.21.15.32:0/1926631139 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:55.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.591+0000 7fa61affd640 1 --2- 172.21.15.32:0/1926631139 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5606dc102440 0x5606dc111fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:55.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.592+0000 7fa61b7fe640 1 -- 172.21.15.32:0/1926631139 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5606dc107fa0 msgr2=0x5606dc112a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:30:55.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.592+0000 7fa61b7fe640 1 --2- 172.21.15.32:0/1926631139 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5606dc107fa0 0x5606dc112a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.592+0000 7fa61b7fe640 1 -- 172.21.15.32:0/1926631139 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5606dc102440 msgr2=0x5606dc111fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:30:55.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.592+0000 7fa61b7fe640 1 --2- 172.21.15.32:0/1926631139 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5606dc102440 0x5606dc111fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.592+0000 7fa61b7fe640 1 -- 172.21.15.32:0/1926631139 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5606dc1131c0 con 0x5606dc102e30 2024-04-25T00:30:55.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.592+0000 7fa61b7fe640 1 --2- 172.21.15.32:0/1926631139 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102e30 0x5606dc1124f0 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7fa60c005ec0 tx=0x7fa60c032e60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=71b5425e9f5d47b2 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:55.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.592+0000 7fa5faffd640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:30:55.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.592+0000 7fa5faffd640 1 -- 172.21.15.32:0/1926631139 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa60c040070 con 0x5606dc102e30 2024-04-25T00:30:55.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.592+0000 7fa5faffd640 1 -- 172.21.15.32:0/1926631139 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7fa60c0318c0 con 0x5606dc102e30 2024-04-25T00:30:55.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.592+0000 7fa624816740 1 -- 172.21.15.32:0/1926631139 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa610020250 con 0x5606dc102e30 2024-04-25T00:30:55.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.592+0000 7fa5faffd640 1 -- 172.21.15.32:0/1926631139 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=17) ==== 24+0+0 (secure 0 0 0) 0x7fa60c003fc0 con 0x5606dc102e30 2024-04-25T00:30:55.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.592+0000 7fa624816740 10 client.14787 fetch_fsmap learned FSMap version 17 2024-04-25T00:30:55.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.592+0000 7fa624816740 1 -- 172.21.15.32:0/1926631139 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=17,monmap=4+}) -- 0x5606dc17f1e0 con 0x5606dc102e30 2024-04-25T00:30:55.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.592+0000 7fa5faffd640 1 -- 172.21.15.32:0/1926631139 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa610020250 con 0x5606dc102e30 2024-04-25T00:30:55.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.592+0000 7fa5faffd640 1 -- 172.21.15.32:0/1926631139 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 17) ==== 8342+0+0 (secure 0 0 0) 0x7fa60c003680 con 0x5606dc102e30 2024-04-25T00:30:55.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.592+0000 7fa624816740 10 client.14787 fetch_fsmap finished waiting for FSMap version 17 2024-04-25T00:30:55.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.592+0000 7fa624816740 10 client.14787 resolve_mds: resolved name 'b' to daemon mds.b 2024-04-25T00:30:55.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.592+0000 7fa624816740 20 client.14787 populate_metadata read hostname 'smithi032' 2024-04-25T00:30:55.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.593+0000 7fa624816740 1 --2- 172.21.15.32:0/1926631139 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5606dc183d70 0x5606dc182a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:55.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.593+0000 7fa624816740 4 client.14787 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "status"}] 2024-04-25T00:30:55.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.593+0000 7fa624816740 1 -- 172.21.15.32:0/1926631139 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "status"}) -- 0x5606dc06d790 con 0x5606dc183d70 2024-04-25T00:30:55.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.593+0000 7fa61affd640 1 --2- 172.21.15.32:0/1926631139 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5606dc183d70 0x5606dc182a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:55.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.593+0000 7fa5faffd640 10 client.14787 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:30:55.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.593+0000 7fa61affd640 1 --2- 172.21.15.32:0/1926631139 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5606dc183d70 0x5606dc182a80 crc :-1 s=READY pgs=14 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a9f6068f067bcddd server_cookie=b2e0d83543a4a4a4 in_seq=0 out_seq=0 2024-04-25T00:30:55.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.593+0000 7fa5faffd640 1 -- 172.21.15.32:0/1926631139 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+326 (crc 0 0 0) 0x5606dc06d790 con 0x5606dc183d70 2024-04-25T00:30:55.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.593+0000 7fa5faffd640 10 client.14787 handle_command_reply: tid=0 2024-04-25T00:30:55.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.593+0000 7fa624816740 1 client.14787 shutdown 2024-04-25T00:30:55.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.593+0000 7fa624816740 1 -- 172.21.15.32:0/1926631139 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102e30 msgr2=0x5606dc1124f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:55.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.593+0000 7fa624816740 1 --2- 172.21.15.32:0/1926631139 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102e30 0x5606dc1124f0 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7fa60c005ec0 tx=0x7fa60c032e60 comp rx=0 tx=0).stop 2024-04-25T00:30:55.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.593+0000 7fa624816740 1 -- 172.21.15.32:0/1926631139 shutdown_connections 2024-04-25T00:30:55.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.593+0000 7fa624816740 1 --2- 172.21.15.32:0/1926631139 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5606dc183d70 0x5606dc182a80 crc :-1 s=READY pgs=14 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.593+0000 7fa624816740 1 --2- 172.21.15.32:0/1926631139 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5606dc107fa0 0x5606dc112a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.594+0000 7fa624816740 1 --2- 172.21.15.32:0/1926631139 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5606dc102e30 0x5606dc1124f0 unknown :-1 s=CLOSED pgs=226 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.594+0000 7fa5faffd640 0 client.14787 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:30:55.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.594+0000 7fa624816740 1 --2- 172.21.15.32:0/1926631139 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5606dc102440 0x5606dc111fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.594+0000 7fa624816740 1 -- 172.21.15.32:0/1926631139 >> 172.21.15.32:0/1926631139 conn(0x5606dc0fdab0 msgr2=0x5606dc1041f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:55.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.594+0000 7fa624816740 1 -- 172.21.15.32:0/1926631139 shutdown_connections 2024-04-25T00:30:55.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.594+0000 7fa624816740 1 -- 172.21.15.32:0/1926631139 wait complete. 2024-04-25T00:30:55.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.594+0000 7fa624816740 20 client.14787 trim_cache size 0 max 16384 2024-04-25T00:30:55.602 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:30:55.602 INFO:teuthology.orchestra.run.smithi032.stdout: "cluster_fsid": "cb997156-0299-11ef-bc93-c7b262605968", 2024-04-25T00:30:55.603 INFO:teuthology.orchestra.run.smithi032.stdout: "whoami": 0, 2024-04-25T00:30:55.603 INFO:teuthology.orchestra.run.smithi032.stdout: "id": 24506, 2024-04-25T00:30:55.603 INFO:teuthology.orchestra.run.smithi032.stdout: "want_state": "up:active", 2024-04-25T00:30:55.603 INFO:teuthology.orchestra.run.smithi032.stdout: "state": "up:active", 2024-04-25T00:30:55.603 INFO:teuthology.orchestra.run.smithi032.stdout: "fs_name": "cephfs", 2024-04-25T00:30:55.603 INFO:teuthology.orchestra.run.smithi032.stdout: "rank_uptime": 6.8593082919999997, 2024-04-25T00:30:55.603 INFO:teuthology.orchestra.run.smithi032.stdout: "mdsmap_epoch": 17, 2024-04-25T00:30:55.603 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_epoch": 89, 2024-04-25T00:30:55.603 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_epoch_barrier": 89, 2024-04-25T00:30:55.603 INFO:teuthology.orchestra.run.smithi032.stdout: "uptime": 13.265845847 2024-04-25T00:30:55.603 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:30:55.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.594+0000 7fa621b12640 1 -- 172.21.15.32:0/2833964221 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa6040a2b90 msgr2=0x7fa6040a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:55.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.594+0000 7fa621b12640 1 --2- 172.21.15.32:0/2833964221 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa6040a2b90 0x7fa6040a5050 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7fa6140cc7f0 tx=0x7fa614086040 comp rx=0 tx=0).stop 2024-04-25T00:30:55.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.594+0000 7fa621b12640 1 -- 172.21.15.32:0/2833964221 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa61c107540 msgr2=0x7fa61c1007d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:55.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.594+0000 7fa621b12640 1 --2- 172.21.15.32:0/2833964221 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa61c107540 0x7fa61c1007d0 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7fa614009fd0 tx=0x7fa61400b260 comp rx=0 tx=0).stop 2024-04-25T00:30:55.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.594+0000 7fa621b12640 1 -- 172.21.15.32:0/2833964221 shutdown_connections 2024-04-25T00:30:55.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.594+0000 7fa621b12640 1 --2- 172.21.15.32:0/2833964221 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa6040a2b90 0x7fa6040a5050 unknown :-1 s=CLOSED pgs=162 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.594+0000 7fa621b12640 1 --2- 172.21.15.32:0/2833964221 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa61c107540 0x7fa61c1007d0 unknown :-1 s=CLOSED pgs=222 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.594+0000 7fa621b12640 1 --2- 172.21.15.32:0/2833964221 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa61c104c10 0x7fa61c100290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.594+0000 7fa621b12640 1 --2- 172.21.15.32:0/2833964221 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa61c1022e0 0x7fa61c101c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:55.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.594+0000 7fa621b12640 1 -- 172.21.15.32:0/2833964221 >> 172.21.15.32:0/2833964221 conn(0x7fa61c0fbc30 msgr2=0x7fa61c102da0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:55.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.594+0000 7fa621b12640 1 -- 172.21.15.32:0/2833964221 shutdown_connections 2024-04-25T00:30:55.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:55.594+0000 7fa621b12640 1 -- 172.21.15.32:0/2833964221 wait complete. 2024-04-25T00:30:55.607 INFO:tasks.cephfs.mount:cephfs_mntpt = None 2024-04-25T00:30:55.607 INFO:tasks.cephfs.mount:hostfs_mntpt = /home/ubuntu/cephtest/mnt.admin 2024-04-25T00:30:55.607 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph fs ls 2024-04-25T00:30:55.851 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:30:56.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:55 smithi121 ceph-mon[32231]: pgmap v295: 129 pgs: 105 active+clean, 18 creating+peering, 6 unknown; 579 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s wr, 5 op/s 2024-04-25T00:30:56.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:55 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1533636337' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:56.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:55 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1039545753' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:30:56.133 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:55 smithi032 ceph-mon[29357]: pgmap v295: 129 pgs: 105 active+clean, 18 creating+peering, 6 unknown; 579 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s wr, 5 op/s 2024-04-25T00:30:56.134 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:55 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1533636337' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:56.134 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:55 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1039545753' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:30:56.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:55 smithi130 ceph-mon[32525]: pgmap v295: 129 pgs: 105 active+clean, 18 creating+peering, 6 unknown; 579 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s wr, 5 op/s 2024-04-25T00:30:56.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:55 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1533636337' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:30:56.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:55 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1039545753' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:30:57.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:56 smithi121 ceph-mon[32231]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-25T00:30:57.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:56 smithi121 ceph-mon[32231]: Cluster is now healthy 2024-04-25T00:30:57.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:56 smithi130 ceph-mon[32525]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-25T00:30:57.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:56 smithi130 ceph-mon[32525]: Cluster is now healthy 2024-04-25T00:30:57.179 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:56 smithi032 ceph-mon[29357]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-25T00:30:57.179 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:56 smithi032 ceph-mon[29357]: Cluster is now healthy 2024-04-25T00:30:57.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.756+0000 7f5ae2fbf700 1 -- 172.21.15.32:0/2946294221 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5adc107570 msgr2=0x7f5adc10f140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:57.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.756+0000 7f5ae2fbf700 1 --2- 172.21.15.32:0/2946294221 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5adc107570 0x7f5adc10f140 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7f5ad400b0a0 tx=0x7f5ad400b3b0 comp rx=0 tx=0).stop 2024-04-25T00:30:57.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.756+0000 7f5ae2fbf700 1 -- 172.21.15.32:0/2946294221 shutdown_connections 2024-04-25T00:30:57.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.756+0000 7f5ae2fbf700 1 --2- 172.21.15.32:0/2946294221 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5adc107570 0x7f5adc10f140 unknown :-1 s=CLOSED pgs=227 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:57.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.756+0000 7f5ae2fbf700 1 --2- 172.21.15.32:0/2946294221 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5adc1024e0 0x7f5adc1028c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:57.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.756+0000 7f5ae2fbf700 1 --2- 172.21.15.32:0/2946294221 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5adc102e90 0x7f5adc106e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:57.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.757+0000 7f5ae2fbf700 1 -- 172.21.15.32:0/2946294221 >> 172.21.15.32:0/2946294221 conn(0x7f5adc079390 msgr2=0x7f5adc0797a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:57.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.757+0000 7f5ae2fbf700 1 -- 172.21.15.32:0/2946294221 shutdown_connections 2024-04-25T00:30:57.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.757+0000 7f5ae2fbf700 1 -- 172.21.15.32:0/2946294221 wait complete. 2024-04-25T00:30:57.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.757+0000 7f5ae2fbf700 1 Processor -- start 2024-04-25T00:30:57.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.757+0000 7f5ae2fbf700 1 -- start start 2024-04-25T00:30:57.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.757+0000 7f5ae2fbf700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5adc1024e0 0x7f5adc19e680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:57.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.757+0000 7f5ae2fbf700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5adc102e90 0x7f5adc19ebc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:57.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.757+0000 7f5ae2fbf700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5adc107570 0x7f5adc198850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:57.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.757+0000 7f5ae2fbf700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f5adc1125b0 con 0x7f5adc102e90 2024-04-25T00:30:57.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.757+0000 7f5ae2fbf700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f5adc112430 con 0x7f5adc107570 2024-04-25T00:30:57.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.757+0000 7f5ae2fbf700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f5adc112730 con 0x7f5adc1024e0 2024-04-25T00:30:57.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.758+0000 7f5adbfff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5adc102e90 0x7f5adc19ebc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:57.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.758+0000 7f5adbfff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5adc102e90 0x7f5adc19ebc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46780/0 (socket says 172.21.15.32:46780) 2024-04-25T00:30:57.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.758+0000 7f5adbfff700 1 -- 172.21.15.32:0/2255683060 learned_addr learned my addr 172.21.15.32:0/2255683060 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:30:57.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.758+0000 7f5ae0d5b700 1 --2- 172.21.15.32:0/2255683060 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5adc1024e0 0x7f5adc19e680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:57.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.758+0000 7f5ae155c700 1 --2- 172.21.15.32:0/2255683060 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5adc107570 0x7f5adc198850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:57.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.758+0000 7f5adbfff700 1 -- 172.21.15.32:0/2255683060 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5adc1024e0 msgr2=0x7f5adc19e680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:57.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.758+0000 7f5adbfff700 1 --2- 172.21.15.32:0/2255683060 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5adc1024e0 0x7f5adc19e680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:57.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.758+0000 7f5adbfff700 1 -- 172.21.15.32:0/2255683060 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5adc107570 msgr2=0x7f5adc198850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:57.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.758+0000 7f5adbfff700 1 --2- 172.21.15.32:0/2255683060 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5adc107570 0x7f5adc198850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:57.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.758+0000 7f5adbfff700 1 -- 172.21.15.32:0/2255683060 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5adc199030 con 0x7f5adc102e90 2024-04-25T00:30:57.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.758+0000 7f5adbfff700 1 --2- 172.21.15.32:0/2255683060 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5adc102e90 0x7f5adc19ebc0 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f5ad000d980 tx=0x7f5ad000dc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:57.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.758+0000 7f5ad9ffb700 1 -- 172.21.15.32:0/2255683060 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ad0017070 con 0x7f5adc102e90 2024-04-25T00:30:57.763 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.758+0000 7f5ad9ffb700 1 -- 172.21.15.32:0/2255683060 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5ad0009e10 con 0x7f5adc102e90 2024-04-25T00:30:57.763 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.758+0000 7f5ae2fbf700 1 -- 172.21.15.32:0/2255683060 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5adc199320 con 0x7f5adc102e90 2024-04-25T00:30:57.763 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.758+0000 7f5ad9ffb700 1 -- 172.21.15.32:0/2255683060 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ad000f620 con 0x7f5adc102e90 2024-04-25T00:30:57.763 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.759+0000 7f5ae2fbf700 1 -- 172.21.15.32:0/2255683060 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5adc1a54e0 con 0x7f5adc102e90 2024-04-25T00:30:57.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.760+0000 7f5ae2fbf700 1 -- 172.21.15.32:0/2255683060 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5adc103970 con 0x7f5adc102e90 2024-04-25T00:30:57.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.761+0000 7f5ad9ffb700 1 -- 172.21.15.32:0/2255683060 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f5ad0010460 con 0x7f5adc102e90 2024-04-25T00:30:57.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.761+0000 7f5ad9ffb700 1 --2- 172.21.15.32:0/2255683060 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f5ac40a2c20 0x7f5ac40a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:30:57.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.761+0000 7f5ae0d5b700 1 --2- 172.21.15.32:0/2255683060 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f5ac40a2c20 0x7f5ac40a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:30:57.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.761+0000 7f5ad9ffb700 1 -- 172.21.15.32:0/2255683060 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f5ad00cedb0 con 0x7f5adc102e90 2024-04-25T00:30:57.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.762+0000 7f5ae0d5b700 1 --2- 172.21.15.32:0/2255683060 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f5ac40a2c20 0x7f5ac40a50e0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f5acc009a10 tx=0x7f5acc009190 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:30:57.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:57.767+0000 7f5ad9ffb700 1 -- 172.21.15.32:0/2255683060 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f5ad009bad0 con 0x7f5adc102e90 2024-04-25T00:30:58.019 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:58.018+0000 7f5ae2fbf700 1 -- 172.21.15.32:0/2255683060 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs ls"} v 0) -- 0x7f5adc199d30 con 0x7f5adc102e90 2024-04-25T00:30:58.201 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:57 smithi032 ceph-mon[29357]: pgmap v296: 129 pgs: 111 active+clean, 18 creating+peering; 579 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.2 KiB/s wr, 3 op/s 2024-04-25T00:30:58.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:58.018+0000 7f5ad9ffb700 1 -- 172.21.15.32:0/2255683060 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v17) ==== 53+0+88 (secure 0 0 0) 0x7f5ad009b3d0 con 0x7f5adc102e90 2024-04-25T00:30:58.201 INFO:teuthology.orchestra.run.smithi032.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data cephfs_data_ec ] 2024-04-25T00:30:58.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:58.020+0000 7f5ae2fbf700 1 -- 172.21.15.32:0/2255683060 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f5ac40a2c20 msgr2=0x7f5ac40a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:58.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:58.020+0000 7f5ae2fbf700 1 --2- 172.21.15.32:0/2255683060 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f5ac40a2c20 0x7f5ac40a50e0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f5acc009a10 tx=0x7f5acc009190 comp rx=0 tx=0).stop 2024-04-25T00:30:58.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:58.020+0000 7f5ae2fbf700 1 -- 172.21.15.32:0/2255683060 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5adc102e90 msgr2=0x7f5adc19ebc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:30:58.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:58.020+0000 7f5ae2fbf700 1 --2- 172.21.15.32:0/2255683060 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5adc102e90 0x7f5adc19ebc0 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f5ad000d980 tx=0x7f5ad000dc90 comp rx=0 tx=0).stop 2024-04-25T00:30:58.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:58.020+0000 7f5ae2fbf700 1 -- 172.21.15.32:0/2255683060 shutdown_connections 2024-04-25T00:30:58.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:58.020+0000 7f5ae2fbf700 1 --2- 172.21.15.32:0/2255683060 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f5ac40a2c20 0x7f5ac40a50e0 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:58.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:58.020+0000 7f5ae2fbf700 1 --2- 172.21.15.32:0/2255683060 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5adc107570 0x7f5adc198850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:58.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:58.020+0000 7f5ae2fbf700 1 --2- 172.21.15.32:0/2255683060 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5adc1024e0 0x7f5adc19e680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:58.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:58.020+0000 7f5ae2fbf700 1 --2- 172.21.15.32:0/2255683060 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5adc102e90 0x7f5adc19ebc0 unknown :-1 s=CLOSED pgs=228 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:30:58.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:58.020+0000 7f5ae2fbf700 1 -- 172.21.15.32:0/2255683060 >> 172.21.15.32:0/2255683060 conn(0x7f5adc079390 msgr2=0x7f5adc1002d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:30:58.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:58.021+0000 7f5ae2fbf700 1 -- 172.21.15.32:0/2255683060 shutdown_connections 2024-04-25T00:30:58.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:58.021+0000 7f5ae2fbf700 1 -- 172.21.15.32:0/2255683060 wait complete. 2024-04-25T00:30:58.203 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:57 smithi130 ceph-mon[32525]: pgmap v296: 129 pgs: 111 active+clean, 18 creating+peering; 579 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.2 KiB/s wr, 3 op/s 2024-04-25T00:30:58.204 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:57 smithi121 ceph-mon[32231]: pgmap v296: 129 pgs: 111 active+clean, 18 creating+peering; 579 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.2 KiB/s wr, 3 op/s 2024-04-25T00:30:58.709 INFO:tasks.cephfs.mount:Mounting Ceph FS cephfs; just confirmed its presence on cluster 2024-04-25T00:30:58.709 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-04-25T00:30:58.709 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi032.front.sepia.ceph.com 2024-04-25T00:30:58.709 INFO:tasks.cephfs.mount:self.client.name = client.admin 2024-04-25T00:30:58.709 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.admin 2024-04-25T00:30:58.709 INFO:tasks.cephfs.mount:self.cephfs_name = cephfs 2024-04-25T00:30:58.709 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-04-25T00:30:58.709 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-04-25T00:30:58.710 INFO:tasks.cephfs.mount:Setting the 'None' netns for '/home/ubuntu/cephtest/mnt.admin' 2024-04-25T00:30:58.710 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:30:58.710 DEBUG:teuthology.orchestra.run.smithi032:> ip addr 2024-04-25T00:30:58.727 INFO:teuthology.orchestra.run.smithi032.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-04-25T00:30:58.727 INFO:teuthology.orchestra.run.smithi032.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-04-25T00:30:58.727 INFO:teuthology.orchestra.run.smithi032.stdout: inet 127.0.0.1/8 scope host lo 2024-04-25T00:30:58.727 INFO:teuthology.orchestra.run.smithi032.stdout: valid_lft forever preferred_lft forever 2024-04-25T00:30:58.727 INFO:teuthology.orchestra.run.smithi032.stdout: inet6 ::1/128 scope host 2024-04-25T00:30:58.727 INFO:teuthology.orchestra.run.smithi032.stdout: valid_lft forever preferred_lft forever 2024-04-25T00:30:58.727 INFO:teuthology.orchestra.run.smithi032.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-04-25T00:30:58.727 INFO:teuthology.orchestra.run.smithi032.stdout: link/ether 0c:c4:7a:6b:fd:0c brd ff:ff:ff:ff:ff:ff 2024-04-25T00:30:58.727 INFO:teuthology.orchestra.run.smithi032.stdout: altname enp7s0f0 2024-04-25T00:30:58.727 INFO:teuthology.orchestra.run.smithi032.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-04-25T00:30:58.728 INFO:teuthology.orchestra.run.smithi032.stdout: link/ether 0c:c4:7a:6b:fd:0d brd ff:ff:ff:ff:ff:ff 2024-04-25T00:30:58.728 INFO:teuthology.orchestra.run.smithi032.stdout: altname enp7s0f1 2024-04-25T00:30:58.728 INFO:teuthology.orchestra.run.smithi032.stdout:4: ens1f0: mtu 1500 qdisc mq state UP group default qlen 1000 2024-04-25T00:30:58.728 INFO:teuthology.orchestra.run.smithi032.stdout: link/ether ac:1f:6b:f8:26:02 brd ff:ff:ff:ff:ff:ff 2024-04-25T00:30:58.728 INFO:teuthology.orchestra.run.smithi032.stdout: altname enp1s0f0 2024-04-25T00:30:58.728 INFO:teuthology.orchestra.run.smithi032.stdout: inet 172.21.15.32/20 brd 172.21.15.255 scope global dynamic noprefixroute ens1f0 2024-04-25T00:30:58.728 INFO:teuthology.orchestra.run.smithi032.stdout: valid_lft 42201sec preferred_lft 42201sec 2024-04-25T00:30:58.728 INFO:teuthology.orchestra.run.smithi032.stdout: inet6 fe80::ae1f:6bff:fef8:2602/64 scope link noprefixroute 2024-04-25T00:30:58.728 INFO:teuthology.orchestra.run.smithi032.stdout: valid_lft forever preferred_lft forever 2024-04-25T00:30:58.728 INFO:teuthology.orchestra.run.smithi032.stdout:5: ens1f1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-04-25T00:30:58.728 INFO:teuthology.orchestra.run.smithi032.stdout: link/ether ac:1f:6b:f8:26:03 brd ff:ff:ff:ff:ff:ff 2024-04-25T00:30:58.728 INFO:teuthology.orchestra.run.smithi032.stdout: altname enp1s0f1 2024-04-25T00:30:58.729 INFO:tasks.cephfs.mount:Setuping the 'ceph-brx' with 192.168.255.254/16 2024-04-25T00:30:58.729 DEBUG:teuthology.orchestra.run.smithi032:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-04-25T00:30:58.729 DEBUG:teuthology.orchestra.run.smithi032:> set -e 2024-04-25T00:30:58.729 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip link add name ceph-brx type bridge 2024-04-25T00:30:58.729 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip addr flush dev ceph-brx 2024-04-25T00:30:58.729 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip link set ceph-brx up 2024-04-25T00:30:58.729 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip addr add 192.168.255.254/16 brd 192.168.255.255 dev ceph-brx 2024-04-25T00:30:58.729 DEBUG:teuthology.orchestra.run.smithi032:> ') 2024-04-25T00:30:58.818 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:58 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-04-25T00:30:58.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:58 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-25T00:30:58.916 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:30:58.916 DEBUG:teuthology.orchestra.run.smithi032:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-04-25T00:30:58.930 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:58 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2255683060' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2024-04-25T00:30:58.942 INFO:teuthology.orchestra.run.smithi032.stdout:1 2024-04-25T00:30:58.943 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:30:58.943 DEBUG:teuthology.orchestra.run.smithi032:> route 2024-04-25T00:30:59.015 INFO:teuthology.orchestra.run.smithi032.stdout:Kernel IP routing table 2024-04-25T00:30:59.015 INFO:teuthology.orchestra.run.smithi032.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-04-25T00:30:59.015 INFO:teuthology.orchestra.run.smithi032.stdout:default _gateway 0.0.0.0 UG 100 0 0 ens1f0 2024-04-25T00:30:59.016 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 ens1f0 2024-04-25T00:30:59.016 INFO:teuthology.orchestra.run.smithi032.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-04-25T00:30:59.016 DEBUG:teuthology.orchestra.run.smithi032:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-04-25T00:30:59.016 DEBUG:teuthology.orchestra.run.smithi032:> set -e 2024-04-25T00:30:59.016 DEBUG:teuthology.orchestra.run.smithi032:> sudo iptables -A FORWARD -o ens1f0 -i ceph-brx -j ACCEPT 2024-04-25T00:30:59.016 DEBUG:teuthology.orchestra.run.smithi032:> sudo iptables -A FORWARD -i ens1f0 -o ceph-brx -j ACCEPT 2024-04-25T00:30:59.016 DEBUG:teuthology.orchestra.run.smithi032:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o ens1f0 -j MASQUERADE 2024-04-25T00:30:59.016 DEBUG:teuthology.orchestra.run.smithi032:> ') 2024-04-25T00:30:59.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:58 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2255683060' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2024-04-25T00:30:59.104 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:59 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-04-25T00:30:59.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:58 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2255683060' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2024-04-25T00:30:59.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:59 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-25T00:30:59.286 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:30:59.286 DEBUG:teuthology.orchestra.run.smithi032:> ip netns list 2024-04-25T00:30:59.301 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:30:59.301 DEBUG:teuthology.orchestra.run.smithi032:> ip netns list-id 2024-04-25T00:30:59.356 DEBUG:teuthology.orchestra.run.smithi032:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-04-25T00:30:59.357 DEBUG:teuthology.orchestra.run.smithi032:> set -e 2024-04-25T00:30:59.357 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip netns add ceph-ns--home-ubuntu-cephtest-mnt.admin 2024-04-25T00:30:59.357 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip netns set ceph-ns--home-ubuntu-cephtest-mnt.admin 0 2024-04-25T00:30:59.357 DEBUG:teuthology.orchestra.run.smithi032:> ') 2024-04-25T00:30:59.445 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:59 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-04-25T00:30:59.468 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:59 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-25T00:30:59.474 INFO:tasks.cephfs.mount:Setuping the netns 'ceph-ns--home-ubuntu-cephtest-mnt.admin' with 192.168.0.1/16 2024-04-25T00:30:59.475 DEBUG:teuthology.orchestra.run.smithi032:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-04-25T00:30:59.475 DEBUG:teuthology.orchestra.run.smithi032:> set -e 2024-04-25T00:30:59.475 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip link add veth0 netns ceph-ns--home-ubuntu-cephtest-mnt.admin type veth peer name brx.0 2024-04-25T00:30:59.475 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.admin ip addr add 192.168.0.1/16 brd 192.168.255.255 dev veth0 2024-04-25T00:30:59.475 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.admin ip link set veth0 up 2024-04-25T00:30:59.475 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.admin ip link set lo up 2024-04-25T00:30:59.475 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.admin ip route add default via 192.168.255.254 2024-04-25T00:30:59.475 DEBUG:teuthology.orchestra.run.smithi032:> ') 2024-04-25T00:30:59.525 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:59 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-04-25T00:30:59.594 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:59 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-25T00:30:59.600 DEBUG:teuthology.orchestra.run.smithi032:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-04-25T00:30:59.600 DEBUG:teuthology.orchestra.run.smithi032:> set -e 2024-04-25T00:30:59.601 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip link set brx.0 up 2024-04-25T00:30:59.601 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip link set dev brx.0 master ceph-brx 2024-04-25T00:30:59.601 DEBUG:teuthology.orchestra.run.smithi032:> ') 2024-04-25T00:30:59.650 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:59 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-04-25T00:30:59.672 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:30:59 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-25T00:30:59.679 INFO:tasks.cephfs.fuse_mount:Client client.admin config is {} 2024-04-25T00:30:59.679 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-04-25T00:30:59.679 DEBUG:teuthology.orchestra.run.smithi032:> mkdir -p -v /home/ubuntu/cephtest/mnt.admin 2024-04-25T00:30:59.694 INFO:teuthology.orchestra.run.smithi032.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.admin' 2024-04-25T00:30:59.694 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-04-25T00:30:59.694 DEBUG:teuthology.orchestra.run.smithi032:> chmod 0000 /home/ubuntu/cephtest/mnt.admin 2024-04-25T00:30:59.749 DEBUG:teuthology.orchestra.run.smithi032:> sudo modprobe fuse 2024-04-25T00:30:59.814 DEBUG:teuthology.orchestra.run.smithi032:> cat /proc/self/mounts | awk '{print $2}' 2024-04-25T00:30:59.870 INFO:teuthology.orchestra.run.smithi032.stdout:/proc 2024-04-25T00:30:59.870 INFO:teuthology.orchestra.run.smithi032.stdout:/sys 2024-04-25T00:30:59.870 INFO:teuthology.orchestra.run.smithi032.stdout:/dev 2024-04-25T00:30:59.870 INFO:teuthology.orchestra.run.smithi032.stdout:/sys/kernel/security 2024-04-25T00:30:59.870 INFO:teuthology.orchestra.run.smithi032.stdout:/dev/shm 2024-04-25T00:30:59.871 INFO:teuthology.orchestra.run.smithi032.stdout:/dev/pts 2024-04-25T00:30:59.871 INFO:teuthology.orchestra.run.smithi032.stdout:/run 2024-04-25T00:30:59.871 INFO:teuthology.orchestra.run.smithi032.stdout:/sys/fs/cgroup 2024-04-25T00:30:59.871 INFO:teuthology.orchestra.run.smithi032.stdout:/sys/fs/pstore 2024-04-25T00:30:59.871 INFO:teuthology.orchestra.run.smithi032.stdout:/sys/fs/bpf 2024-04-25T00:30:59.871 INFO:teuthology.orchestra.run.smithi032.stdout:/ 2024-04-25T00:30:59.871 INFO:teuthology.orchestra.run.smithi032.stdout:/sys/fs/selinux 2024-04-25T00:30:59.871 INFO:teuthology.orchestra.run.smithi032.stdout:/proc/sys/fs/binfmt_misc 2024-04-25T00:30:59.871 INFO:teuthology.orchestra.run.smithi032.stdout:/dev/hugepages 2024-04-25T00:30:59.871 INFO:teuthology.orchestra.run.smithi032.stdout:/dev/mqueue 2024-04-25T00:30:59.871 INFO:teuthology.orchestra.run.smithi032.stdout:/sys/kernel/debug 2024-04-25T00:30:59.871 INFO:teuthology.orchestra.run.smithi032.stdout:/sys/kernel/tracing 2024-04-25T00:30:59.871 INFO:teuthology.orchestra.run.smithi032.stdout:/sys/kernel/config 2024-04-25T00:30:59.871 INFO:teuthology.orchestra.run.smithi032.stdout:/run/credentials/systemd-sysusers.service 2024-04-25T00:30:59.872 INFO:teuthology.orchestra.run.smithi032.stdout:/sys/fs/fuse/connections 2024-04-25T00:30:59.872 INFO:teuthology.orchestra.run.smithi032.stdout:/run/credentials/systemd-sysctl.service 2024-04-25T00:30:59.872 INFO:teuthology.orchestra.run.smithi032.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-04-25T00:30:59.872 INFO:teuthology.orchestra.run.smithi032.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-04-25T00:30:59.872 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/nfs/rpc_pipefs 2024-04-25T00:30:59.872 INFO:teuthology.orchestra.run.smithi032.stdout:/run/user/1000 2024-04-25T00:30:59.872 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/ceph 2024-04-25T00:30:59.872 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/containers/storage/overlay 2024-04-25T00:30:59.872 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/containers/storage/overlay/34155947345d5e1279ddfaff075403f9c08260a5fc5ab9972fd1f762a12420da/merged 2024-04-25T00:30:59.872 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/containers/storage/overlay/5ec22dd33e87923e6628917767aab474f46515b93275f46511f7969afe245636/merged 2024-04-25T00:30:59.872 INFO:teuthology.orchestra.run.smithi032.stdout:/run/user/0 2024-04-25T00:30:59.872 INFO:teuthology.orchestra.run.smithi032.stdout:/proc/sys/fs/binfmt_misc 2024-04-25T00:30:59.872 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/containers/storage/overlay/16272f03717e283e4c165a9667c2d715b6be7134b2abb1f4bb581d16d317c39c/merged 2024-04-25T00:30:59.872 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/containers/storage/overlay/b334735334bcc68b2451d311c5255685794a3f6606fab96a8e1bcbd4ae59064d/merged 2024-04-25T00:30:59.873 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/containers/storage/overlay/d0f8d7db666f4cd21acee5182c8f8fb8f6acfbb902e8bf267047ba30de52aa79/merged 2024-04-25T00:30:59.873 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/containers/storage/overlay/092d3e3c11700cbab46f6e752060078cc9e7b6f24911398ce163290571048a0d/merged 2024-04-25T00:30:59.873 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/containers/storage/overlay/564269b61a46d927f7753189d846a278374e70b4d55e1db4fec3a1a6dee9032b/merged 2024-04-25T00:30:59.873 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/containers/storage/overlay/3e4c6be55aa7f47d463d2b46969db01be7907349b6bed80e2307ea969e868b71/merged 2024-04-25T00:30:59.873 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/containers/storage/overlay/1478e67e3c70fb5971457aac4f57e040e6364153eaa3cd74ed348a42546213f7/merged 2024-04-25T00:30:59.873 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/containers/storage/overlay/1b758a1eadfa81ab2a2105f4bf50a80b68784c080304d78e93e1c871d4e450b2/merged 2024-04-25T00:30:59.873 INFO:teuthology.orchestra.run.smithi032.stdout:/run/netns 2024-04-25T00:30:59.873 INFO:teuthology.orchestra.run.smithi032.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.admin 2024-04-25T00:30:59.873 INFO:teuthology.orchestra.run.smithi032.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.admin 2024-04-25T00:30:59.873 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:30:59.873 DEBUG:teuthology.orchestra.run.smithi032:> ls /sys/fs/fuse/connections 2024-04-25T00:30:59.926 INFO:tasks.cephfs.fuse_mount:Pre-mount connections: [] 2024-04-25T00:30:59.926 DEBUG:teuthology.orchestra.run.smithi032:> (cd /home/ubuntu/cephtest && exec sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.admin sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-fuse -f --admin-socket '/var/run/ceph/$cluster-$name.$pid.asok' /home/ubuntu/cephtest/mnt.admin --id admin --client_fs=cephfs) 2024-04-25T00:30:59.969 DEBUG:teuthology.orchestra.run.smithi032:> sudo modprobe fuse 2024-04-25T00:30:59.993 DEBUG:teuthology.orchestra.run.smithi032:> cat /proc/self/mounts | awk '{print $2}' 2024-04-25T00:31:00.050 INFO:teuthology.orchestra.run.smithi032.stdout:/proc 2024-04-25T00:31:00.050 INFO:teuthology.orchestra.run.smithi032.stdout:/sys 2024-04-25T00:31:00.051 INFO:teuthology.orchestra.run.smithi032.stdout:/dev 2024-04-25T00:31:00.051 INFO:teuthology.orchestra.run.smithi032.stdout:/sys/kernel/security 2024-04-25T00:31:00.051 INFO:teuthology.orchestra.run.smithi032.stdout:/dev/shm 2024-04-25T00:31:00.051 INFO:teuthology.orchestra.run.smithi032.stdout:/dev/pts 2024-04-25T00:31:00.051 INFO:teuthology.orchestra.run.smithi032.stdout:/run 2024-04-25T00:31:00.051 INFO:teuthology.orchestra.run.smithi032.stdout:/sys/fs/cgroup 2024-04-25T00:31:00.051 INFO:teuthology.orchestra.run.smithi032.stdout:/sys/fs/pstore 2024-04-25T00:31:00.051 INFO:teuthology.orchestra.run.smithi032.stdout:/sys/fs/bpf 2024-04-25T00:31:00.051 INFO:teuthology.orchestra.run.smithi032.stdout:/ 2024-04-25T00:31:00.051 INFO:teuthology.orchestra.run.smithi032.stdout:/sys/fs/selinux 2024-04-25T00:31:00.051 INFO:teuthology.orchestra.run.smithi032.stdout:/proc/sys/fs/binfmt_misc 2024-04-25T00:31:00.051 INFO:teuthology.orchestra.run.smithi032.stdout:/dev/hugepages 2024-04-25T00:31:00.051 INFO:teuthology.orchestra.run.smithi032.stdout:/dev/mqueue 2024-04-25T00:31:00.051 INFO:teuthology.orchestra.run.smithi032.stdout:/sys/kernel/debug 2024-04-25T00:31:00.052 INFO:teuthology.orchestra.run.smithi032.stdout:/sys/kernel/tracing 2024-04-25T00:31:00.052 INFO:teuthology.orchestra.run.smithi032.stdout:/sys/kernel/config 2024-04-25T00:31:00.052 INFO:teuthology.orchestra.run.smithi032.stdout:/run/credentials/systemd-sysusers.service 2024-04-25T00:31:00.052 INFO:teuthology.orchestra.run.smithi032.stdout:/sys/fs/fuse/connections 2024-04-25T00:31:00.052 INFO:teuthology.orchestra.run.smithi032.stdout:/run/credentials/systemd-sysctl.service 2024-04-25T00:31:00.052 INFO:teuthology.orchestra.run.smithi032.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-04-25T00:31:00.052 INFO:teuthology.orchestra.run.smithi032.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-04-25T00:31:00.052 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/nfs/rpc_pipefs 2024-04-25T00:31:00.052 INFO:teuthology.orchestra.run.smithi032.stdout:/run/user/1000 2024-04-25T00:31:00.052 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/ceph 2024-04-25T00:31:00.052 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/containers/storage/overlay 2024-04-25T00:31:00.052 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/containers/storage/overlay/34155947345d5e1279ddfaff075403f9c08260a5fc5ab9972fd1f762a12420da/merged 2024-04-25T00:31:00.052 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/containers/storage/overlay/5ec22dd33e87923e6628917767aab474f46515b93275f46511f7969afe245636/merged 2024-04-25T00:31:00.052 INFO:teuthology.orchestra.run.smithi032.stdout:/run/user/0 2024-04-25T00:31:00.053 INFO:teuthology.orchestra.run.smithi032.stdout:/proc/sys/fs/binfmt_misc 2024-04-25T00:31:00.053 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/containers/storage/overlay/16272f03717e283e4c165a9667c2d715b6be7134b2abb1f4bb581d16d317c39c/merged 2024-04-25T00:31:00.053 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/containers/storage/overlay/b334735334bcc68b2451d311c5255685794a3f6606fab96a8e1bcbd4ae59064d/merged 2024-04-25T00:31:00.053 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/containers/storage/overlay/d0f8d7db666f4cd21acee5182c8f8fb8f6acfbb902e8bf267047ba30de52aa79/merged 2024-04-25T00:31:00.053 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/containers/storage/overlay/092d3e3c11700cbab46f6e752060078cc9e7b6f24911398ce163290571048a0d/merged 2024-04-25T00:31:00.053 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/containers/storage/overlay/564269b61a46d927f7753189d846a278374e70b4d55e1db4fec3a1a6dee9032b/merged 2024-04-25T00:31:00.053 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/containers/storage/overlay/3e4c6be55aa7f47d463d2b46969db01be7907349b6bed80e2307ea969e868b71/merged 2024-04-25T00:31:00.053 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/containers/storage/overlay/1478e67e3c70fb5971457aac4f57e040e6364153eaa3cd74ed348a42546213f7/merged 2024-04-25T00:31:00.053 INFO:teuthology.orchestra.run.smithi032.stdout:/var/lib/containers/storage/overlay/1b758a1eadfa81ab2a2105f4bf50a80b68784c080304d78e93e1c871d4e450b2/merged 2024-04-25T00:31:00.053 INFO:teuthology.orchestra.run.smithi032.stdout:/run/netns 2024-04-25T00:31:00.053 INFO:teuthology.orchestra.run.smithi032.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.admin 2024-04-25T00:31:00.053 INFO:teuthology.orchestra.run.smithi032.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.admin 2024-04-25T00:31:00.053 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:31:00.054 DEBUG:teuthology.orchestra.run.smithi032:> ls /sys/fs/fuse/connections 2024-04-25T00:31:00.076 INFO:tasks.cephfs.fuse_mount.ceph-fuse.admin.smithi032.stderr:2024-04-25T00:31:00.075+0000 7fd6256e7480 -1 init, newargv = 0x5571fe2291c0 newargc=15 2024-04-25T00:31:00.076 INFO:tasks.cephfs.fuse_mount.ceph-fuse.admin.smithi032.stderr:ceph-fuse[73401]: starting ceph client 2024-04-25T00:31:00.082 INFO:tasks.cephfs.fuse_mount.ceph-fuse.admin.smithi032.stderr:ceph-fuse[73401]: starting fuse 2024-04-25T00:31:00.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:30:59 smithi121 ceph-mon[32231]: pgmap v297: 129 pgs: 129 active+clean; 579 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 1.7 KiB/s wr, 2 op/s 2024-04-25T00:31:00.091 INFO:teuthology.orchestra.run.smithi032.stdout:115 2024-04-25T00:31:00.092 INFO:tasks.cephfs.fuse_mount:Post-mount connections: [115] 2024-04-25T00:31:00.092 DEBUG:teuthology.orchestra.run.smithi032:> sudo stdin-killer -- python3 -c ' 2024-04-25T00:31:00.092 DEBUG:teuthology.orchestra.run.smithi032:> import glob 2024-04-25T00:31:00.092 DEBUG:teuthology.orchestra.run.smithi032:> import re 2024-04-25T00:31:00.092 DEBUG:teuthology.orchestra.run.smithi032:> import os 2024-04-25T00:31:00.092 DEBUG:teuthology.orchestra.run.smithi032:> import subprocess 2024-04-25T00:31:00.092 DEBUG:teuthology.orchestra.run.smithi032:> 2024-04-25T00:31:00.092 DEBUG:teuthology.orchestra.run.smithi032:> def _find_admin_socket(client_name): 2024-04-25T00:31:00.092 DEBUG:teuthology.orchestra.run.smithi032:> asok_path = "/var/run/ceph/ceph-client.admin.*.asok" 2024-04-25T00:31:00.092 DEBUG:teuthology.orchestra.run.smithi032:> files = glob.glob(asok_path) 2024-04-25T00:31:00.092 DEBUG:teuthology.orchestra.run.smithi032:> mountpoint = "/home/ubuntu/cephtest/mnt.admin" 2024-04-25T00:31:00.093 DEBUG:teuthology.orchestra.run.smithi032:> 2024-04-25T00:31:00.093 DEBUG:teuthology.orchestra.run.smithi032:> # Given a non-glob path, it better be there 2024-04-25T00:31:00.093 DEBUG:teuthology.orchestra.run.smithi032:> if "*" not in asok_path: 2024-04-25T00:31:00.093 DEBUG:teuthology.orchestra.run.smithi032:> assert(len(files) == 1) 2024-04-25T00:31:00.093 DEBUG:teuthology.orchestra.run.smithi032:> return files[0] 2024-04-25T00:31:00.093 DEBUG:teuthology.orchestra.run.smithi032:> 2024-04-25T00:31:00.093 DEBUG:teuthology.orchestra.run.smithi032:> for f in files: 2024-04-25T00:31:00.093 DEBUG:teuthology.orchestra.run.smithi032:> pid = re.match(".*\.(\d+)\.asok$", f).group(1) 2024-04-25T00:31:00.093 DEBUG:teuthology.orchestra.run.smithi032:> if os.path.exists("/proc/{0}".format(pid)): 2024-04-25T00:31:00.093 DEBUG:teuthology.orchestra.run.smithi032:> with open("/proc/{0}/cmdline".format(pid), '"'"'r'"'"') as proc_f: 2024-04-25T00:31:00.093 DEBUG:teuthology.orchestra.run.smithi032:> contents = proc_f.read() 2024-04-25T00:31:00.093 DEBUG:teuthology.orchestra.run.smithi032:> if mountpoint in contents: 2024-04-25T00:31:00.093 DEBUG:teuthology.orchestra.run.smithi032:> return f 2024-04-25T00:31:00.093 DEBUG:teuthology.orchestra.run.smithi032:> raise RuntimeError("Client socket {0} not found".format(client_name)) 2024-04-25T00:31:00.093 DEBUG:teuthology.orchestra.run.smithi032:> 2024-04-25T00:31:00.094 DEBUG:teuthology.orchestra.run.smithi032:> print(_find_admin_socket("client.admin")) 2024-04-25T00:31:00.094 DEBUG:teuthology.orchestra.run.smithi032:> ' 2024-04-25T00:31:00.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:30:59 smithi130 ceph-mon[32525]: pgmap v297: 129 pgs: 129 active+clean; 579 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 1.7 KiB/s wr, 2 op/s 2024-04-25T00:31:00.153 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:30:59 smithi032 ceph-mon[29357]: pgmap v297: 129 pgs: 129 active+clean; 579 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 1.7 KiB/s wr, 2 op/s 2024-04-25T00:31:00.212 INFO:teuthology.orchestra.run.smithi032.stdout:/var/run/ceph/ceph-client.admin.73401.asok 2024-04-25T00:31:00.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-25T00:31:00.223 INFO:tasks.cephfs.fuse_mount:Found client admin socket at /var/run/ceph/ceph-client.admin.73401.asok 2024-04-25T00:31:00.223 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:31:00.223 DEBUG:teuthology.orchestra.run.smithi032:> sudo ceph --admin-daemon /var/run/ceph/ceph-client.admin.73401.asok status 2024-04-25T00:31:00.360 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:31:00.360 INFO:teuthology.orchestra.run.smithi032.stdout: "metadata": { 2024-04-25T00:31:00.360 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph_sha1": "d2c45d977142e024a59b68a8f1dce8f0b9840551", 2024-04-25T00:31:00.360 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph_version": "ceph version 19.0.0-2445-gd2c45d97 (d2c45d977142e024a59b68a8f1dce8f0b9840551) squid (dev)", 2024-04-25T00:31:00.361 INFO:teuthology.orchestra.run.smithi032.stdout: "entity_id": "admin", 2024-04-25T00:31:00.361 INFO:teuthology.orchestra.run.smithi032.stdout: "hostname": "smithi032", 2024-04-25T00:31:00.361 INFO:teuthology.orchestra.run.smithi032.stdout: "mount_point": "/home/ubuntu/cephtest/mnt.admin", 2024-04-25T00:31:00.361 INFO:teuthology.orchestra.run.smithi032.stdout: "pid": "73401", 2024-04-25T00:31:00.361 INFO:teuthology.orchestra.run.smithi032.stdout: "root": "/" 2024-04-25T00:31:00.361 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:31:00.361 INFO:teuthology.orchestra.run.smithi032.stdout: "dentry_count": 0, 2024-04-25T00:31:00.361 INFO:teuthology.orchestra.run.smithi032.stdout: "dentry_pinned_count": 0, 2024-04-25T00:31:00.361 INFO:teuthology.orchestra.run.smithi032.stdout: "id": 14799, 2024-04-25T00:31:00.361 INFO:teuthology.orchestra.run.smithi032.stdout: "inst": { 2024-04-25T00:31:00.361 INFO:teuthology.orchestra.run.smithi032.stdout: "name": { 2024-04-25T00:31:00.361 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "client", 2024-04-25T00:31:00.361 INFO:teuthology.orchestra.run.smithi032.stdout: "num": 14799 2024-04-25T00:31:00.361 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:31:00.361 INFO:teuthology.orchestra.run.smithi032.stdout: "addr": { 2024-04-25T00:31:00.361 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "v1", 2024-04-25T00:31:00.362 INFO:teuthology.orchestra.run.smithi032.stdout: "addr": "192.168.0.1:0", 2024-04-25T00:31:00.362 INFO:teuthology.orchestra.run.smithi032.stdout: "nonce": 2147048959 2024-04-25T00:31:00.362 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:31:00.362 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:31:00.362 INFO:teuthology.orchestra.run.smithi032.stdout: "addr": { 2024-04-25T00:31:00.362 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "v1", 2024-04-25T00:31:00.362 INFO:teuthology.orchestra.run.smithi032.stdout: "addr": "192.168.0.1:0", 2024-04-25T00:31:00.362 INFO:teuthology.orchestra.run.smithi032.stdout: "nonce": 2147048959 2024-04-25T00:31:00.362 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:31:00.362 INFO:teuthology.orchestra.run.smithi032.stdout: "inst_str": "client.14799 192.168.0.1:0/2147048959", 2024-04-25T00:31:00.362 INFO:teuthology.orchestra.run.smithi032.stdout: "addr_str": "192.168.0.1:0/2147048959", 2024-04-25T00:31:00.362 INFO:teuthology.orchestra.run.smithi032.stdout: "inode_count": 1, 2024-04-25T00:31:00.362 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_epoch": 17, 2024-04-25T00:31:00.362 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_epoch": 89, 2024-04-25T00:31:00.362 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_epoch_barrier": 0, 2024-04-25T00:31:00.362 INFO:teuthology.orchestra.run.smithi032.stdout: "blocklisted": false, 2024-04-25T00:31:00.363 INFO:teuthology.orchestra.run.smithi032.stdout: "fs_name": "cephfs" 2024-04-25T00:31:00.363 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:31:00.369 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:31:00.369 DEBUG:teuthology.orchestra.run.smithi032:> stat --file-system '--printf=%T 2024-04-25T00:31:00.369 DEBUG:teuthology.orchestra.run.smithi032:> ' -- /home/ubuntu/cephtest/mnt.admin 2024-04-25T00:31:00.384 INFO:teuthology.orchestra.run.smithi032.stdout:fuseblk 2024-04-25T00:31:00.384 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /home/ubuntu/cephtest/mnt.admin 2024-04-25T00:31:00.385 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:31:00.385 DEBUG:teuthology.orchestra.run.smithi032:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.admin 2024-04-25T00:31:00.450 DEBUG:teuthology.orchestra.run.smithi032:> (cd /home/ubuntu/cephtest/mnt.admin && exec stdin-killer --timeout=900 -- bash -c 'setfattr -n ceph.dir.layout.pool -v cephfs_data_ec . && getfattr -n ceph.dir.layout .') 2024-04-25T00:31:00.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00 stdin-killer INFO: expiration expected; waiting 900 seconds for command to complete 2024-04-25T00:31:00.556 INFO:teuthology.orchestra.run.smithi032.stdout:# file: . 2024-04-25T00:31:00.556 INFO:teuthology.orchestra.run.smithi032.stdout:ceph.dir.layout="stripe_unit=4194304 stripe_count=1 object_size=4194304 pool=cephfs_data_ec" 2024-04-25T00:31:00.557 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:31:00.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-25T00:31:00.563 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-04-25T00:31:00.563 DEBUG:teuthology.orchestra.run.smithi032:> dd if=/proc/self/mounts of=/dev/stdout 2024-04-25T00:31:00.579 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-25T00:31:00.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.719+0000 7f1359942640 1 -- 172.21.15.32:0/3190988055 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1354104bb0 msgr2=0x7f1354106fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:00.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.719+0000 7f1359942640 1 --2- 172.21.15.32:0/3190988055 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1354104bb0 0x7f1354106fa0 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f133c009900 tx=0x7f133c02a410 comp rx=0 tx=0).stop 2024-04-25T00:31:00.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.719+0000 7f1359942640 1 -- 172.21.15.32:0/3190988055 shutdown_connections 2024-04-25T00:31:00.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.719+0000 7f1359942640 1 --2- 172.21.15.32:0/3190988055 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f13541074e0 0x7f13541098d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:00.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.719+0000 7f1359942640 1 --2- 172.21.15.32:0/3190988055 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1354104bb0 0x7f1354106fa0 unknown :-1 s=CLOSED pgs=231 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:00.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.719+0000 7f1359942640 1 --2- 172.21.15.32:0/3190988055 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1354102280 0x7f1354104670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:00.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.719+0000 7f1359942640 1 -- 172.21.15.32:0/3190988055 >> 172.21.15.32:0/3190988055 conn(0x7f13540fbc30 msgr2=0x7f13540fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:00.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.719+0000 7f1359942640 1 -- 172.21.15.32:0/3190988055 shutdown_connections 2024-04-25T00:31:00.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.719+0000 7f1359942640 1 -- 172.21.15.32:0/3190988055 wait complete. 2024-04-25T00:31:00.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.720+0000 7f1359942640 1 Processor -- start 2024-04-25T00:31:00.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.720+0000 7f1359942640 1 -- start start 2024-04-25T00:31:00.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.720+0000 7f1359942640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1354102280 0x7f1354101bf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:00.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.720+0000 7f1359942640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1354104bb0 0x7f1354100230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:00.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.720+0000 7f1359942640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f13541074e0 0x7f1354100770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:00.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.720+0000 7f1359942640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f135410d420 con 0x7f1354102280 2024-04-25T00:31:00.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.720+0000 7f1359942640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f135410d2a0 con 0x7f13541074e0 2024-04-25T00:31:00.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.720+0000 7f1359942640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f135410d5a0 con 0x7f1354104bb0 2024-04-25T00:31:00.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.720+0000 7f1352ffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1354102280 0x7f1354101bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:00.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.720+0000 7f1352ffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1354102280 0x7f1354101bf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46784/0 (socket says 172.21.15.32:46784) 2024-04-25T00:31:00.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.720+0000 7f1352ffd640 1 -- 172.21.15.32:0/3039641119 learned_addr learned my addr 172.21.15.32:0/3039641119 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:00.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.720+0000 7f134a7fc640 1 --2- 172.21.15.32:0/3039641119 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1354104bb0 0x7f1354100230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:00.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.720+0000 7f13537fe640 1 --2- 172.21.15.32:0/3039641119 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f13541074e0 0x7f1354100770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:00.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.720+0000 7f1352ffd640 1 -- 172.21.15.32:0/3039641119 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1354104bb0 msgr2=0x7f1354100230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:00.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.720+0000 7f1352ffd640 1 --2- 172.21.15.32:0/3039641119 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1354104bb0 0x7f1354100230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:00.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.720+0000 7f1352ffd640 1 -- 172.21.15.32:0/3039641119 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f13541074e0 msgr2=0x7f1354100770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:00.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.720+0000 7f1352ffd640 1 --2- 172.21.15.32:0/3039641119 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f13541074e0 0x7f1354100770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:00.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.720+0000 7f1352ffd640 1 -- 172.21.15.32:0/3039641119 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1354101000 con 0x7f1354102280 2024-04-25T00:31:00.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.720+0000 7f13537fe640 1 --2- 172.21.15.32:0/3039641119 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f13541074e0 0x7f1354100770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:31:00.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.721+0000 7f1352ffd640 1 --2- 172.21.15.32:0/3039641119 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1354102280 0x7f1354101bf0 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f134400d950 tx=0x7f134400de10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:00.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.721+0000 7f1358940640 1 -- 172.21.15.32:0/3039641119 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1344017030 con 0x7f1354102280 2024-04-25T00:31:00.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.721+0000 7f1359942640 1 -- 172.21.15.32:0/3039641119 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f13541a1c10 con 0x7f1354102280 2024-04-25T00:31:00.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.721+0000 7f1358940640 1 -- 172.21.15.32:0/3039641119 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1344004020 con 0x7f1354102280 2024-04-25T00:31:00.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.721+0000 7f1358940640 1 -- 172.21.15.32:0/3039641119 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f13440047a0 con 0x7f1354102280 2024-04-25T00:31:00.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.721+0000 7f1359942640 1 -- 172.21.15.32:0/3039641119 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f13541a2130 con 0x7f1354102280 2024-04-25T00:31:00.726 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.722+0000 7f1359942640 1 -- 172.21.15.32:0/3039641119 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f13540689b0 con 0x7f1354102280 2024-04-25T00:31:00.726 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.723+0000 7f1358940640 1 -- 172.21.15.32:0/3039641119 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f13440041d0 con 0x7f1354102280 2024-04-25T00:31:00.726 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.724+0000 7f1358940640 1 --2- 172.21.15.32:0/3039641119 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f13400a2b90 0x7f13400a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:00.727 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.724+0000 7f134a7fc640 1 --2- 172.21.15.32:0/3039641119 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f13400a2b90 0x7f13400a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:00.727 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.724+0000 7f1358940640 1 -- 172.21.15.32:0/3039641119 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f13440cf420 con 0x7f1354102280 2024-04-25T00:31:00.727 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.724+0000 7f134a7fc640 1 --2- 172.21.15.32:0/3039641119 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f13400a2b90 0x7f13400a5050 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f133c005ab0 tx=0x7f133c000bc0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:00.728 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.727+0000 7f1358940640 1 -- 172.21.15.32:0/3039641119 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f134409c0e0 con 0x7f1354102280 2024-04-25T00:31:00.918 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.917+0000 7f1359942640 1 -- 172.21.15.32:0/3039641119 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f1354062370 con 0x7f1354102280 2024-04-25T00:31:00.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.918+0000 7f1358940640 1 -- 172.21.15.32:0/3039641119 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v89) ==== 74+0+24128 (secure 0 0 0) 0x7f134409b9e0 con 0x7f1354102280 2024-04-25T00:31:00.920 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:31:00.920 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":89,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:30:53.134605+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-25T00:30:46.239556+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-25T00:30:47.257182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2024-04-25T00:30:51.193975+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":83,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-num-failure-domains":"0","crush-osds-per-failure-domain":"0","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:31:00.922 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.920+0000 7f1359942640 1 -- 172.21.15.32:0/3039641119 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f13400a2b90 msgr2=0x7f13400a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:00.922 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.920+0000 7f1359942640 1 --2- 172.21.15.32:0/3039641119 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f13400a2b90 0x7f13400a5050 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f133c005ab0 tx=0x7f133c000bc0 comp rx=0 tx=0).stop 2024-04-25T00:31:00.922 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.920+0000 7f1359942640 1 -- 172.21.15.32:0/3039641119 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1354102280 msgr2=0x7f1354101bf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:00.922 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.920+0000 7f1359942640 1 --2- 172.21.15.32:0/3039641119 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1354102280 0x7f1354101bf0 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f134400d950 tx=0x7f134400de10 comp rx=0 tx=0).stop 2024-04-25T00:31:00.922 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.920+0000 7f1359942640 1 -- 172.21.15.32:0/3039641119 shutdown_connections 2024-04-25T00:31:00.922 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.920+0000 7f1359942640 1 --2- 172.21.15.32:0/3039641119 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f13400a2b90 0x7f13400a5050 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:00.922 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.920+0000 7f1359942640 1 --2- 172.21.15.32:0/3039641119 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f13541074e0 0x7f1354100770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:00.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.920+0000 7f1359942640 1 --2- 172.21.15.32:0/3039641119 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1354104bb0 0x7f1354100230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:00.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.920+0000 7f1359942640 1 --2- 172.21.15.32:0/3039641119 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1354102280 0x7f1354101bf0 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:00.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.920+0000 7f1359942640 1 -- 172.21.15.32:0/3039641119 >> 172.21.15.32:0/3039641119 conn(0x7f13540fbc30 msgr2=0x7f1354102be0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:00.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.920+0000 7f1359942640 1 -- 172.21.15.32:0/3039641119 shutdown_connections 2024-04-25T00:31:00.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:00.920+0000 7f1359942640 1 -- 172.21.15.32:0/3039641119 wait complete. 2024-04-25T00:31:00.933 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-25T00:26:07.306839+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-04-25T00:30:46.239556+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '84', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-04-25T00:30:47.257182+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '85', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.05000000074505806}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_data_ec', 'create_time': '2024-04-25T00:30:51.193975+0000', 'flags': 5, 'flags_names': 'hashpspool,ec_overwrites', 'type': 3, 'size': 4, 'min_size': 3, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '89', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'cephfs_data_ec', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 8192, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}] 2024-04-25T00:31:00.934 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-25T00:31:01.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.038+0000 7f823c65d640 1 -- 172.21.15.32:0/1664294314 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8234101790 msgr2=0x7f8234101bb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:01.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.038+0000 7f823c65d640 1 --2- 172.21.15.32:0/1664294314 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8234101790 0x7f8234101bb0 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f8230009900 tx=0x7f823002a410 comp rx=0 tx=0).stop 2024-04-25T00:31:01.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.038+0000 7f823c65d640 1 -- 172.21.15.32:0/1664294314 shutdown_connections 2024-04-25T00:31:01.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.038+0000 7f823c65d640 1 --2- 172.21.15.32:0/1664294314 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8234103340 0x7f8234109bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.038+0000 7f823c65d640 1 --2- 172.21.15.32:0/1664294314 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8234102980 0x7f8234102e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.038+0000 7f823c65d640 1 --2- 172.21.15.32:0/1664294314 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8234101790 0x7f8234101bb0 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.038+0000 7f823c65d640 1 -- 172.21.15.32:0/1664294314 >> 172.21.15.32:0/1664294314 conn(0x7f82340fcd20 msgr2=0x7f82340ff140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:01.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.038+0000 7f823c65d640 1 -- 172.21.15.32:0/1664294314 shutdown_connections 2024-04-25T00:31:01.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.038+0000 7f823c65d640 1 -- 172.21.15.32:0/1664294314 wait complete. 2024-04-25T00:31:01.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.038+0000 7f823c65d640 1 Processor -- start 2024-04-25T00:31:01.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.038+0000 7f823c65d640 1 -- start start 2024-04-25T00:31:01.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.039+0000 7f823c65d640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8234101790 0x7f8234197af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:01.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.039+0000 7f823c65d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8234102980 0x7f8234198030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:01.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.039+0000 7f823c65d640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8234103340 0x7f823419f0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:01.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.039+0000 7f823c65d640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f823410d450 con 0x7f8234102980 2024-04-25T00:31:01.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.039+0000 7f823c65d640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f823410d2d0 con 0x7f8234103340 2024-04-25T00:31:01.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.039+0000 7f823c65d640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f823410d5d0 con 0x7f8234101790 2024-04-25T00:31:01.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.039+0000 7f8239bf8640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8234102980 0x7f8234198030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:01.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.039+0000 7f8239bf8640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8234102980 0x7f8234198030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46800/0 (socket says 172.21.15.32:46800) 2024-04-25T00:31:01.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.039+0000 7f8239bf8640 1 -- 172.21.15.32:0/1817031357 learned_addr learned my addr 172.21.15.32:0/1817031357 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:01.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.039+0000 7f823a3f9640 1 --2- 172.21.15.32:0/1817031357 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8234101790 0x7f8234197af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:01.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.039+0000 7f823abfa640 1 --2- 172.21.15.32:0/1817031357 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8234103340 0x7f823419f0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:01.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.039+0000 7f8239bf8640 1 -- 172.21.15.32:0/1817031357 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8234101790 msgr2=0x7f8234197af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:01.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.039+0000 7f8239bf8640 1 --2- 172.21.15.32:0/1817031357 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8234101790 0x7f8234197af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.039+0000 7f8239bf8640 1 -- 172.21.15.32:0/1817031357 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8234103340 msgr2=0x7f823419f0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:01.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.039+0000 7f8239bf8640 1 --2- 172.21.15.32:0/1817031357 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8234103340 0x7f823419f0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.039+0000 7f8239bf8640 1 -- 172.21.15.32:0/1817031357 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f823419f710 con 0x7f8234102980 2024-04-25T00:31:01.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.039+0000 7f823abfa640 1 --2- 172.21.15.32:0/1817031357 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8234103340 0x7f823419f0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:31:01.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.039+0000 7f8239bf8640 1 --2- 172.21.15.32:0/1817031357 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8234102980 0x7f8234198030 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f822400c970 tx=0x7f822400ce30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:01.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.039+0000 7f822b7fe640 1 -- 172.21.15.32:0/1817031357 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8224007bb0 con 0x7f8234102980 2024-04-25T00:31:01.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.039+0000 7f822b7fe640 1 -- 172.21.15.32:0/1817031357 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8224007d50 con 0x7f8234102980 2024-04-25T00:31:01.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.039+0000 7f823c65d640 1 -- 172.21.15.32:0/1817031357 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f823419fa00 con 0x7f8234102980 2024-04-25T00:31:01.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.039+0000 7f822b7fe640 1 -- 172.21.15.32:0/1817031357 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f822400f450 con 0x7f8234102980 2024-04-25T00:31:01.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.039+0000 7f823c65d640 1 -- 172.21.15.32:0/1817031357 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f823419ff00 con 0x7f8234102980 2024-04-25T00:31:01.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.040+0000 7f823c65d640 1 -- 172.21.15.32:0/1817031357 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8234108b30 con 0x7f8234102980 2024-04-25T00:31:01.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.042+0000 7f822b7fe640 1 -- 172.21.15.32:0/1817031357 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f8224004fe0 con 0x7f8234102980 2024-04-25T00:31:01.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.042+0000 7f822b7fe640 1 --2- 172.21.15.32:0/1817031357 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f82200a2b90 0x7f82200a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:01.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.042+0000 7f823a3f9640 1 --2- 172.21.15.32:0/1817031357 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f82200a2b90 0x7f82200a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:01.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.042+0000 7f822b7fe640 1 -- 172.21.15.32:0/1817031357 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f82240cf400 con 0x7f8234102980 2024-04-25T00:31:01.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.043+0000 7f823a3f9640 1 --2- 172.21.15.32:0/1817031357 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f82200a2b90 0x7f82200a5050 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7f823002aa00 tx=0x7f82300315b0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:01.047 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.046+0000 7f822b7fe640 1 -- 172.21.15.32:0/1817031357 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f8224014030 con 0x7f8234102980 2024-04-25T00:31:01.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.231+0000 7f823c65d640 1 -- 172.21.15.32:0/1817031357 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7f823402d3c0 con 0x7f8234102980 2024-04-25T00:31:01.233 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.231+0000 7f822b7fe640 1 -- 172.21.15.32:0/1817031357 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v89) ==== 93+0+10 (secure 0 0 0) 0x7f822409c120 con 0x7f8234102980 2024-04-25T00:31:01.233 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 1 2024-04-25T00:31:01.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.233+0000 7f823c65d640 1 -- 172.21.15.32:0/1817031357 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f82200a2b90 msgr2=0x7f82200a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:01.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.233+0000 7f823c65d640 1 --2- 172.21.15.32:0/1817031357 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f82200a2b90 0x7f82200a5050 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7f823002aa00 tx=0x7f82300315b0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.233+0000 7f823c65d640 1 -- 172.21.15.32:0/1817031357 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8234102980 msgr2=0x7f8234198030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:01.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.233+0000 7f823c65d640 1 --2- 172.21.15.32:0/1817031357 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8234102980 0x7f8234198030 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f822400c970 tx=0x7f822400ce30 comp rx=0 tx=0).stop 2024-04-25T00:31:01.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.233+0000 7f823c65d640 1 -- 172.21.15.32:0/1817031357 shutdown_connections 2024-04-25T00:31:01.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.233+0000 7f823c65d640 1 --2- 172.21.15.32:0/1817031357 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f82200a2b90 0x7f82200a5050 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.233+0000 7f823c65d640 1 --2- 172.21.15.32:0/1817031357 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8234103340 0x7f823419f0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.233+0000 7f823c65d640 1 --2- 172.21.15.32:0/1817031357 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8234102980 0x7f8234198030 unknown :-1 s=CLOSED pgs=234 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.233+0000 7f823c65d640 1 --2- 172.21.15.32:0/1817031357 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8234101790 0x7f8234197af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.233+0000 7f823c65d640 1 -- 172.21.15.32:0/1817031357 >> 172.21.15.32:0/1817031357 conn(0x7f82340fcd20 msgr2=0x7f8234105d00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:01.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.234+0000 7f823c65d640 1 -- 172.21.15.32:0/1817031357 shutdown_connections 2024-04-25T00:31:01.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.234+0000 7f823c65d640 1 -- 172.21.15.32:0/1817031357 wait complete. 2024-04-25T00:31:01.246 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-04-25T00:31:01.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.350+0000 7f130f712640 1 -- 172.21.15.32:0/1315336723 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1308102230 msgr2=0x7f1308108ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:01.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.350+0000 7f130f712640 1 --2- 172.21.15.32:0/1315336723 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1308102230 0x7f1308108ac0 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f1300009900 tx=0x7f130002a410 comp rx=0 tx=0).stop 2024-04-25T00:31:01.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.350+0000 7f130f712640 1 -- 172.21.15.32:0/1315336723 shutdown_connections 2024-04-25T00:31:01.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.350+0000 7f130f712640 1 --2- 172.21.15.32:0/1315336723 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1308102230 0x7f1308108ac0 unknown :-1 s=CLOSED pgs=235 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.350+0000 7f130f712640 1 --2- 172.21.15.32:0/1315336723 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1308101870 0x7f1308101cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.350+0000 7f130f712640 1 --2- 172.21.15.32:0/1315336723 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1308100680 0x7f1308100aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.350+0000 7f130f712640 1 -- 172.21.15.32:0/1315336723 >> 172.21.15.32:0/1315336723 conn(0x7f13080fbbf0 msgr2=0x7f13080fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:01.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.350+0000 7f130f712640 1 -- 172.21.15.32:0/1315336723 shutdown_connections 2024-04-25T00:31:01.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.350+0000 7f130f712640 1 -- 172.21.15.32:0/1315336723 wait complete. 2024-04-25T00:31:01.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.350+0000 7f130f712640 1 Processor -- start 2024-04-25T00:31:01.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.350+0000 7f130f712640 1 -- start start 2024-04-25T00:31:01.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.350+0000 7f130f712640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1308100680 0x7f1308197a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:01.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.351+0000 7f130f712640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1308101870 0x7f1308197f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:01.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.351+0000 7f130f712640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1308102230 0x7f130819eff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:01.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.351+0000 7f130f712640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f130810c2e0 con 0x7f1308102230 2024-04-25T00:31:01.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.351+0000 7f130f712640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f130810c160 con 0x7f1308100680 2024-04-25T00:31:01.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.351+0000 7f130f712640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f130810c460 con 0x7f1308101870 2024-04-25T00:31:01.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.351+0000 7f130dcaf640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1308102230 0x7f130819eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:01.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.351+0000 7f130dcaf640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1308102230 0x7f130819eff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46808/0 (socket says 172.21.15.32:46808) 2024-04-25T00:31:01.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.351+0000 7f130dcaf640 1 -- 172.21.15.32:0/1107348831 learned_addr learned my addr 172.21.15.32:0/1107348831 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:01.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.351+0000 7f130ccad640 1 --2- 172.21.15.32:0/1107348831 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1308101870 0x7f1308197f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:01.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.351+0000 7f130d4ae640 1 --2- 172.21.15.32:0/1107348831 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1308100680 0x7f1308197a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:01.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.351+0000 7f130dcaf640 1 -- 172.21.15.32:0/1107348831 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1308101870 msgr2=0x7f1308197f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:01.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.351+0000 7f130dcaf640 1 --2- 172.21.15.32:0/1107348831 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1308101870 0x7f1308197f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.351+0000 7f130dcaf640 1 -- 172.21.15.32:0/1107348831 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1308100680 msgr2=0x7f1308197a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:01.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.351+0000 7f130dcaf640 1 --2- 172.21.15.32:0/1107348831 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1308100680 0x7f1308197a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.351+0000 7f130dcaf640 1 -- 172.21.15.32:0/1107348831 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f130819f660 con 0x7f1308102230 2024-04-25T00:31:01.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.351+0000 7f130dcaf640 1 --2- 172.21.15.32:0/1107348831 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1308102230 0x7f130819eff0 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f13000314e0 tx=0x7f1300031b60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:01.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.351+0000 7f13067fc640 1 -- 172.21.15.32:0/1107348831 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f130003f030 con 0x7f1308102230 2024-04-25T00:31:01.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.351+0000 7f13067fc640 1 -- 172.21.15.32:0/1107348831 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1300003ad0 con 0x7f1308102230 2024-04-25T00:31:01.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.351+0000 7f130f712640 1 -- 172.21.15.32:0/1107348831 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f130819f8f0 con 0x7f1308102230 2024-04-25T00:31:01.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.351+0000 7f13067fc640 1 -- 172.21.15.32:0/1107348831 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1300035070 con 0x7f1308102230 2024-04-25T00:31:01.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.351+0000 7f130f712640 1 -- 172.21.15.32:0/1107348831 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f130819fde0 con 0x7f1308102230 2024-04-25T00:31:01.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.352+0000 7f130f712640 1 -- 172.21.15.32:0/1107348831 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f130819ffe0 con 0x7f1308102230 2024-04-25T00:31:01.357 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.353+0000 7f13067fc640 1 -- 172.21.15.32:0/1107348831 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f1300042030 con 0x7f1308102230 2024-04-25T00:31:01.357 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.354+0000 7f13067fc640 1 --2- 172.21.15.32:0/1107348831 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f12f00a2b90 0x7f12f00a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:01.357 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.354+0000 7f13067fc640 1 -- 172.21.15.32:0/1107348831 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f13000ef8b0 con 0x7f1308102230 2024-04-25T00:31:01.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.354+0000 7f130d4ae640 1 --2- 172.21.15.32:0/1107348831 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f12f00a2b90 0x7f12f00a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:01.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.355+0000 7f130d4ae640 1 --2- 172.21.15.32:0/1107348831 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f12f00a2b90 0x7f12f00a5050 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f13081016d0 tx=0x7f12f8009500 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:01.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.359+0000 7f13067fc640 1 -- 172.21.15.32:0/1107348831 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f13000bc5d0 con 0x7f1308102230 2024-04-25T00:31:01.544 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.542+0000 7f130f712640 1 -- 172.21.15.32:0/1107348831 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) -- 0x7f13081a0370 con 0x7f1308102230 2024-04-25T00:31:01.544 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.543+0000 7f13067fc640 1 -- 172.21.15.32:0/1107348831 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v89) ==== 104+0+11 (secure 0 0 0) 0x7f13000bbed0 con 0x7f1308102230 2024-04-25T00:31:01.544 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 32 2024-04-25T00:31:01.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.545+0000 7f130f712640 1 -- 172.21.15.32:0/1107348831 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f12f00a2b90 msgr2=0x7f12f00a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:01.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.545+0000 7f130f712640 1 --2- 172.21.15.32:0/1107348831 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f12f00a2b90 0x7f12f00a5050 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f13081016d0 tx=0x7f12f8009500 comp rx=0 tx=0).stop 2024-04-25T00:31:01.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.545+0000 7f130f712640 1 -- 172.21.15.32:0/1107348831 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1308102230 msgr2=0x7f130819eff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:01.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.545+0000 7f130f712640 1 --2- 172.21.15.32:0/1107348831 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1308102230 0x7f130819eff0 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f13000314e0 tx=0x7f1300031b60 comp rx=0 tx=0).stop 2024-04-25T00:31:01.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.545+0000 7f130f712640 1 -- 172.21.15.32:0/1107348831 shutdown_connections 2024-04-25T00:31:01.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.545+0000 7f130f712640 1 --2- 172.21.15.32:0/1107348831 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f12f00a2b90 0x7f12f00a5050 unknown :-1 s=CLOSED pgs=166 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.545+0000 7f130f712640 1 --2- 172.21.15.32:0/1107348831 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1308102230 0x7f130819eff0 unknown :-1 s=CLOSED pgs=236 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.545+0000 7f130f712640 1 --2- 172.21.15.32:0/1107348831 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1308101870 0x7f1308197f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.545+0000 7f130f712640 1 --2- 172.21.15.32:0/1107348831 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1308100680 0x7f1308197a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.545+0000 7f130f712640 1 -- 172.21.15.32:0/1107348831 >> 172.21.15.32:0/1107348831 conn(0x7f13080fbbf0 msgr2=0x7f13080fd700 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:01.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.545+0000 7f130f712640 1 -- 172.21.15.32:0/1107348831 shutdown_connections 2024-04-25T00:31:01.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.545+0000 7f130f712640 1 -- 172.21.15.32:0/1107348831 wait complete. 2024-04-25T00:31:01.557 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-04-25T00:31:01.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.660+0000 7f4cd1015640 1 -- 172.21.15.32:0/3194963020 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ccc1006a0 msgr2=0x7f4ccc100ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:01.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.660+0000 7f4cd1015640 1 --2- 172.21.15.32:0/3194963020 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ccc1006a0 0x7f4ccc100ac0 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f4cbc009900 tx=0x7f4cbc02a410 comp rx=0 tx=0).stop 2024-04-25T00:31:01.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.660+0000 7f4cd1015640 1 -- 172.21.15.32:0/3194963020 shutdown_connections 2024-04-25T00:31:01.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.660+0000 7f4cd1015640 1 --2- 172.21.15.32:0/3194963020 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ccc102250 0x7f4ccc108ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.660+0000 7f4cd1015640 1 --2- 172.21.15.32:0/3194963020 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4ccc101890 0x7f4ccc101d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.660+0000 7f4cd1015640 1 --2- 172.21.15.32:0/3194963020 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ccc1006a0 0x7f4ccc100ac0 unknown :-1 s=CLOSED pgs=237 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.660+0000 7f4cd1015640 1 -- 172.21.15.32:0/3194963020 >> 172.21.15.32:0/3194963020 conn(0x7f4ccc0fbc30 msgr2=0x7f4ccc0fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:01.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.660+0000 7f4cd1015640 1 -- 172.21.15.32:0/3194963020 shutdown_connections 2024-04-25T00:31:01.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.660+0000 7f4cd1015640 1 -- 172.21.15.32:0/3194963020 wait complete. 2024-04-25T00:31:01.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.661+0000 7f4cd1015640 1 Processor -- start 2024-04-25T00:31:01.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.661+0000 7f4cd1015640 1 -- start start 2024-04-25T00:31:01.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.661+0000 7f4cd1015640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ccc1006a0 0x7f4ccc197af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:01.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.661+0000 7f4cd1015640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ccc101890 0x7f4ccc198030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:01.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.661+0000 7f4cd1015640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4ccc102250 0x7f4ccc19f0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:01.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.661+0000 7f4cd1015640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f4ccc10d440 con 0x7f4ccc1006a0 2024-04-25T00:31:01.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.661+0000 7f4cd1015640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f4ccc10d2c0 con 0x7f4ccc102250 2024-04-25T00:31:01.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.661+0000 7f4cd1015640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f4ccc10d5c0 con 0x7f4ccc101890 2024-04-25T00:31:01.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.661+0000 7f4ccad9d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ccc1006a0 0x7f4ccc197af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:01.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.661+0000 7f4ccad9d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ccc1006a0 0x7f4ccc197af0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46832/0 (socket says 172.21.15.32:46832) 2024-04-25T00:31:01.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.661+0000 7f4ccad9d640 1 -- 172.21.15.32:0/3901243021 learned_addr learned my addr 172.21.15.32:0/3901243021 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:01.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.662+0000 7f4cca59c640 1 --2- 172.21.15.32:0/3901243021 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ccc101890 0x7f4ccc198030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:01.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.662+0000 7f4ccb59e640 1 --2- 172.21.15.32:0/3901243021 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4ccc102250 0x7f4ccc19f0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:01.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.662+0000 7f4ccad9d640 1 -- 172.21.15.32:0/3901243021 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ccc101890 msgr2=0x7f4ccc198030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:01.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.662+0000 7f4ccad9d640 1 --2- 172.21.15.32:0/3901243021 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ccc101890 0x7f4ccc198030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.662+0000 7f4ccad9d640 1 -- 172.21.15.32:0/3901243021 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4ccc102250 msgr2=0x7f4ccc19f0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:01.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.662+0000 7f4ccad9d640 1 --2- 172.21.15.32:0/3901243021 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4ccc102250 0x7f4ccc19f0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.662+0000 7f4ccad9d640 1 -- 172.21.15.32:0/3901243021 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4ccc19f710 con 0x7f4ccc1006a0 2024-04-25T00:31:01.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.662+0000 7f4cca59c640 1 --2- 172.21.15.32:0/3901243021 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ccc101890 0x7f4ccc198030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:31:01.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.662+0000 7f4ccb59e640 1 --2- 172.21.15.32:0/3901243021 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4ccc102250 0x7f4ccc19f0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:31:01.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.662+0000 7f4ccad9d640 1 --2- 172.21.15.32:0/3901243021 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ccc1006a0 0x7f4ccc197af0 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f4cbc031610 tx=0x7f4cbc0043b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:01.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.662+0000 7f4cb3fff640 1 -- 172.21.15.32:0/3901243021 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4cbc03f030 con 0x7f4ccc1006a0 2024-04-25T00:31:01.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.662+0000 7f4cd1015640 1 -- 172.21.15.32:0/3901243021 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4ccc19f9a0 con 0x7f4ccc1006a0 2024-04-25T00:31:01.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.662+0000 7f4cb3fff640 1 -- 172.21.15.32:0/3901243021 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4cbc003ce0 con 0x7f4ccc1006a0 2024-04-25T00:31:01.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.662+0000 7f4cb3fff640 1 -- 172.21.15.32:0/3901243021 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4cbc035070 con 0x7f4ccc1006a0 2024-04-25T00:31:01.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.662+0000 7f4cd1015640 1 -- 172.21.15.32:0/3901243021 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4ccc19fee0 con 0x7f4ccc1006a0 2024-04-25T00:31:01.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.663+0000 7f4cd1015640 1 -- 172.21.15.32:0/3901243021 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4ccc107a90 con 0x7f4ccc1006a0 2024-04-25T00:31:01.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.664+0000 7f4cb3fff640 1 -- 172.21.15.32:0/3901243021 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f4cbc042030 con 0x7f4ccc1006a0 2024-04-25T00:31:01.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.665+0000 7f4cb3fff640 1 --2- 172.21.15.32:0/3901243021 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4cb40a2b90 0x7f4cb40a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:01.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.665+0000 7f4cb3fff640 1 -- 172.21.15.32:0/3901243021 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f4cbc0efc80 con 0x7f4ccc1006a0 2024-04-25T00:31:01.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.665+0000 7f4cca59c640 1 --2- 172.21.15.32:0/3901243021 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4cb40a2b90 0x7f4cb40a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:01.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.665+0000 7f4cca59c640 1 --2- 172.21.15.32:0/3901243021 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4cb40a2b90 0x7f4cb40a5050 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7f4ccc199000 tx=0x7f4cc0005ec0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:01.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.668+0000 7f4cb3fff640 1 -- 172.21.15.32:0/3901243021 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f4cbc0bc9a0 con 0x7f4ccc1006a0 2024-04-25T00:31:01.854 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.852+0000 7f4cd1015640 1 -- 172.21.15.32:0/3901243021 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) -- 0x7f4ccc062340 con 0x7f4ccc1006a0 2024-04-25T00:31:01.854 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.852+0000 7f4cb3fff640 1 -- 172.21.15.32:0/3901243021 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v89) ==== 100+0+11 (secure 0 0 0) 0x7f4cbc0bc2a0 con 0x7f4ccc1006a0 2024-04-25T00:31:01.854 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 64 2024-04-25T00:31:01.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.854+0000 7f4cd1015640 1 -- 172.21.15.32:0/3901243021 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4cb40a2b90 msgr2=0x7f4cb40a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:01.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.854+0000 7f4cd1015640 1 --2- 172.21.15.32:0/3901243021 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4cb40a2b90 0x7f4cb40a5050 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7f4ccc199000 tx=0x7f4cc0005ec0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.854+0000 7f4cd1015640 1 -- 172.21.15.32:0/3901243021 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ccc1006a0 msgr2=0x7f4ccc197af0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:01.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.854+0000 7f4cd1015640 1 --2- 172.21.15.32:0/3901243021 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ccc1006a0 0x7f4ccc197af0 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f4cbc031610 tx=0x7f4cbc0043b0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.854+0000 7f4cd1015640 1 -- 172.21.15.32:0/3901243021 shutdown_connections 2024-04-25T00:31:01.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.854+0000 7f4cd1015640 1 --2- 172.21.15.32:0/3901243021 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4cb40a2b90 0x7f4cb40a5050 unknown :-1 s=CLOSED pgs=167 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.854+0000 7f4cd1015640 1 --2- 172.21.15.32:0/3901243021 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4ccc102250 0x7f4ccc19f0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.854+0000 7f4cd1015640 1 --2- 172.21.15.32:0/3901243021 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ccc101890 0x7f4ccc198030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.854+0000 7f4cd1015640 1 --2- 172.21.15.32:0/3901243021 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ccc1006a0 0x7f4ccc197af0 unknown :-1 s=CLOSED pgs=238 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.854+0000 7f4cd1015640 1 -- 172.21.15.32:0/3901243021 >> 172.21.15.32:0/3901243021 conn(0x7f4ccc0fbc30 msgr2=0x7f4ccc104c60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:01.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.854+0000 7f4cd1015640 1 -- 172.21.15.32:0/3901243021 shutdown_connections 2024-04-25T00:31:01.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.854+0000 7f4cd1015640 1 -- 172.21.15.32:0/3901243021 wait complete. 2024-04-25T00:31:01.867 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data_ec pg_num 2024-04-25T00:31:01.975 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.972+0000 7f07c3e15640 1 -- 172.21.15.32:0/1695483265 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f07bc104b50 msgr2=0x7f07bc106f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:01.975 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.972+0000 7f07c3e15640 1 --2- 172.21.15.32:0/1695483265 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f07bc104b50 0x7f07bc106f40 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7f07b0009900 tx=0x7f07b002a410 comp rx=0 tx=0).stop 2024-04-25T00:31:01.975 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.972+0000 7f07c3e15640 1 -- 172.21.15.32:0/1695483265 shutdown_connections 2024-04-25T00:31:01.975 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.972+0000 7f07c3e15640 1 --2- 172.21.15.32:0/1695483265 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f07bc107480 0x7f07bc1098d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.975 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.972+0000 7f07c3e15640 1 --2- 172.21.15.32:0/1695483265 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f07bc104b50 0x7f07bc106f40 unknown :-1 s=CLOSED pgs=239 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.975 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.972+0000 7f07c3e15640 1 --2- 172.21.15.32:0/1695483265 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f07bc102220 0x7f07bc104610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.975 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.972+0000 7f07c3e15640 1 -- 172.21.15.32:0/1695483265 >> 172.21.15.32:0/1695483265 conn(0x7f07bc0fbc30 msgr2=0x7f07bc0fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:01.975 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.972+0000 7f07c3e15640 1 -- 172.21.15.32:0/1695483265 shutdown_connections 2024-04-25T00:31:01.975 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.972+0000 7f07c3e15640 1 -- 172.21.15.32:0/1695483265 wait complete. 2024-04-25T00:31:01.976 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.972+0000 7f07c3e15640 1 Processor -- start 2024-04-25T00:31:01.976 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.972+0000 7f07c3e15640 1 -- start start 2024-04-25T00:31:01.976 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.973+0000 7f07c3e15640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f07bc102220 0x7f07bc197ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:01.976 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.973+0000 7f07c3e15640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f07bc104b50 0x7f07bc198010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:01.976 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.973+0000 7f07c3e15640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f07bc107480 0x7f07bc19f080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:01.976 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.973+0000 7f07c3e15640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f07bc10d460 con 0x7f07bc102220 2024-04-25T00:31:01.976 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.973+0000 7f07c3e15640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f07bc10d2e0 con 0x7f07bc104b50 2024-04-25T00:31:01.976 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.973+0000 7f07c3e15640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f07bc10d5e0 con 0x7f07bc107480 2024-04-25T00:31:01.976 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.973+0000 7f07c1bb1640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f07bc102220 0x7f07bc197ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:01.976 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.973+0000 7f07c1bb1640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f07bc102220 0x7f07bc197ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:60790/0 (socket says 172.21.15.32:60790) 2024-04-25T00:31:01.976 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.973+0000 7f07c1bb1640 1 -- 172.21.15.32:0/2465606290 learned_addr learned my addr 172.21.15.32:0/2465606290 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:01.976 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.973+0000 7f07c13b0640 1 --2- 172.21.15.32:0/2465606290 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f07bc104b50 0x7f07bc198010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:01.976 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.973+0000 7f07c23b2640 1 --2- 172.21.15.32:0/2465606290 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f07bc107480 0x7f07bc19f080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:01.977 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.973+0000 7f07c1bb1640 1 -- 172.21.15.32:0/2465606290 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f07bc107480 msgr2=0x7f07bc19f080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:01.977 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.973+0000 7f07c1bb1640 1 --2- 172.21.15.32:0/2465606290 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f07bc107480 0x7f07bc19f080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.977 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.973+0000 7f07c1bb1640 1 -- 172.21.15.32:0/2465606290 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f07bc104b50 msgr2=0x7f07bc198010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:01.977 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.973+0000 7f07c1bb1640 1 --2- 172.21.15.32:0/2465606290 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f07bc104b50 0x7f07bc198010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:01.977 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.973+0000 7f07c1bb1640 1 -- 172.21.15.32:0/2465606290 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f07bc19f6f0 con 0x7f07bc102220 2024-04-25T00:31:01.977 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.973+0000 7f07c23b2640 1 --2- 172.21.15.32:0/2465606290 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f07bc107480 0x7f07bc19f080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:31:01.977 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.973+0000 7f07c1bb1640 1 --2- 172.21.15.32:0/2465606290 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f07bc102220 0x7f07bc197ad0 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7f07b400d950 tx=0x7f07b400de10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:01.977 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.973+0000 7f07baffd640 1 -- 172.21.15.32:0/2465606290 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f07b4017070 con 0x7f07bc102220 2024-04-25T00:31:01.977 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.973+0000 7f07c3e15640 1 -- 172.21.15.32:0/2465606290 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f07bc19f9e0 con 0x7f07bc102220 2024-04-25T00:31:01.977 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.973+0000 7f07baffd640 1 -- 172.21.15.32:0/2465606290 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f07b4009c30 con 0x7f07bc102220 2024-04-25T00:31:01.977 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.973+0000 7f07baffd640 1 -- 172.21.15.32:0/2465606290 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f07b400f6b0 con 0x7f07bc102220 2024-04-25T00:31:01.977 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.974+0000 7f07c3e15640 1 -- 172.21.15.32:0/2465606290 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f07bc19ff00 con 0x7f07bc102220 2024-04-25T00:31:01.977 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.974+0000 7f07c3e15640 1 -- 172.21.15.32:0/2465606290 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f07bc0fd720 con 0x7f07bc102220 2024-04-25T00:31:01.978 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.976+0000 7f07baffd640 1 -- 172.21.15.32:0/2465606290 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f07b40109d0 con 0x7f07bc102220 2024-04-25T00:31:01.978 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.976+0000 7f07baffd640 1 --2- 172.21.15.32:0/2465606290 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f07a40a2b90 0x7f07a40a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:01.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.976+0000 7f07baffd640 1 -- 172.21.15.32:0/2465606290 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f07b40ceeb0 con 0x7f07bc102220 2024-04-25T00:31:01.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.976+0000 7f07c13b0640 1 --2- 172.21.15.32:0/2465606290 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f07a40a2b90 0x7f07a40a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:01.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.977+0000 7f07c13b0640 1 --2- 172.21.15.32:0/2465606290 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f07a40a2b90 0x7f07a40a5050 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7f07b002aa00 tx=0x7f07b00315b0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:01.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:01.980+0000 7f07baffd640 1 -- 172.21.15.32:0/2465606290 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f07b40fd740 con 0x7f07bc102220 2024-04-25T00:31:02.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.167+0000 7f07c3e15640 1 -- 172.21.15.32:0/2465606290 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"} v 0) -- 0x7f07bc02d420 con 0x7f07bc102220 2024-04-25T00:31:02.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.167+0000 7f07baffd640 1 -- 172.21.15.32:0/2465606290 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]=0 v89) ==== 103+0+11 (secure 0 0 0) 0x7f07b4009de0 con 0x7f07bc102220 2024-04-25T00:31:02.168 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 32 2024-04-25T00:31:02.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.169+0000 7f07c3e15640 1 -- 172.21.15.32:0/2465606290 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f07a40a2b90 msgr2=0x7f07a40a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:02.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.169+0000 7f07c3e15640 1 --2- 172.21.15.32:0/2465606290 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f07a40a2b90 0x7f07a40a5050 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7f07b002aa00 tx=0x7f07b00315b0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.169+0000 7f07c3e15640 1 -- 172.21.15.32:0/2465606290 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f07bc102220 msgr2=0x7f07bc197ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:02.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.169+0000 7f07c3e15640 1 --2- 172.21.15.32:0/2465606290 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f07bc102220 0x7f07bc197ad0 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7f07b400d950 tx=0x7f07b400de10 comp rx=0 tx=0).stop 2024-04-25T00:31:02.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.169+0000 7f07c3e15640 1 -- 172.21.15.32:0/2465606290 shutdown_connections 2024-04-25T00:31:02.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.169+0000 7f07c3e15640 1 --2- 172.21.15.32:0/2465606290 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f07a40a2b90 0x7f07a40a5050 unknown :-1 s=CLOSED pgs=168 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.169+0000 7f07c3e15640 1 --2- 172.21.15.32:0/2465606290 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f07bc107480 0x7f07bc19f080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.169+0000 7f07c3e15640 1 --2- 172.21.15.32:0/2465606290 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f07bc104b50 0x7f07bc198010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.169+0000 7f07c3e15640 1 --2- 172.21.15.32:0/2465606290 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f07bc102220 0x7f07bc197ad0 unknown :-1 s=CLOSED pgs=240 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.169+0000 7f07c3e15640 1 -- 172.21.15.32:0/2465606290 >> 172.21.15.32:0/2465606290 conn(0x7f07bc0fbc30 msgr2=0x7f07bc102ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:02.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.169+0000 7f07c3e15640 1 -- 172.21.15.32:0/2465606290 shutdown_connections 2024-04-25T00:31:02.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.169+0000 7f07c3e15640 1 -- 172.21.15.32:0/2465606290 wait complete. 2024-04-25T00:31:02.179 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:01 smithi032 ceph-mon[29357]: pgmap v298: 129 pgs: 129 active+clean; 579 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:31:02.180 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:01 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3039641119' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:31:02.180 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:01 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1817031357' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:31:02.180 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:01 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1107348831' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:31:02.180 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:01 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3901243021' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:31:02.181 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-04-25T00:31:02.287 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.284+0000 7f3315d9e640 1 -- 172.21.15.32:0/603845032 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3310102100 msgr2=0x7f33101044f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:02.287 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.284+0000 7f3315d9e640 1 --2- 172.21.15.32:0/603845032 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3310102100 0x7f33101044f0 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7f3300009900 tx=0x7f330002a410 comp rx=0 tx=0).stop 2024-04-25T00:31:02.287 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.284+0000 7f3315d9e640 1 -- 172.21.15.32:0/603845032 shutdown_connections 2024-04-25T00:31:02.287 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.284+0000 7f3315d9e640 1 --2- 172.21.15.32:0/603845032 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3310107360 0x7f33101097b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.287 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.284+0000 7f3315d9e640 1 --2- 172.21.15.32:0/603845032 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3310104a30 0x7f3310106e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.284+0000 7f3315d9e640 1 --2- 172.21.15.32:0/603845032 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3310102100 0x7f33101044f0 unknown :-1 s=CLOSED pgs=241 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.284+0000 7f3315d9e640 1 -- 172.21.15.32:0/603845032 >> 172.21.15.32:0/603845032 conn(0x7f33100fbaf0 msgr2=0x7f33100fdf30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:02.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.284+0000 7f3315d9e640 1 -- 172.21.15.32:0/603845032 shutdown_connections 2024-04-25T00:31:02.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.284+0000 7f3315d9e640 1 -- 172.21.15.32:0/603845032 wait complete. 2024-04-25T00:31:02.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.284+0000 7f3315d9e640 1 Processor -- start 2024-04-25T00:31:02.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.284+0000 7f3315d9e640 1 -- start start 2024-04-25T00:31:02.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.285+0000 7f3315d9e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3310102100 0x7f3310101a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:02.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.285+0000 7f3315d9e640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3310104a30 0x7f33101000b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:02.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.285+0000 7f3315d9e640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3310107360 0x7f33101005f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:02.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.285+0000 7f3315d9e640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f331010d310 con 0x7f3310102100 2024-04-25T00:31:02.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.285+0000 7f3315d9e640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f331010d190 con 0x7f3310104a30 2024-04-25T00:31:02.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.285+0000 7f3315d9e640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f331010d490 con 0x7f3310107360 2024-04-25T00:31:02.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.285+0000 7f3314d9c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3310102100 0x7f3310101a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:02.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.285+0000 7f3314d9c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3310102100 0x7f3310101a70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:60796/0 (socket says 172.21.15.32:60796) 2024-04-25T00:31:02.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.285+0000 7f3314d9c640 1 -- 172.21.15.32:0/3111745697 learned_addr learned my addr 172.21.15.32:0/3111745697 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:02.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.285+0000 7f331559d640 1 --2- 172.21.15.32:0/3111745697 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3310107360 0x7f33101005f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:02.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.285+0000 7f330ffff640 1 --2- 172.21.15.32:0/3111745697 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3310104a30 0x7f33101000b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:02.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.285+0000 7f3314d9c640 1 -- 172.21.15.32:0/3111745697 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3310107360 msgr2=0x7f33101005f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:02.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.285+0000 7f3314d9c640 1 --2- 172.21.15.32:0/3111745697 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3310107360 0x7f33101005f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.285+0000 7f3314d9c640 1 -- 172.21.15.32:0/3111745697 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3310104a30 msgr2=0x7f33101000b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:02.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.285+0000 7f3314d9c640 1 --2- 172.21.15.32:0/3111745697 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3310104a30 0x7f33101000b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.285+0000 7f3314d9c640 1 -- 172.21.15.32:0/3111745697 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3310100e80 con 0x7f3310102100 2024-04-25T00:31:02.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.285+0000 7f330ffff640 1 --2- 172.21.15.32:0/3111745697 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3310104a30 0x7f33101000b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:31:02.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.285+0000 7f331559d640 1 --2- 172.21.15.32:0/3111745697 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3310107360 0x7f33101005f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:31:02.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.285+0000 7f3314d9c640 1 --2- 172.21.15.32:0/3111745697 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3310102100 0x7f3310101a70 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7f3300009230 tx=0x7f3300031b60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:02.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.285+0000 7f330dffb640 1 -- 172.21.15.32:0/3111745697 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f330003f030 con 0x7f3310102100 2024-04-25T00:31:02.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.286+0000 7f330dffb640 1 -- 172.21.15.32:0/3111745697 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3300003ad0 con 0x7f3310102100 2024-04-25T00:31:02.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.286+0000 7f3315d9e640 1 -- 172.21.15.32:0/3111745697 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f33101a1a40 con 0x7f3310102100 2024-04-25T00:31:02.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.286+0000 7f330dffb640 1 -- 172.21.15.32:0/3111745697 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3300035070 con 0x7f3310102100 2024-04-25T00:31:02.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.286+0000 7f3315d9e640 1 -- 172.21.15.32:0/3111745697 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f33101a1f90 con 0x7f3310102100 2024-04-25T00:31:02.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.287+0000 7f3315d9e640 1 -- 172.21.15.32:0/3111745697 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f33100fd5e0 con 0x7f3310102100 2024-04-25T00:31:02.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.288+0000 7f330dffb640 1 -- 172.21.15.32:0/3111745697 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f3300042030 con 0x7f3310102100 2024-04-25T00:31:02.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.289+0000 7f330dffb640 1 --2- 172.21.15.32:0/3111745697 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f32f80a2b90 0x7f32f80a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:02.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.289+0000 7f330ffff640 1 --2- 172.21.15.32:0/3111745697 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f32f80a2b90 0x7f32f80a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:02.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.289+0000 7f330dffb640 1 -- 172.21.15.32:0/3111745697 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f33000ef8b0 con 0x7f3310102100 2024-04-25T00:31:02.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.289+0000 7f330ffff640 1 --2- 172.21.15.32:0/3111745697 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f32f80a2b90 0x7f32f80a5050 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f33101017e0 tx=0x7f3304005f90 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:02.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.293+0000 7f330dffb640 1 -- 172.21.15.32:0/3111745697 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f33000bc5d0 con 0x7f3310102100 2024-04-25T00:31:02.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:01 smithi121 ceph-mon[32231]: pgmap v298: 129 pgs: 129 active+clean; 579 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:31:02.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:01 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3039641119' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:31:02.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:01 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1817031357' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:31:02.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:01 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1107348831' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:31:02.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:01 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3901243021' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:31:02.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:01 smithi130 ceph-mon[32525]: pgmap v298: 129 pgs: 129 active+clean; 579 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:31:02.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:01 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3039641119' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:31:02.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:01 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1817031357' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:31:02.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:01 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1107348831' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:31:02.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:01 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3901243021' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:31:02.491 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.490+0000 7f3315d9e640 1 -- 172.21.15.32:0/3111745697 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) -- 0x7f331002d170 con 0x7f3310102100 2024-04-25T00:31:02.491 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.491+0000 7f330dffb640 1 -- 172.21.15.32:0/3111745697 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 10 entries v89) ==== 81+0+590 (secure 0 0 0) 0x7f33000bbed0 con 0x7f3310102100 2024-04-25T00:31:02.492 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:0/2999636731 2024-04-26T00:23:42.776439+0000 2024-04-25T00:31:02.492 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:6801/4160192081 2024-04-26T00:23:42.776439+0000 2024-04-25T00:31:02.492 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:6800/4160192081 2024-04-26T00:23:42.776439+0000 2024-04-25T00:31:02.492 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:0/53550767 2024-04-26T00:23:42.776439+0000 2024-04-25T00:31:02.492 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:0/1066471448 2024-04-26T00:23:14.613921+0000 2024-04-25T00:31:02.492 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:0/3987019264 2024-04-26T00:23:14.613921+0000 2024-04-25T00:31:02.492 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:0/3041942084 2024-04-26T00:23:14.613921+0000 2024-04-25T00:31:02.492 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:0/3004184616 2024-04-26T00:23:42.776439+0000 2024-04-25T00:31:02.492 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:6800/1637256323 2024-04-26T00:23:14.613921+0000 2024-04-25T00:31:02.492 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:6801/1637256323 2024-04-26T00:23:14.613921+0000 2024-04-25T00:31:02.493 INFO:teuthology.orchestra.run.smithi032.stderr:listed 10 entries 2024-04-25T00:31:02.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.493+0000 7f3315d9e640 1 -- 172.21.15.32:0/3111745697 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f32f80a2b90 msgr2=0x7f32f80a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:02.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.493+0000 7f3315d9e640 1 --2- 172.21.15.32:0/3111745697 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f32f80a2b90 0x7f32f80a5050 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f33101017e0 tx=0x7f3304005f90 comp rx=0 tx=0).stop 2024-04-25T00:31:02.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.493+0000 7f3315d9e640 1 -- 172.21.15.32:0/3111745697 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3310102100 msgr2=0x7f3310101a70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:02.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.493+0000 7f3315d9e640 1 --2- 172.21.15.32:0/3111745697 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3310102100 0x7f3310101a70 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7f3300009230 tx=0x7f3300031b60 comp rx=0 tx=0).stop 2024-04-25T00:31:02.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.493+0000 7f3315d9e640 1 -- 172.21.15.32:0/3111745697 shutdown_connections 2024-04-25T00:31:02.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.493+0000 7f3315d9e640 1 --2- 172.21.15.32:0/3111745697 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f32f80a2b90 0x7f32f80a5050 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.493+0000 7f3315d9e640 1 --2- 172.21.15.32:0/3111745697 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3310107360 0x7f33101005f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.493+0000 7f3315d9e640 1 --2- 172.21.15.32:0/3111745697 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3310104a30 0x7f33101000b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.493+0000 7f3315d9e640 1 --2- 172.21.15.32:0/3111745697 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3310102100 0x7f3310101a70 unknown :-1 s=CLOSED pgs=242 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.493+0000 7f3315d9e640 1 -- 172.21.15.32:0/3111745697 >> 172.21.15.32:0/3111745697 conn(0x7f33100fbaf0 msgr2=0x7f3310102b60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:02.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.493+0000 7f3315d9e640 1 -- 172.21.15.32:0/3111745697 shutdown_connections 2024-04-25T00:31:02.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.493+0000 7f3315d9e640 1 -- 172.21.15.32:0/3111745697 wait complete. 2024-04-25T00:31:02.505 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-04-25T00:31:02.505 DEBUG:teuthology.orchestra.run.smithi032:> dd if=/proc/self/mounts of=/dev/stdout 2024-04-25T00:31:02.521 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-25T00:31:02.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.663+0000 7f3e0e638640 1 -- 172.21.15.32:0/1686432195 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3e081017b0 msgr2=0x7f3e08101bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:02.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.663+0000 7f3e0e638640 1 --2- 172.21.15.32:0/1686432195 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3e081017b0 0x7f3e08101bd0 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7f3df8009900 tx=0x7f3df802a410 comp rx=0 tx=0).stop 2024-04-25T00:31:02.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.663+0000 7f3e0e638640 1 -- 172.21.15.32:0/1686432195 shutdown_connections 2024-04-25T00:31:02.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.663+0000 7f3e0e638640 1 --2- 172.21.15.32:0/1686432195 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3e08103360 0x7f3e08109bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.663+0000 7f3e0e638640 1 --2- 172.21.15.32:0/1686432195 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3e081029a0 0x7f3e08102e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.663+0000 7f3e0e638640 1 --2- 172.21.15.32:0/1686432195 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3e081017b0 0x7f3e08101bd0 unknown :-1 s=CLOSED pgs=243 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.663+0000 7f3e0e638640 1 -- 172.21.15.32:0/1686432195 >> 172.21.15.32:0/1686432195 conn(0x7f3e080fcd40 msgr2=0x7f3e080ff160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:02.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.663+0000 7f3e0e638640 1 -- 172.21.15.32:0/1686432195 shutdown_connections 2024-04-25T00:31:02.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.663+0000 7f3e0e638640 1 -- 172.21.15.32:0/1686432195 wait complete. 2024-04-25T00:31:02.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.663+0000 7f3e0e638640 1 Processor -- start 2024-04-25T00:31:02.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.663+0000 7f3e0e638640 1 -- start start 2024-04-25T00:31:02.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.664+0000 7f3e0e638640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3e081017b0 0x7f3e08197ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:02.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.664+0000 7f3e0e638640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3e081029a0 0x7f3e08198000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:02.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.664+0000 7f3e0e638640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3e08103360 0x7f3e0819f070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:02.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.664+0000 7f3e0e638640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f3e0810d480 con 0x7f3e08103360 2024-04-25T00:31:02.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.664+0000 7f3e0e638640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f3e0810d300 con 0x7f3e081029a0 2024-04-25T00:31:02.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.664+0000 7f3e0e638640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f3e0810d600 con 0x7f3e081017b0 2024-04-25T00:31:02.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.664+0000 7f3e0cbd5640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3e08103360 0x7f3e0819f070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:02.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.664+0000 7f3e0cbd5640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3e08103360 0x7f3e0819f070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:60818/0 (socket says 172.21.15.32:60818) 2024-04-25T00:31:02.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.664+0000 7f3e0cbd5640 1 -- 172.21.15.32:0/1088771809 learned_addr learned my addr 172.21.15.32:0/1088771809 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:02.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.664+0000 7f3e077fe640 1 --2- 172.21.15.32:0/1088771809 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3e081029a0 0x7f3e08198000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:02.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.664+0000 7f3e07fff640 1 --2- 172.21.15.32:0/1088771809 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3e081017b0 0x7f3e08197ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:02.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.664+0000 7f3e0cbd5640 1 -- 172.21.15.32:0/1088771809 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3e081017b0 msgr2=0x7f3e08197ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:02.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.664+0000 7f3e0cbd5640 1 --2- 172.21.15.32:0/1088771809 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3e081017b0 0x7f3e08197ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.664+0000 7f3e0cbd5640 1 -- 172.21.15.32:0/1088771809 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3e081029a0 msgr2=0x7f3e08198000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:02.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.664+0000 7f3e0cbd5640 1 --2- 172.21.15.32:0/1088771809 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3e081029a0 0x7f3e08198000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.664+0000 7f3e0cbd5640 1 -- 172.21.15.32:0/1088771809 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3e0819f6e0 con 0x7f3e08103360 2024-04-25T00:31:02.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.664+0000 7f3e0cbd5640 1 --2- 172.21.15.32:0/1088771809 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3e08103360 0x7f3e0819f070 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7f3e0000eb10 tx=0x7f3e0000c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:02.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.664+0000 7f3e057fa640 1 -- 172.21.15.32:0/1088771809 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3e00004b20 con 0x7f3e08103360 2024-04-25T00:31:02.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.665+0000 7f3e057fa640 1 -- 172.21.15.32:0/1088771809 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3e00004cc0 con 0x7f3e08103360 2024-04-25T00:31:02.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.665+0000 7f3e0e638640 1 -- 172.21.15.32:0/1088771809 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3e0819f9d0 con 0x7f3e08103360 2024-04-25T00:31:02.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.665+0000 7f3e057fa640 1 -- 172.21.15.32:0/1088771809 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3e00010760 con 0x7f3e08103360 2024-04-25T00:31:02.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.665+0000 7f3e0e638640 1 -- 172.21.15.32:0/1088771809 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f3e0819ff20 con 0x7f3e08103360 2024-04-25T00:31:02.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.666+0000 7f3e0e638640 1 -- 172.21.15.32:0/1088771809 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f3e08108a70 con 0x7f3e08103360 2024-04-25T00:31:02.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.667+0000 7f3e057fa640 1 -- 172.21.15.32:0/1088771809 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f3e00005020 con 0x7f3e08103360 2024-04-25T00:31:02.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.667+0000 7f3e057fa640 1 --2- 172.21.15.32:0/1088771809 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3df00a2b90 0x7f3df00a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:02.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.667+0000 7f3e07fff640 1 --2- 172.21.15.32:0/1088771809 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3df00a2b90 0x7f3df00a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:02.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.667+0000 7f3e057fa640 1 -- 172.21.15.32:0/1088771809 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f3e000cf920 con 0x7f3e08103360 2024-04-25T00:31:02.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.668+0000 7f3e07fff640 1 --2- 172.21.15.32:0/1088771809 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3df00a2b90 0x7f3df00a5050 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f3df802aa00 tx=0x7f3df80315b0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:02.672 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.672+0000 7f3e057fa640 1 -- 172.21.15.32:0/1088771809 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f3e0009c640 con 0x7f3e08103360 2024-04-25T00:31:02.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.857+0000 7f3e0e638640 1 -- 172.21.15.32:0/1088771809 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f3e08062340 con 0x7f3e08103360 2024-04-25T00:31:02.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.859+0000 7f3e057fa640 1 -- 172.21.15.32:0/1088771809 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v89) ==== 74+0+24128 (secure 0 0 0) 0x7f3e0009bf40 con 0x7f3e08103360 2024-04-25T00:31:02.860 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:31:02.860 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":89,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:30:53.134605+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-25T00:30:46.239556+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-25T00:30:47.257182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2024-04-25T00:30:51.193975+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":83,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-num-failure-domains":"0","crush-osds-per-failure-domain":"0","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:31:02.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.861+0000 7f3e0e638640 1 -- 172.21.15.32:0/1088771809 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3df00a2b90 msgr2=0x7f3df00a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:02.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.861+0000 7f3e0e638640 1 --2- 172.21.15.32:0/1088771809 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3df00a2b90 0x7f3df00a5050 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f3df802aa00 tx=0x7f3df80315b0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.861+0000 7f3e0e638640 1 -- 172.21.15.32:0/1088771809 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3e08103360 msgr2=0x7f3e0819f070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:02.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.861+0000 7f3e0e638640 1 --2- 172.21.15.32:0/1088771809 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3e08103360 0x7f3e0819f070 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7f3e0000eb10 tx=0x7f3e0000c2d0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.861+0000 7f3e0e638640 1 -- 172.21.15.32:0/1088771809 shutdown_connections 2024-04-25T00:31:02.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.861+0000 7f3e0e638640 1 --2- 172.21.15.32:0/1088771809 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3df00a2b90 0x7f3df00a5050 unknown :-1 s=CLOSED pgs=170 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.861+0000 7f3e0e638640 1 --2- 172.21.15.32:0/1088771809 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3e08103360 0x7f3e0819f070 unknown :-1 s=CLOSED pgs=244 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.861+0000 7f3e0e638640 1 --2- 172.21.15.32:0/1088771809 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3e081029a0 0x7f3e08198000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.861+0000 7f3e0e638640 1 --2- 172.21.15.32:0/1088771809 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3e081017b0 0x7f3e08197ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.861+0000 7f3e0e638640 1 -- 172.21.15.32:0/1088771809 >> 172.21.15.32:0/1088771809 conn(0x7f3e080fcd40 msgr2=0x7f3e08105c40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:02.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.861+0000 7f3e0e638640 1 -- 172.21.15.32:0/1088771809 shutdown_connections 2024-04-25T00:31:02.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.861+0000 7f3e0e638640 1 -- 172.21.15.32:0/1088771809 wait complete. 2024-04-25T00:31:02.873 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-25T00:26:07.306839+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-04-25T00:30:46.239556+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '84', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-04-25T00:30:47.257182+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '85', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.05000000074505806}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_data_ec', 'create_time': '2024-04-25T00:30:51.193975+0000', 'flags': 5, 'flags_names': 'hashpspool,ec_overwrites', 'type': 3, 'size': 4, 'min_size': 3, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '89', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'cephfs_data_ec', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 8192, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}] 2024-04-25T00:31:02.873 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-25T00:31:02.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.978+0000 7f32be132640 1 -- 172.21.15.32:0/3667631319 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f32b8102250 msgr2=0x7f32b8108ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:02.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.978+0000 7f32be132640 1 --2- 172.21.15.32:0/3667631319 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f32b8102250 0x7f32b8108ae0 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7f32b0009ca0 tx=0x7f32b002a8c0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.979+0000 7f32be132640 1 -- 172.21.15.32:0/3667631319 shutdown_connections 2024-04-25T00:31:02.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.979+0000 7f32be132640 1 --2- 172.21.15.32:0/3667631319 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f32b8102250 0x7f32b8108ae0 unknown :-1 s=CLOSED pgs=245 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.979+0000 7f32be132640 1 --2- 172.21.15.32:0/3667631319 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f32b8101890 0x7f32b8101d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.979+0000 7f32be132640 1 --2- 172.21.15.32:0/3667631319 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f32b81006a0 0x7f32b8100ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.979+0000 7f32be132640 1 -- 172.21.15.32:0/3667631319 >> 172.21.15.32:0/3667631319 conn(0x7f32b80fbc30 msgr2=0x7f32b80fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:02.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.979+0000 7f32be132640 1 -- 172.21.15.32:0/3667631319 shutdown_connections 2024-04-25T00:31:02.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.979+0000 7f32be132640 1 -- 172.21.15.32:0/3667631319 wait complete. 2024-04-25T00:31:02.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.979+0000 7f32be132640 1 Processor -- start 2024-04-25T00:31:02.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.979+0000 7f32be132640 1 -- start start 2024-04-25T00:31:02.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.979+0000 7f32be132640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f32b81006a0 0x7f32b8197a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:02.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.979+0000 7f32be132640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f32b8101890 0x7f32b8197f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:02.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.979+0000 7f32be132640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f32b8102250 0x7f32b819eff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:02.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.979+0000 7f32be132640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f32b810d440 con 0x7f32b8101890 2024-04-25T00:31:02.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.979+0000 7f32be132640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f32b810d2c0 con 0x7f32b8102250 2024-04-25T00:31:02.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.979+0000 7f32be132640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f32b810d5c0 con 0x7f32b81006a0 2024-04-25T00:31:02.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.980+0000 7f32b6ffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f32b8101890 0x7f32b8197f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:02.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.980+0000 7f32b6ffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f32b8101890 0x7f32b8197f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:60840/0 (socket says 172.21.15.32:60840) 2024-04-25T00:31:02.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.980+0000 7f32b6ffd640 1 -- 172.21.15.32:0/2617296648 learned_addr learned my addr 172.21.15.32:0/2617296648 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:02.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.980+0000 7f32b77fe640 1 --2- 172.21.15.32:0/2617296648 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f32b81006a0 0x7f32b8197a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:02.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.980+0000 7f32b6ffd640 1 -- 172.21.15.32:0/2617296648 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f32b81006a0 msgr2=0x7f32b8197a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:02.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.980+0000 7f32b6ffd640 1 --2- 172.21.15.32:0/2617296648 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f32b81006a0 0x7f32b8197a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.980+0000 7f32b6ffd640 1 -- 172.21.15.32:0/2617296648 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f32b8102250 msgr2=0x7f32b819eff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:02.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.980+0000 7f32b6ffd640 1 --2- 172.21.15.32:0/2617296648 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f32b8102250 0x7f32b819eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:02.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.980+0000 7f32b6ffd640 1 -- 172.21.15.32:0/2617296648 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f32b819f660 con 0x7f32b8101890 2024-04-25T00:31:02.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.980+0000 7f32b6ffd640 1 --2- 172.21.15.32:0/2617296648 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f32b8101890 0x7f32b8197f80 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7f32ac00d770 tx=0x7f32ac00dc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:02.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.980+0000 7f32b4ff9640 1 -- 172.21.15.32:0/2617296648 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f32ac017070 con 0x7f32b8101890 2024-04-25T00:31:02.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.980+0000 7f32be132640 1 -- 172.21.15.32:0/2617296648 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f32b819f950 con 0x7f32b8101890 2024-04-25T00:31:02.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.980+0000 7f32b4ff9640 1 -- 172.21.15.32:0/2617296648 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f32ac005500 con 0x7f32b8101890 2024-04-25T00:31:02.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.980+0000 7f32b4ff9640 1 -- 172.21.15.32:0/2617296648 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f32ac004e00 con 0x7f32b8101890 2024-04-25T00:31:02.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.980+0000 7f32be132640 1 -- 172.21.15.32:0/2617296648 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f32b819fea0 con 0x7f32b8101890 2024-04-25T00:31:02.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.981+0000 7f32be132640 1 -- 172.21.15.32:0/2617296648 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f32b8107960 con 0x7f32b8101890 2024-04-25T00:31:02.985 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.982+0000 7f32b4ff9640 1 -- 172.21.15.32:0/2617296648 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f32ac004fa0 con 0x7f32b8101890 2024-04-25T00:31:02.985 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.983+0000 7f32b4ff9640 1 --2- 172.21.15.32:0/2617296648 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f32a00a2b90 0x7f32a00a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:02.985 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.983+0000 7f32b77fe640 1 --2- 172.21.15.32:0/2617296648 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f32a00a2b90 0x7f32a00a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:02.986 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.983+0000 7f32b4ff9640 1 -- 172.21.15.32:0/2617296648 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f32ac0cecf0 con 0x7f32b8101890 2024-04-25T00:31:02.986 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.983+0000 7f32b77fe640 1 --2- 172.21.15.32:0/2617296648 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f32a00a2b90 0x7f32a00a5050 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f32a8005d00 tx=0x7f32a8005c50 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:02.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:02.987+0000 7f32b4ff9640 1 -- 172.21.15.32:0/2617296648 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f32ac0d3070 con 0x7f32b8101890 2024-04-25T00:31:03.171 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.171+0000 7f32be132640 1 -- 172.21.15.32:0/2617296648 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7f32b802d170 con 0x7f32b8101890 2024-04-25T00:31:03.172 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.171+0000 7f32b4ff9640 1 -- 172.21.15.32:0/2617296648 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v89) ==== 93+0+10 (secure 0 0 0) 0x7f32ac09ba10 con 0x7f32b8101890 2024-04-25T00:31:03.172 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 1 2024-04-25T00:31:03.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.173+0000 7f32be132640 1 -- 172.21.15.32:0/2617296648 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f32a00a2b90 msgr2=0x7f32a00a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:03.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.173+0000 7f32be132640 1 --2- 172.21.15.32:0/2617296648 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f32a00a2b90 0x7f32a00a5050 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f32a8005d00 tx=0x7f32a8005c50 comp rx=0 tx=0).stop 2024-04-25T00:31:03.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.173+0000 7f32be132640 1 -- 172.21.15.32:0/2617296648 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f32b8101890 msgr2=0x7f32b8197f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:03.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.173+0000 7f32be132640 1 --2- 172.21.15.32:0/2617296648 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f32b8101890 0x7f32b8197f80 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7f32ac00d770 tx=0x7f32ac00dc30 comp rx=0 tx=0).stop 2024-04-25T00:31:03.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.173+0000 7f32be132640 1 -- 172.21.15.32:0/2617296648 shutdown_connections 2024-04-25T00:31:03.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.173+0000 7f32be132640 1 --2- 172.21.15.32:0/2617296648 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f32a00a2b90 0x7f32a00a5050 unknown :-1 s=CLOSED pgs=171 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.175 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.173+0000 7f32be132640 1 --2- 172.21.15.32:0/2617296648 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f32b8102250 0x7f32b819eff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.175 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.173+0000 7f32be132640 1 --2- 172.21.15.32:0/2617296648 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f32b8101890 0x7f32b8197f80 unknown :-1 s=CLOSED pgs=246 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.175 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.173+0000 7f32be132640 1 --2- 172.21.15.32:0/2617296648 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f32b81006a0 0x7f32b8197a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.175 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.173+0000 7f32be132640 1 -- 172.21.15.32:0/2617296648 >> 172.21.15.32:0/2617296648 conn(0x7f32b80fbc30 msgr2=0x7f32b8104b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:03.175 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.173+0000 7f32be132640 1 -- 172.21.15.32:0/2617296648 shutdown_connections 2024-04-25T00:31:03.175 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.173+0000 7f32be132640 1 -- 172.21.15.32:0/2617296648 wait complete. 2024-04-25T00:31:03.184 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:02 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2465606290' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:31:03.184 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:02 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3111745697' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-25T00:31:03.184 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:02 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1088771809' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:31:03.185 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-04-25T00:31:03.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.289+0000 7fdc0ba4c640 1 -- 172.21.15.32:0/1979701946 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdc041006a0 msgr2=0x7fdc04100ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:03.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.289+0000 7fdc0ba4c640 1 --2- 172.21.15.32:0/1979701946 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdc041006a0 0x7fdc04100ac0 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7fdc00009900 tx=0x7fdc0002a410 comp rx=0 tx=0).stop 2024-04-25T00:31:03.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.289+0000 7fdc0ba4c640 1 -- 172.21.15.32:0/1979701946 shutdown_connections 2024-04-25T00:31:03.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.289+0000 7fdc0ba4c640 1 --2- 172.21.15.32:0/1979701946 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdc04102250 0x7fdc04108ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.289+0000 7fdc0ba4c640 1 --2- 172.21.15.32:0/1979701946 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdc04101890 0x7fdc04101d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.289+0000 7fdc0ba4c640 1 --2- 172.21.15.32:0/1979701946 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdc041006a0 0x7fdc04100ac0 unknown :-1 s=CLOSED pgs=247 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.289+0000 7fdc0ba4c640 1 -- 172.21.15.32:0/1979701946 >> 172.21.15.32:0/1979701946 conn(0x7fdc040fbc30 msgr2=0x7fdc040fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:03.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.289+0000 7fdc0ba4c640 1 -- 172.21.15.32:0/1979701946 shutdown_connections 2024-04-25T00:31:03.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.289+0000 7fdc0ba4c640 1 -- 172.21.15.32:0/1979701946 wait complete. 2024-04-25T00:31:03.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.289+0000 7fdc0ba4c640 1 Processor -- start 2024-04-25T00:31:03.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.289+0000 7fdc0ba4c640 1 -- start start 2024-04-25T00:31:03.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.290+0000 7fdc0ba4c640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdc041006a0 0x7fdc04197a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:03.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.290+0000 7fdc0ba4c640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdc04101890 0x7fdc04197fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:03.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.290+0000 7fdc0ba4c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdc04102250 0x7fdc0419f040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:03.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.290+0000 7fdc0ba4c640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fdc0410c320 con 0x7fdc04102250 2024-04-25T00:31:03.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.290+0000 7fdc0ba4c640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fdc0410c1a0 con 0x7fdc041006a0 2024-04-25T00:31:03.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.290+0000 7fdc0ba4c640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fdc0410c4a0 con 0x7fdc04101890 2024-04-25T00:31:03.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.290+0000 7fdc09fe9640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdc04102250 0x7fdc0419f040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:03.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.290+0000 7fdc09fe9640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdc04102250 0x7fdc0419f040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:60860/0 (socket says 172.21.15.32:60860) 2024-04-25T00:31:03.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.290+0000 7fdc09fe9640 1 -- 172.21.15.32:0/3104844163 learned_addr learned my addr 172.21.15.32:0/3104844163 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:03.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.290+0000 7fdc08fe7640 1 --2- 172.21.15.32:0/3104844163 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdc04101890 0x7fdc04197fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:03.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.290+0000 7fdc097e8640 1 --2- 172.21.15.32:0/3104844163 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdc041006a0 0x7fdc04197a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:03.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.290+0000 7fdc09fe9640 1 -- 172.21.15.32:0/3104844163 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdc04101890 msgr2=0x7fdc04197fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:03.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.290+0000 7fdc09fe9640 1 --2- 172.21.15.32:0/3104844163 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdc04101890 0x7fdc04197fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.290+0000 7fdc09fe9640 1 -- 172.21.15.32:0/3104844163 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdc041006a0 msgr2=0x7fdc04197a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:03.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.290+0000 7fdc09fe9640 1 --2- 172.21.15.32:0/3104844163 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdc041006a0 0x7fdc04197a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.290+0000 7fdc09fe9640 1 -- 172.21.15.32:0/3104844163 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fdc0419f6b0 con 0x7fdc04102250 2024-04-25T00:31:03.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.290+0000 7fdc097e8640 1 --2- 172.21.15.32:0/3104844163 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdc041006a0 0x7fdc04197a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:31:03.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.290+0000 7fdc09fe9640 1 --2- 172.21.15.32:0/3104844163 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdc04102250 0x7fdc0419f040 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7fdbfc00eaa0 tx=0x7fdbfc00eea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:03.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.290+0000 7fdbfa7fc640 1 -- 172.21.15.32:0/3104844163 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdbfc004d20 con 0x7fdc04102250 2024-04-25T00:31:03.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.290+0000 7fdc0ba4c640 1 -- 172.21.15.32:0/3104844163 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fdc0419f9a0 con 0x7fdc04102250 2024-04-25T00:31:03.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.290+0000 7fdbfa7fc640 1 -- 172.21.15.32:0/3104844163 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fdbfc005020 con 0x7fdc04102250 2024-04-25T00:31:03.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.290+0000 7fdbfa7fc640 1 -- 172.21.15.32:0/3104844163 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdbfc0116d0 con 0x7fdc04102250 2024-04-25T00:31:03.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.290+0000 7fdc0ba4c640 1 -- 172.21.15.32:0/3104844163 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fdc0419ff40 con 0x7fdc04102250 2024-04-25T00:31:03.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.291+0000 7fdc0ba4c640 1 -- 172.21.15.32:0/3104844163 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fdc041a0170 con 0x7fdc04102250 2024-04-25T00:31:03.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.292+0000 7fdbfa7fc640 1 -- 172.21.15.32:0/3104844163 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7fdbfc0051d0 con 0x7fdc04102250 2024-04-25T00:31:03.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.293+0000 7fdbfa7fc640 1 --2- 172.21.15.32:0/3104844163 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdbf00a2b90 0x7fdbf00a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:03.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.293+0000 7fdc097e8640 1 --2- 172.21.15.32:0/3104844163 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdbf00a2b90 0x7fdbf00a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:03.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.293+0000 7fdbfa7fc640 1 -- 172.21.15.32:0/3104844163 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7fdbfc0cf490 con 0x7fdc04102250 2024-04-25T00:31:03.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.294+0000 7fdc097e8640 1 --2- 172.21.15.32:0/3104844163 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdbf00a2b90 0x7fdbf00a5050 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7fdc00006010 tx=0x7fdc00032dd0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:03.298 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.298+0000 7fdbfa7fc640 1 -- 172.21.15.32:0/3104844163 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fdbfc016030 con 0x7fdc04102250 2024-04-25T00:31:03.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:02 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2465606290' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:31:03.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:02 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3111745697' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-25T00:31:03.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:02 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1088771809' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:31:03.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:02 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2465606290' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:31:03.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:02 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3111745697' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-25T00:31:03.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:02 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1088771809' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:31:03.480 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.480+0000 7fdc0ba4c640 1 -- 172.21.15.32:0/3104844163 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) -- 0x7fdc041a0340 con 0x7fdc04102250 2024-04-25T00:31:03.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.480+0000 7fdbfa7fc640 1 -- 172.21.15.32:0/3104844163 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v89) ==== 104+0+11 (secure 0 0 0) 0x7fdbfc09c1b0 con 0x7fdc04102250 2024-04-25T00:31:03.481 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 32 2024-04-25T00:31:03.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.482+0000 7fdc0ba4c640 1 -- 172.21.15.32:0/3104844163 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdbf00a2b90 msgr2=0x7fdbf00a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:03.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.482+0000 7fdc0ba4c640 1 --2- 172.21.15.32:0/3104844163 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdbf00a2b90 0x7fdbf00a5050 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7fdc00006010 tx=0x7fdc00032dd0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.482+0000 7fdc0ba4c640 1 -- 172.21.15.32:0/3104844163 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdc04102250 msgr2=0x7fdc0419f040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:03.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.482+0000 7fdc0ba4c640 1 --2- 172.21.15.32:0/3104844163 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdc04102250 0x7fdc0419f040 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7fdbfc00eaa0 tx=0x7fdbfc00eea0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.482+0000 7fdc0ba4c640 1 -- 172.21.15.32:0/3104844163 shutdown_connections 2024-04-25T00:31:03.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.482+0000 7fdc0ba4c640 1 --2- 172.21.15.32:0/3104844163 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdbf00a2b90 0x7fdbf00a5050 unknown :-1 s=CLOSED pgs=172 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.482+0000 7fdc0ba4c640 1 --2- 172.21.15.32:0/3104844163 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdc04102250 0x7fdc0419f040 unknown :-1 s=CLOSED pgs=248 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.482+0000 7fdc0ba4c640 1 --2- 172.21.15.32:0/3104844163 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdc04101890 0x7fdc04197fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.482+0000 7fdc0ba4c640 1 --2- 172.21.15.32:0/3104844163 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdc041006a0 0x7fdc04197a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.482+0000 7fdc0ba4c640 1 -- 172.21.15.32:0/3104844163 >> 172.21.15.32:0/3104844163 conn(0x7fdc040fbc30 msgr2=0x7fdc040fd6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:03.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.482+0000 7fdc0ba4c640 1 -- 172.21.15.32:0/3104844163 shutdown_connections 2024-04-25T00:31:03.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.482+0000 7fdc0ba4c640 1 -- 172.21.15.32:0/3104844163 wait complete. 2024-04-25T00:31:03.493 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-04-25T00:31:03.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.598+0000 7f2871ef5640 1 -- 172.21.15.32:0/1804022590 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f286c101870 msgr2=0x7f286c101cf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:03.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.598+0000 7f2871ef5640 1 --2- 172.21.15.32:0/1804022590 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f286c101870 0x7f286c101cf0 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7f2854009900 tx=0x7f285402a410 comp rx=0 tx=0).stop 2024-04-25T00:31:03.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.598+0000 7f2871ef5640 1 -- 172.21.15.32:0/1804022590 shutdown_connections 2024-04-25T00:31:03.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.598+0000 7f2871ef5640 1 --2- 172.21.15.32:0/1804022590 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f286c102230 0x7f286c108ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.598+0000 7f2871ef5640 1 --2- 172.21.15.32:0/1804022590 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f286c101870 0x7f286c101cf0 unknown :-1 s=CLOSED pgs=249 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.598+0000 7f2871ef5640 1 --2- 172.21.15.32:0/1804022590 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f286c100680 0x7f286c100aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.598+0000 7f2871ef5640 1 -- 172.21.15.32:0/1804022590 >> 172.21.15.32:0/1804022590 conn(0x7f286c0fbbf0 msgr2=0x7f286c0fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:03.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.598+0000 7f2871ef5640 1 -- 172.21.15.32:0/1804022590 shutdown_connections 2024-04-25T00:31:03.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.598+0000 7f2871ef5640 1 -- 172.21.15.32:0/1804022590 wait complete. 2024-04-25T00:31:03.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.599+0000 7f2871ef5640 1 Processor -- start 2024-04-25T00:31:03.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.599+0000 7f2871ef5640 1 -- start start 2024-04-25T00:31:03.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.599+0000 7f2871ef5640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f286c100680 0x7f286c197b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:03.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.599+0000 7f2871ef5640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f286c101870 0x7f286c198060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:03.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.599+0000 7f2871ef5640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f286c102230 0x7f286c19f0d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:03.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.599+0000 7f2871ef5640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f286c10d430 con 0x7f286c101870 2024-04-25T00:31:03.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.599+0000 7f2871ef5640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f286c10d2b0 con 0x7f286c100680 2024-04-25T00:31:03.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.599+0000 7f2871ef5640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f286c10d5b0 con 0x7f286c102230 2024-04-25T00:31:03.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.599+0000 7f286affd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f286c101870 0x7f286c198060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:03.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.599+0000 7f286affd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f286c101870 0x7f286c198060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:60880/0 (socket says 172.21.15.32:60880) 2024-04-25T00:31:03.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.599+0000 7f286affd640 1 -- 172.21.15.32:0/2211883451 learned_addr learned my addr 172.21.15.32:0/2211883451 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:03.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.600+0000 7f286bfff640 1 --2- 172.21.15.32:0/2211883451 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f286c102230 0x7f286c19f0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:03.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.600+0000 7f286b7fe640 1 --2- 172.21.15.32:0/2211883451 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f286c100680 0x7f286c197b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:03.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.600+0000 7f286affd640 1 -- 172.21.15.32:0/2211883451 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f286c102230 msgr2=0x7f286c19f0d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:03.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.600+0000 7f286affd640 1 --2- 172.21.15.32:0/2211883451 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f286c102230 0x7f286c19f0d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.600+0000 7f286affd640 1 -- 172.21.15.32:0/2211883451 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f286c100680 msgr2=0x7f286c197b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:03.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.600+0000 7f286affd640 1 --2- 172.21.15.32:0/2211883451 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f286c100680 0x7f286c197b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.600+0000 7f286affd640 1 -- 172.21.15.32:0/2211883451 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f286c19f740 con 0x7f286c101870 2024-04-25T00:31:03.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.600+0000 7f286b7fe640 1 --2- 172.21.15.32:0/2211883451 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f286c100680 0x7f286c197b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:31:03.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.600+0000 7f286affd640 1 --2- 172.21.15.32:0/2211883451 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f286c101870 0x7f286c198060 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7f2854009230 tx=0x7f2854004360 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:03.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.600+0000 7f2868ff9640 1 -- 172.21.15.32:0/2211883451 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f285403f030 con 0x7f286c101870 2024-04-25T00:31:03.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.600+0000 7f2871ef5640 1 -- 172.21.15.32:0/2211883451 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f286c19f9d0 con 0x7f286c101870 2024-04-25T00:31:03.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.600+0000 7f2868ff9640 1 -- 172.21.15.32:0/2211883451 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f2854003ce0 con 0x7f286c101870 2024-04-25T00:31:03.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.600+0000 7f2868ff9640 1 -- 172.21.15.32:0/2211883451 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2854035070 con 0x7f286c101870 2024-04-25T00:31:03.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.600+0000 7f2871ef5640 1 -- 172.21.15.32:0/2211883451 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f286c19ff10 con 0x7f286c101870 2024-04-25T00:31:03.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.601+0000 7f2871ef5640 1 -- 172.21.15.32:0/2211883451 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f286c107a90 con 0x7f286c101870 2024-04-25T00:31:03.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.602+0000 7f2868ff9640 1 -- 172.21.15.32:0/2211883451 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f2854042030 con 0x7f286c101870 2024-04-25T00:31:03.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.603+0000 7f2868ff9640 1 --2- 172.21.15.32:0/2211883451 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f28580a2b90 0x7f28580a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:03.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.603+0000 7f286b7fe640 1 --2- 172.21.15.32:0/2211883451 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f28580a2b90 0x7f28580a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:03.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.603+0000 7f2868ff9640 1 -- 172.21.15.32:0/2211883451 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f28540efcd0 con 0x7f286c101870 2024-04-25T00:31:03.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.603+0000 7f286b7fe640 1 --2- 172.21.15.32:0/2211883451 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f28580a2b90 0x7f28580a5050 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f286c1016d0 tx=0x7f285c009400 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:03.610 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.607+0000 7f2868ff9640 1 -- 172.21.15.32:0/2211883451 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f28540bc990 con 0x7f286c101870 2024-04-25T00:31:03.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.793+0000 7f2871ef5640 1 -- 172.21.15.32:0/2211883451 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) -- 0x7f286c062340 con 0x7f286c101870 2024-04-25T00:31:03.794 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.793+0000 7f2868ff9640 1 -- 172.21.15.32:0/2211883451 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v89) ==== 100+0+11 (secure 0 0 0) 0x7f28540bc290 con 0x7f286c101870 2024-04-25T00:31:03.794 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 64 2024-04-25T00:31:03.796 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.796+0000 7f2871ef5640 1 -- 172.21.15.32:0/2211883451 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f28580a2b90 msgr2=0x7f28580a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:03.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.796+0000 7f2871ef5640 1 --2- 172.21.15.32:0/2211883451 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f28580a2b90 0x7f28580a5050 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f286c1016d0 tx=0x7f285c009400 comp rx=0 tx=0).stop 2024-04-25T00:31:03.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.796+0000 7f2871ef5640 1 -- 172.21.15.32:0/2211883451 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f286c101870 msgr2=0x7f286c198060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:03.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.796+0000 7f2871ef5640 1 --2- 172.21.15.32:0/2211883451 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f286c101870 0x7f286c198060 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7f2854009230 tx=0x7f2854004360 comp rx=0 tx=0).stop 2024-04-25T00:31:03.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.796+0000 7f2871ef5640 1 -- 172.21.15.32:0/2211883451 shutdown_connections 2024-04-25T00:31:03.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.796+0000 7f2871ef5640 1 --2- 172.21.15.32:0/2211883451 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f28580a2b90 0x7f28580a5050 unknown :-1 s=CLOSED pgs=173 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.796+0000 7f2871ef5640 1 --2- 172.21.15.32:0/2211883451 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f286c102230 0x7f286c19f0d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.796+0000 7f2871ef5640 1 --2- 172.21.15.32:0/2211883451 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f286c101870 0x7f286c198060 unknown :-1 s=CLOSED pgs=250 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.796+0000 7f2871ef5640 1 --2- 172.21.15.32:0/2211883451 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f286c100680 0x7f286c197b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.796+0000 7f2871ef5640 1 -- 172.21.15.32:0/2211883451 >> 172.21.15.32:0/2211883451 conn(0x7f286c0fbbf0 msgr2=0x7f286c104c60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:03.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.796+0000 7f2871ef5640 1 -- 172.21.15.32:0/2211883451 shutdown_connections 2024-04-25T00:31:03.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.796+0000 7f2871ef5640 1 -- 172.21.15.32:0/2211883451 wait complete. 2024-04-25T00:31:03.807 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data_ec pg_num 2024-04-25T00:31:03.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.911+0000 7feaae363640 1 -- 172.21.15.32:0/1986681859 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7feaa81006a0 msgr2=0x7feaa8100ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:03.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.911+0000 7feaae363640 1 --2- 172.21.15.32:0/1986681859 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7feaa81006a0 0x7feaa8100ac0 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7fea98009900 tx=0x7fea9802a410 comp rx=0 tx=0).stop 2024-04-25T00:31:03.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.911+0000 7feaae363640 1 -- 172.21.15.32:0/1986681859 shutdown_connections 2024-04-25T00:31:03.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.911+0000 7feaae363640 1 --2- 172.21.15.32:0/1986681859 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7feaa8102250 0x7feaa8108ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.911+0000 7feaae363640 1 --2- 172.21.15.32:0/1986681859 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7feaa8101890 0x7feaa8101d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.911+0000 7feaae363640 1 --2- 172.21.15.32:0/1986681859 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7feaa81006a0 0x7feaa8100ac0 unknown :-1 s=CLOSED pgs=251 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.911+0000 7feaae363640 1 -- 172.21.15.32:0/1986681859 >> 172.21.15.32:0/1986681859 conn(0x7feaa80fbc30 msgr2=0x7feaa80fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:03.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.911+0000 7feaae363640 1 -- 172.21.15.32:0/1986681859 shutdown_connections 2024-04-25T00:31:03.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.911+0000 7feaae363640 1 -- 172.21.15.32:0/1986681859 wait complete. 2024-04-25T00:31:03.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.912+0000 7feaae363640 1 Processor -- start 2024-04-25T00:31:03.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.912+0000 7feaae363640 1 -- start start 2024-04-25T00:31:03.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.912+0000 7feaae363640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7feaa81006a0 0x7feaa8197ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:03.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.912+0000 7feaae363640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7feaa8101890 0x7feaa81980e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:03.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.912+0000 7feaae363640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7feaa8102250 0x7feaa819f100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:03.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.912+0000 7feaae363640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7feaa810c320 con 0x7feaa81006a0 2024-04-25T00:31:03.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.912+0000 7feaae363640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7feaa810c1a0 con 0x7feaa8101890 2024-04-25T00:31:03.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.912+0000 7feaae363640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7feaa810c4a0 con 0x7feaa8102250 2024-04-25T00:31:03.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.912+0000 7feaa7fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7feaa81006a0 0x7feaa8197ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:03.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.912+0000 7feaa7fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7feaa81006a0 0x7feaa8197ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:60908/0 (socket says 172.21.15.32:60908) 2024-04-25T00:31:03.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.912+0000 7feaa7fff640 1 -- 172.21.15.32:0/3988014647 learned_addr learned my addr 172.21.15.32:0/3988014647 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:03.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.912+0000 7feaac900640 1 --2- 172.21.15.32:0/3988014647 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7feaa8102250 0x7feaa819f100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:03.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.912+0000 7feaa77fe640 1 --2- 172.21.15.32:0/3988014647 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7feaa8101890 0x7feaa81980e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:03.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.913+0000 7feaa7fff640 1 -- 172.21.15.32:0/3988014647 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7feaa8102250 msgr2=0x7feaa819f100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:03.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.913+0000 7feaa7fff640 1 --2- 172.21.15.32:0/3988014647 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7feaa8102250 0x7feaa819f100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.913+0000 7feaa7fff640 1 -- 172.21.15.32:0/3988014647 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7feaa8101890 msgr2=0x7feaa81980e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:03.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.913+0000 7feaa7fff640 1 --2- 172.21.15.32:0/3988014647 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7feaa8101890 0x7feaa81980e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:03.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.913+0000 7feaa7fff640 1 -- 172.21.15.32:0/3988014647 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7feaa819f770 con 0x7feaa81006a0 2024-04-25T00:31:03.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.913+0000 7feaa77fe640 1 --2- 172.21.15.32:0/3988014647 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7feaa8101890 0x7feaa81980e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:31:03.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.913+0000 7feaa7fff640 1 --2- 172.21.15.32:0/3988014647 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7feaa81006a0 0x7feaa8197ba0 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7fea98031610 tx=0x7fea98004270 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:03.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.913+0000 7feaa57fa640 1 -- 172.21.15.32:0/3988014647 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fea9803f030 con 0x7feaa81006a0 2024-04-25T00:31:03.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.913+0000 7feaae363640 1 -- 172.21.15.32:0/3988014647 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7feaa819fa00 con 0x7feaa81006a0 2024-04-25T00:31:03.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.913+0000 7feaa57fa640 1 -- 172.21.15.32:0/3988014647 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fea98003ad0 con 0x7feaa81006a0 2024-04-25T00:31:03.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.913+0000 7feaa57fa640 1 -- 172.21.15.32:0/3988014647 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fea98035070 con 0x7feaa81006a0 2024-04-25T00:31:03.917 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.913+0000 7feaae363640 1 -- 172.21.15.32:0/3988014647 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7feaa819fef0 con 0x7feaa81006a0 2024-04-25T00:31:03.917 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.914+0000 7feaae363640 1 -- 172.21.15.32:0/3988014647 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7feaa81a00c0 con 0x7feaa81006a0 2024-04-25T00:31:03.917 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.915+0000 7feaa57fa640 1 -- 172.21.15.32:0/3988014647 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7fea98042030 con 0x7feaa81006a0 2024-04-25T00:31:03.918 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.916+0000 7feaa57fa640 1 --2- 172.21.15.32:0/3988014647 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fea900a2b90 0x7fea900a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:03.918 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.916+0000 7feaa77fe640 1 --2- 172.21.15.32:0/3988014647 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fea900a2b90 0x7fea900a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:03.918 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.916+0000 7feaa57fa640 1 -- 172.21.15.32:0/3988014647 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7fea980ef8c0 con 0x7feaa81006a0 2024-04-25T00:31:03.918 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.916+0000 7feaa77fe640 1 --2- 172.21.15.32:0/3988014647 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fea900a2b90 0x7fea900a5050 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7feaa8199060 tx=0x7fea9c005f90 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:03.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:03.920+0000 7feaa57fa640 1 -- 172.21.15.32:0/3988014647 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fea980bc5e0 con 0x7feaa81006a0 2024-04-25T00:31:04.106 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.106+0000 7feaae363640 1 -- 172.21.15.32:0/3988014647 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"} v 0) -- 0x7feaa81a0290 con 0x7feaa81006a0 2024-04-25T00:31:04.107 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.106+0000 7feaa57fa640 1 -- 172.21.15.32:0/3988014647 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]=0 v89) ==== 103+0+11 (secure 0 0 0) 0x7fea980bbee0 con 0x7feaa81006a0 2024-04-25T00:31:04.107 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 32 2024-04-25T00:31:04.109 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.108+0000 7feaae363640 1 -- 172.21.15.32:0/3988014647 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fea900a2b90 msgr2=0x7fea900a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:04.109 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.108+0000 7feaae363640 1 --2- 172.21.15.32:0/3988014647 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fea900a2b90 0x7fea900a5050 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7feaa8199060 tx=0x7fea9c005f90 comp rx=0 tx=0).stop 2024-04-25T00:31:04.109 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.109+0000 7feaae363640 1 -- 172.21.15.32:0/3988014647 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7feaa81006a0 msgr2=0x7feaa8197ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:04.109 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.109+0000 7feaae363640 1 --2- 172.21.15.32:0/3988014647 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7feaa81006a0 0x7feaa8197ba0 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7fea98031610 tx=0x7fea98004270 comp rx=0 tx=0).stop 2024-04-25T00:31:04.109 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.109+0000 7feaae363640 1 -- 172.21.15.32:0/3988014647 shutdown_connections 2024-04-25T00:31:04.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.109+0000 7feaae363640 1 --2- 172.21.15.32:0/3988014647 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fea900a2b90 0x7fea900a5050 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:04.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.109+0000 7feaae363640 1 --2- 172.21.15.32:0/3988014647 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7feaa8102250 0x7feaa819f100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:04.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.109+0000 7feaae363640 1 --2- 172.21.15.32:0/3988014647 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7feaa8101890 0x7feaa81980e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:04.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.109+0000 7feaae363640 1 --2- 172.21.15.32:0/3988014647 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7feaa81006a0 0x7feaa8197ba0 unknown :-1 s=CLOSED pgs=252 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:04.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.109+0000 7feaae363640 1 -- 172.21.15.32:0/3988014647 >> 172.21.15.32:0/3988014647 conn(0x7feaa80fbc30 msgr2=0x7feaa80fd730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:04.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.109+0000 7feaae363640 1 -- 172.21.15.32:0/3988014647 shutdown_connections 2024-04-25T00:31:04.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.109+0000 7feaae363640 1 -- 172.21.15.32:0/3988014647 wait complete. 2024-04-25T00:31:04.119 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:03 smithi032 ceph-mon[29357]: pgmap v299: 129 pgs: 129 active+clean; 581 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 307 B/s wr, 0 op/s 2024-04-25T00:31:04.119 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:03 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2617296648' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:31:04.119 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:03 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3104844163' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:31:04.119 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:03 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2211883451' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:31:04.120 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-04-25T00:31:04.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:03 smithi130 ceph-mon[32525]: pgmap v299: 129 pgs: 129 active+clean; 581 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 307 B/s wr, 0 op/s 2024-04-25T00:31:04.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:03 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2617296648' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:31:04.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:03 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3104844163' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:31:04.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:03 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2211883451' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:31:04.224 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.223+0000 7f76a8012640 1 -- 172.21.15.32:0/2705810075 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f76a0107520 msgr2=0x7f76a0109910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:04.224 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.223+0000 7f76a8012640 1 --2- 172.21.15.32:0/2705810075 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f76a0107520 0x7f76a0109910 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f7698009ca0 tx=0x7f769802a8c0 comp rx=0 tx=0).stop 2024-04-25T00:31:04.224 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.223+0000 7f76a8012640 1 -- 172.21.15.32:0/2705810075 shutdown_connections 2024-04-25T00:31:04.224 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.223+0000 7f76a8012640 1 --2- 172.21.15.32:0/2705810075 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f76a0107520 0x7f76a0109910 unknown :-1 s=CLOSED pgs=253 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:04.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.223+0000 7f76a8012640 1 --2- 172.21.15.32:0/2705810075 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f76a0104bf0 0x7f76a0106fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:04.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.223+0000 7f76a8012640 1 --2- 172.21.15.32:0/2705810075 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f76a01022c0 0x7f76a01046b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:04.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.223+0000 7f76a8012640 1 -- 172.21.15.32:0/2705810075 >> 172.21.15.32:0/2705810075 conn(0x7f76a00fbbf0 msgr2=0x7f76a00fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:04.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.224+0000 7f76a8012640 1 -- 172.21.15.32:0/2705810075 shutdown_connections 2024-04-25T00:31:04.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.224+0000 7f76a8012640 1 -- 172.21.15.32:0/2705810075 wait complete. 2024-04-25T00:31:04.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.224+0000 7f76a8012640 1 Processor -- start 2024-04-25T00:31:04.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.224+0000 7f76a8012640 1 -- start start 2024-04-25T00:31:04.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.224+0000 7f76a8012640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f76a01022c0 0x7f76a0101c30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:04.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.224+0000 7f76a8012640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f76a0104bf0 0x7f76a0100270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:04.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.224+0000 7f76a8012640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f76a0107520 0x7f76a01007b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:04.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.224+0000 7f76a8012640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f76a010d480 con 0x7f76a01022c0 2024-04-25T00:31:04.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.224+0000 7f76a8012640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f76a010d300 con 0x7f76a0107520 2024-04-25T00:31:04.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.224+0000 7f76a8012640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f76a010d600 con 0x7f76a0104bf0 2024-04-25T00:31:04.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.224+0000 7f76a5dae640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f76a01022c0 0x7f76a0101c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:04.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.224+0000 7f76a5dae640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f76a01022c0 0x7f76a0101c30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:60928/0 (socket says 172.21.15.32:60928) 2024-04-25T00:31:04.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.224+0000 7f76a5dae640 1 -- 172.21.15.32:0/493287235 learned_addr learned my addr 172.21.15.32:0/493287235 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:04.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.225+0000 7f76a5dae640 1 -- 172.21.15.32:0/493287235 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f76a0104bf0 msgr2=0x7f76a0100270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:04.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.225+0000 7f76a5dae640 1 --2- 172.21.15.32:0/493287235 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f76a0104bf0 0x7f76a0100270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:04.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.225+0000 7f76a5dae640 1 -- 172.21.15.32:0/493287235 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f76a0107520 msgr2=0x7f76a01007b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:04.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.225+0000 7f76a5dae640 1 --2- 172.21.15.32:0/493287235 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f76a0107520 0x7f76a01007b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:04.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.225+0000 7f76a5dae640 1 -- 172.21.15.32:0/493287235 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f76a0101040 con 0x7f76a01022c0 2024-04-25T00:31:04.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.225+0000 7f76a5dae640 1 --2- 172.21.15.32:0/493287235 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f76a01022c0 0x7f76a0101c30 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f769c00c970 tx=0x7f769c00ce30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:04.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.225+0000 7f7696ffd640 1 -- 172.21.15.32:0/493287235 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f769c007bb0 con 0x7f76a01022c0 2024-04-25T00:31:04.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.225+0000 7f7696ffd640 1 -- 172.21.15.32:0/493287235 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f769c007d50 con 0x7f76a01022c0 2024-04-25T00:31:04.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.225+0000 7f76a8012640 1 -- 172.21.15.32:0/493287235 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f76a01a1be0 con 0x7f76a01022c0 2024-04-25T00:31:04.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.225+0000 7f7696ffd640 1 -- 172.21.15.32:0/493287235 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f769c00f450 con 0x7f76a01022c0 2024-04-25T00:31:04.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.225+0000 7f76a8012640 1 -- 172.21.15.32:0/493287235 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f76a01a20e0 con 0x7f76a01022c0 2024-04-25T00:31:04.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.226+0000 7f76a8012640 1 -- 172.21.15.32:0/493287235 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f76a00fd6e0 con 0x7f76a01022c0 2024-04-25T00:31:04.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.227+0000 7f7696ffd640 1 -- 172.21.15.32:0/493287235 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f769c004fe0 con 0x7f76a01022c0 2024-04-25T00:31:04.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.228+0000 7f7696ffd640 1 --2- 172.21.15.32:0/493287235 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f768c0a2b90 0x7f768c0a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:04.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.228+0000 7f76a55ad640 1 --2- 172.21.15.32:0/493287235 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f768c0a2b90 0x7f768c0a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:04.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.228+0000 7f7696ffd640 1 -- 172.21.15.32:0/493287235 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f769c0cf520 con 0x7f76a01022c0 2024-04-25T00:31:04.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.228+0000 7f76a55ad640 1 --2- 172.21.15.32:0/493287235 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f768c0a2b90 0x7f768c0a5050 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f7690005fd0 tx=0x7f7690005c70 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:04.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.231+0000 7f7696ffd640 1 -- 172.21.15.32:0/493287235 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f769c014030 con 0x7f76a01022c0 2024-04-25T00:31:04.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:03 smithi121 ceph-mon[32231]: pgmap v299: 129 pgs: 129 active+clean; 581 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 307 B/s wr, 0 op/s 2024-04-25T00:31:04.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:03 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2617296648' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:31:04.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:03 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3104844163' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:31:04.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:03 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2211883451' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:31:04.430 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.430+0000 7f76a8012640 1 -- 172.21.15.32:0/493287235 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) -- 0x7f76a002d170 con 0x7f76a01022c0 2024-04-25T00:31:04.431 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.430+0000 7f7696ffd640 1 -- 172.21.15.32:0/493287235 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 10 entries v89) ==== 81+0+590 (secure 0 0 0) 0x7f769c09c240 con 0x7f76a01022c0 2024-04-25T00:31:04.431 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:0/2999636731 2024-04-26T00:23:42.776439+0000 2024-04-25T00:31:04.431 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:6801/4160192081 2024-04-26T00:23:42.776439+0000 2024-04-25T00:31:04.431 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:6800/4160192081 2024-04-26T00:23:42.776439+0000 2024-04-25T00:31:04.431 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:0/53550767 2024-04-26T00:23:42.776439+0000 2024-04-25T00:31:04.431 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:0/1066471448 2024-04-26T00:23:14.613921+0000 2024-04-25T00:31:04.431 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:0/3987019264 2024-04-26T00:23:14.613921+0000 2024-04-25T00:31:04.431 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:0/3041942084 2024-04-26T00:23:14.613921+0000 2024-04-25T00:31:04.431 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:0/3004184616 2024-04-26T00:23:42.776439+0000 2024-04-25T00:31:04.431 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:6800/1637256323 2024-04-26T00:23:14.613921+0000 2024-04-25T00:31:04.432 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:6801/1637256323 2024-04-26T00:23:14.613921+0000 2024-04-25T00:31:04.432 INFO:teuthology.orchestra.run.smithi032.stderr:listed 10 entries 2024-04-25T00:31:04.432 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.432+0000 7f76a8012640 1 -- 172.21.15.32:0/493287235 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f768c0a2b90 msgr2=0x7f768c0a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:04.433 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.432+0000 7f76a8012640 1 --2- 172.21.15.32:0/493287235 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f768c0a2b90 0x7f768c0a5050 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f7690005fd0 tx=0x7f7690005c70 comp rx=0 tx=0).stop 2024-04-25T00:31:04.433 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.432+0000 7f76a8012640 1 -- 172.21.15.32:0/493287235 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f76a01022c0 msgr2=0x7f76a0101c30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:04.433 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.432+0000 7f76a8012640 1 --2- 172.21.15.32:0/493287235 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f76a01022c0 0x7f76a0101c30 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f769c00c970 tx=0x7f769c00ce30 comp rx=0 tx=0).stop 2024-04-25T00:31:04.433 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.432+0000 7f76a8012640 1 -- 172.21.15.32:0/493287235 shutdown_connections 2024-04-25T00:31:04.433 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.432+0000 7f76a8012640 1 --2- 172.21.15.32:0/493287235 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f768c0a2b90 0x7f768c0a5050 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:04.433 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.432+0000 7f76a8012640 1 --2- 172.21.15.32:0/493287235 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f76a0107520 0x7f76a01007b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:04.433 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.432+0000 7f76a8012640 1 --2- 172.21.15.32:0/493287235 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f76a0104bf0 0x7f76a0100270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:04.433 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.432+0000 7f76a8012640 1 --2- 172.21.15.32:0/493287235 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f76a01022c0 0x7f76a0101c30 unknown :-1 s=CLOSED pgs=254 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:04.433 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.432+0000 7f76a8012640 1 -- 172.21.15.32:0/493287235 >> 172.21.15.32:0/493287235 conn(0x7f76a00fbbf0 msgr2=0x7f76a0102c00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:04.433 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.432+0000 7f76a8012640 1 -- 172.21.15.32:0/493287235 shutdown_connections 2024-04-25T00:31:04.433 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:04.432+0000 7f76a8012640 1 -- 172.21.15.32:0/493287235 wait complete. 2024-04-25T00:31:04.443 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on ubuntu@smithi032.front.sepia.ceph.com... 2024-04-25T00:31:04.444 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:31:04.444 DEBUG:teuthology.orchestra.run.smithi032:> sudo fusermount -u /home/ubuntu/cephtest/mnt.admin 2024-04-25T00:31:04.466 INFO:tasks.cephfs.fuse_mount.ceph-fuse.admin.smithi032.stderr:ceph-fuse[73401]: fuse finished with error 0 and tester_r 0 2024-04-25T00:31:04.468 INFO:teuthology.orchestra.run:waiting for 300 2024-04-25T00:31:05.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:04 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3988014647' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:31:05.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:04 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/493287235' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-25T00:31:05.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:04 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3988014647' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:31:05.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:04 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/493287235' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-25T00:31:05.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:04 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3988014647' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:31:05.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:04 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/493287235' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-25T00:31:06.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:05 smithi121 ceph-mon[32231]: pgmap v300: 129 pgs: 129 active+clean; 581 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 276 B/s wr, 0 op/s 2024-04-25T00:31:06.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:05 smithi032 ceph-mon[29357]: pgmap v300: 129 pgs: 129 active+clean; 581 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 276 B/s wr, 0 op/s 2024-04-25T00:31:06.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:05 smithi130 ceph-mon[32525]: pgmap v300: 129 pgs: 129 active+clean; 581 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 276 B/s wr, 0 op/s 2024-04-25T00:31:08.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:07 smithi032 ceph-mon[29357]: pgmap v301: 129 pgs: 129 active+clean; 581 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 255 B/s wr, 0 op/s 2024-04-25T00:31:08.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:07 smithi121 ceph-mon[32231]: pgmap v301: 129 pgs: 129 active+clean; 581 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 255 B/s wr, 0 op/s 2024-04-25T00:31:08.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:07 smithi130 ceph-mon[32525]: pgmap v301: 129 pgs: 129 active+clean; 581 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 255 B/s wr, 0 op/s 2024-04-25T00:31:10.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:09 smithi032 ceph-mon[29357]: pgmap v302: 129 pgs: 129 active+clean; 581 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 426 B/s wr, 0 op/s 2024-04-25T00:31:10.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:09 smithi121 ceph-mon[32231]: pgmap v302: 129 pgs: 129 active+clean; 581 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 426 B/s wr, 0 op/s 2024-04-25T00:31:10.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:09 smithi130 ceph-mon[32525]: pgmap v302: 129 pgs: 129 active+clean; 581 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 426 B/s wr, 0 op/s 2024-04-25T00:31:10.570 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi032.front.sepia.ceph.com 2024-04-25T00:31:10.570 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:31:10.570 DEBUG:teuthology.orchestra.run.smithi032:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.admin) 2024-04-25T00:31:10.608 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-25T00:31:10.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.763+0000 7fc611429640 1 -- 172.21.15.32:0/2425386005 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc60c1006a0 msgr2=0x7fc60c100ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:10.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.763+0000 7fc611429640 1 --2- 172.21.15.32:0/2425386005 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc60c1006a0 0x7fc60c100ac0 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7fc5fc009900 tx=0x7fc5fc02a410 comp rx=0 tx=0).stop 2024-04-25T00:31:10.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.763+0000 7fc611429640 1 -- 172.21.15.32:0/2425386005 shutdown_connections 2024-04-25T00:31:10.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.763+0000 7fc611429640 1 --2- 172.21.15.32:0/2425386005 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc60c102250 0x7fc60c108ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:10.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.763+0000 7fc611429640 1 --2- 172.21.15.32:0/2425386005 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc60c101890 0x7fc60c101d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:10.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.763+0000 7fc611429640 1 --2- 172.21.15.32:0/2425386005 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc60c1006a0 0x7fc60c100ac0 unknown :-1 s=CLOSED pgs=255 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:10.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.763+0000 7fc611429640 1 -- 172.21.15.32:0/2425386005 >> 172.21.15.32:0/2425386005 conn(0x7fc60c0fbc30 msgr2=0x7fc60c0fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:10.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.763+0000 7fc611429640 1 -- 172.21.15.32:0/2425386005 shutdown_connections 2024-04-25T00:31:10.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.763+0000 7fc611429640 1 -- 172.21.15.32:0/2425386005 wait complete. 2024-04-25T00:31:10.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.763+0000 7fc611429640 1 Processor -- start 2024-04-25T00:31:10.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.763+0000 7fc611429640 1 -- start start 2024-04-25T00:31:10.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.763+0000 7fc611429640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc60c1006a0 0x7fc60c1969c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:10.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.763+0000 7fc611429640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc60c101890 0x7fc60c196f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:10.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.763+0000 7fc611429640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc60c102250 0x7fc60c19df70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:10.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.764+0000 7fc611429640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fc60c10c320 con 0x7fc60c102250 2024-04-25T00:31:10.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.764+0000 7fc611429640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fc60c10c1a0 con 0x7fc60c1006a0 2024-04-25T00:31:10.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.764+0000 7fc611429640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fc60c10c4a0 con 0x7fc60c101890 2024-04-25T00:31:10.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.764+0000 7fc60b7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc60c102250 0x7fc60c19df70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:10.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.764+0000 7fc60b7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc60c102250 0x7fc60c19df70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:60936/0 (socket says 172.21.15.32:60936) 2024-04-25T00:31:10.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.764+0000 7fc60b7fe640 1 -- 172.21.15.32:0/178770114 learned_addr learned my addr 172.21.15.32:0/178770114 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:10.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.764+0000 7fc60affd640 1 --2- 172.21.15.32:0/178770114 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc60c1006a0 0x7fc60c1969c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:10.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.764+0000 7fc603fff640 1 --2- 172.21.15.32:0/178770114 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc60c101890 0x7fc60c196f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:10.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.764+0000 7fc60b7fe640 1 -- 172.21.15.32:0/178770114 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc60c101890 msgr2=0x7fc60c196f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:10.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.764+0000 7fc60b7fe640 1 --2- 172.21.15.32:0/178770114 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc60c101890 0x7fc60c196f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:10.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.764+0000 7fc60b7fe640 1 -- 172.21.15.32:0/178770114 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc60c1006a0 msgr2=0x7fc60c1969c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:10.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.764+0000 7fc60b7fe640 1 --2- 172.21.15.32:0/178770114 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc60c1006a0 0x7fc60c1969c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:10.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.764+0000 7fc60b7fe640 1 -- 172.21.15.32:0/178770114 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc60c19e5e0 con 0x7fc60c102250 2024-04-25T00:31:10.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.764+0000 7fc60b7fe640 1 --2- 172.21.15.32:0/178770114 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc60c102250 0x7fc60c19df70 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7fc60400eaa0 tx=0x7fc60400ef60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:10.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.764+0000 7fc608ff9640 1 -- 172.21.15.32:0/178770114 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc60400cdb0 con 0x7fc60c102250 2024-04-25T00:31:10.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.764+0000 7fc608ff9640 1 -- 172.21.15.32:0/178770114 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc604005020 con 0x7fc60c102250 2024-04-25T00:31:10.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.764+0000 7fc611429640 1 -- 172.21.15.32:0/178770114 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc60c19e8d0 con 0x7fc60c102250 2024-04-25T00:31:10.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.764+0000 7fc608ff9640 1 -- 172.21.15.32:0/178770114 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc6040116b0 con 0x7fc60c102250 2024-04-25T00:31:10.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.764+0000 7fc611429640 1 -- 172.21.15.32:0/178770114 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc60c19ee70 con 0x7fc60c102250 2024-04-25T00:31:10.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.765+0000 7fc611429640 1 -- 172.21.15.32:0/178770114 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc60c107a40 con 0x7fc60c102250 2024-04-25T00:31:10.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.766+0000 7fc608ff9640 1 -- 172.21.15.32:0/178770114 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7fc604005560 con 0x7fc60c102250 2024-04-25T00:31:10.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.767+0000 7fc608ff9640 1 --2- 172.21.15.32:0/178770114 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc5f80a2c10 0x7fc5f80a50d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:10.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.767+0000 7fc60affd640 1 --2- 172.21.15.32:0/178770114 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc5f80a2c10 0x7fc5f80a50d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:10.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.768+0000 7fc608ff9640 1 -- 172.21.15.32:0/178770114 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7fc6040cf3a0 con 0x7fc60c102250 2024-04-25T00:31:10.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.768+0000 7fc60affd640 1 --2- 172.21.15.32:0/178770114 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc5f80a2c10 0x7fc5f80a50d0 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7fc5fc005ab0 tx=0x7fc5fc000bc0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:10.772 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.772+0000 7fc608ff9640 1 -- 172.21.15.32:0/178770114 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fc604016030 con 0x7fc60c102250 2024-04-25T00:31:10.963 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.962+0000 7fc611429640 1 -- 172.21.15.32:0/178770114 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fc60c062340 con 0x7fc60c102250 2024-04-25T00:31:10.964 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.964+0000 7fc608ff9640 1 -- 172.21.15.32:0/178770114 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v89) ==== 74+0+24128 (secure 0 0 0) 0x7fc60409ba70 con 0x7fc60c102250 2024-04-25T00:31:10.965 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:31:10.965 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":89,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:30:53.134605+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-25T00:30:46.239556+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-25T00:30:47.257182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2024-04-25T00:30:51.193975+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":83,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-num-failure-domains":"0","crush-osds-per-failure-domain":"0","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:31:10.967 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.966+0000 7fc611429640 1 -- 172.21.15.32:0/178770114 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc5f80a2c10 msgr2=0x7fc5f80a50d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:10.967 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.966+0000 7fc611429640 1 --2- 172.21.15.32:0/178770114 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc5f80a2c10 0x7fc5f80a50d0 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7fc5fc005ab0 tx=0x7fc5fc000bc0 comp rx=0 tx=0).stop 2024-04-25T00:31:10.967 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.966+0000 7fc611429640 1 -- 172.21.15.32:0/178770114 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc60c102250 msgr2=0x7fc60c19df70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:10.967 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.966+0000 7fc611429640 1 --2- 172.21.15.32:0/178770114 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc60c102250 0x7fc60c19df70 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7fc60400eaa0 tx=0x7fc60400ef60 comp rx=0 tx=0).stop 2024-04-25T00:31:10.967 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.966+0000 7fc611429640 1 -- 172.21.15.32:0/178770114 shutdown_connections 2024-04-25T00:31:10.967 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.966+0000 7fc611429640 1 --2- 172.21.15.32:0/178770114 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc5f80a2c10 0x7fc5f80a50d0 unknown :-1 s=CLOSED pgs=176 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:10.967 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.966+0000 7fc611429640 1 --2- 172.21.15.32:0/178770114 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc60c102250 0x7fc60c19df70 unknown :-1 s=CLOSED pgs=256 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:10.967 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.966+0000 7fc611429640 1 --2- 172.21.15.32:0/178770114 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc60c101890 0x7fc60c196f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:10.967 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.966+0000 7fc611429640 1 --2- 172.21.15.32:0/178770114 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc60c1006a0 0x7fc60c1969c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:10.967 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.966+0000 7fc611429640 1 -- 172.21.15.32:0/178770114 >> 172.21.15.32:0/178770114 conn(0x7fc60c0fbc30 msgr2=0x7fc60c104c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:10.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.966+0000 7fc611429640 1 -- 172.21.15.32:0/178770114 shutdown_connections 2024-04-25T00:31:10.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:10.966+0000 7fc611429640 1 -- 172.21.15.32:0/178770114 wait complete. 2024-04-25T00:31:10.978 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-25T00:31:11.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.079+0000 7f82dfbee640 1 -- 172.21.15.32:0/288211648 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f82d8103360 msgr2=0x7f82d8109bf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:11.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.079+0000 7f82dfbee640 1 --2- 172.21.15.32:0/288211648 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f82d8103360 0x7f82d8109bf0 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7f82d0009ca0 tx=0x7f82d002a8c0 comp rx=0 tx=0).stop 2024-04-25T00:31:11.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.079+0000 7f82dfbee640 1 -- 172.21.15.32:0/288211648 shutdown_connections 2024-04-25T00:31:11.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.079+0000 7f82dfbee640 1 --2- 172.21.15.32:0/288211648 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f82d8103360 0x7f82d8109bf0 unknown :-1 s=CLOSED pgs=257 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:11.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.079+0000 7f82dfbee640 1 --2- 172.21.15.32:0/288211648 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f82d81029a0 0x7f82d8102e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:11.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.079+0000 7f82dfbee640 1 --2- 172.21.15.32:0/288211648 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f82d81017b0 0x7f82d8101bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:11.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.079+0000 7f82dfbee640 1 -- 172.21.15.32:0/288211648 >> 172.21.15.32:0/288211648 conn(0x7f82d80fcd40 msgr2=0x7f82d80ff160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:11.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.079+0000 7f82dfbee640 1 -- 172.21.15.32:0/288211648 shutdown_connections 2024-04-25T00:31:11.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.079+0000 7f82dfbee640 1 -- 172.21.15.32:0/288211648 wait complete. 2024-04-25T00:31:11.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.080+0000 7f82dfbee640 1 Processor -- start 2024-04-25T00:31:11.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.080+0000 7f82dfbee640 1 -- start start 2024-04-25T00:31:11.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.080+0000 7f82dfbee640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f82d81017b0 0x7f82d8197aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:11.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.080+0000 7f82dfbee640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f82d81029a0 0x7f82d8197fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:11.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.080+0000 7f82dfbee640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f82d8103360 0x7f82d819f050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:11.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.080+0000 7f82dfbee640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f82d810d430 con 0x7f82d8103360 2024-04-25T00:31:11.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.080+0000 7f82dfbee640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f82d810d2b0 con 0x7f82d81017b0 2024-04-25T00:31:11.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.080+0000 7f82dfbee640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f82d810d5b0 con 0x7f82d81029a0 2024-04-25T00:31:11.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.080+0000 7f82de18b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f82d8103360 0x7f82d819f050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:11.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.080+0000 7f82de18b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f82d8103360 0x7f82d819f050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:60954/0 (socket says 172.21.15.32:60954) 2024-04-25T00:31:11.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.080+0000 7f82de18b640 1 -- 172.21.15.32:0/2844411489 learned_addr learned my addr 172.21.15.32:0/2844411489 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:11.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.080+0000 7f82dd98a640 1 --2- 172.21.15.32:0/2844411489 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f82d81017b0 0x7f82d8197aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:11.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.080+0000 7f82dd189640 1 --2- 172.21.15.32:0/2844411489 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f82d81029a0 0x7f82d8197fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:11.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.080+0000 7f82de18b640 1 -- 172.21.15.32:0/2844411489 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f82d81029a0 msgr2=0x7f82d8197fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:11.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.080+0000 7f82de18b640 1 --2- 172.21.15.32:0/2844411489 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f82d81029a0 0x7f82d8197fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:11.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.080+0000 7f82de18b640 1 -- 172.21.15.32:0/2844411489 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f82d81017b0 msgr2=0x7f82d8197aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:11.084 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.080+0000 7f82de18b640 1 --2- 172.21.15.32:0/2844411489 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f82d81017b0 0x7f82d8197aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:11.084 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.080+0000 7f82de18b640 1 -- 172.21.15.32:0/2844411489 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f82d819f6c0 con 0x7f82d8103360 2024-04-25T00:31:11.084 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.080+0000 7f82de18b640 1 --2- 172.21.15.32:0/2844411489 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f82d8103360 0x7f82d819f050 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f82d0005e30 tx=0x7f82d0031ee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:11.084 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.081+0000 7f82ceffd640 1 -- 172.21.15.32:0/2844411489 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f82d0003c30 con 0x7f82d8103360 2024-04-25T00:31:11.084 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.081+0000 7f82ceffd640 1 -- 172.21.15.32:0/2844411489 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f82d0003dd0 con 0x7f82d8103360 2024-04-25T00:31:11.084 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.081+0000 7f82dfbee640 1 -- 172.21.15.32:0/2844411489 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f82d819f950 con 0x7f82d8103360 2024-04-25T00:31:11.084 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.081+0000 7f82ceffd640 1 -- 172.21.15.32:0/2844411489 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f82d002fde0 con 0x7f82d8103360 2024-04-25T00:31:11.084 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.081+0000 7f82dfbee640 1 -- 172.21.15.32:0/2844411489 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f82d819fea0 con 0x7f82d8103360 2024-04-25T00:31:11.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.082+0000 7f82dfbee640 1 -- 172.21.15.32:0/2844411489 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f82d8108ba0 con 0x7f82d8103360 2024-04-25T00:31:11.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.083+0000 7f82ceffd640 1 -- 172.21.15.32:0/2844411489 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f82d0004050 con 0x7f82d8103360 2024-04-25T00:31:11.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.084+0000 7f82ceffd640 1 --2- 172.21.15.32:0/2844411489 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f82c40a2b90 0x7f82c40a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:11.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.084+0000 7f82dd98a640 1 --2- 172.21.15.32:0/2844411489 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f82c40a2b90 0x7f82c40a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:11.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.084+0000 7f82ceffd640 1 -- 172.21.15.32:0/2844411489 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f82d00f0540 con 0x7f82d8103360 2024-04-25T00:31:11.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.084+0000 7f82dd98a640 1 --2- 172.21.15.32:0/2844411489 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f82c40a2b90 0x7f82c40a5050 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f82d8102800 tx=0x7f82d400b390 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:11.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.088+0000 7f82ceffd640 1 -- 172.21.15.32:0/2844411489 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f82d00bd200 con 0x7f82d8103360 2024-04-25T00:31:11.277 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.276+0000 7f82dfbee640 1 -- 172.21.15.32:0/2844411489 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f82d802d340 con 0x7f82d8103360 2024-04-25T00:31:11.278 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.278+0000 7f82ceffd640 1 -- 172.21.15.32:0/2844411489 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v89) ==== 74+0+24128 (secure 0 0 0) 0x7f82d00bcb00 con 0x7f82d8103360 2024-04-25T00:31:11.279 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:31:11.279 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":89,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:30:53.134605+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-25T00:30:46.239556+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-25T00:30:47.257182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2024-04-25T00:30:51.193975+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":83,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-num-failure-domains":"0","crush-osds-per-failure-domain":"0","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:31:11.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.280+0000 7f82dfbee640 1 -- 172.21.15.32:0/2844411489 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f82c40a2b90 msgr2=0x7f82c40a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:11.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.280+0000 7f82dfbee640 1 --2- 172.21.15.32:0/2844411489 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f82c40a2b90 0x7f82c40a5050 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f82d8102800 tx=0x7f82d400b390 comp rx=0 tx=0).stop 2024-04-25T00:31:11.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.280+0000 7f82dfbee640 1 -- 172.21.15.32:0/2844411489 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f82d8103360 msgr2=0x7f82d819f050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:11.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.280+0000 7f82dfbee640 1 --2- 172.21.15.32:0/2844411489 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f82d8103360 0x7f82d819f050 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f82d0005e30 tx=0x7f82d0031ee0 comp rx=0 tx=0).stop 2024-04-25T00:31:11.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.280+0000 7f82dfbee640 1 -- 172.21.15.32:0/2844411489 shutdown_connections 2024-04-25T00:31:11.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.280+0000 7f82dfbee640 1 --2- 172.21.15.32:0/2844411489 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f82c40a2b90 0x7f82c40a5050 unknown :-1 s=CLOSED pgs=177 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:11.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.280+0000 7f82dfbee640 1 --2- 172.21.15.32:0/2844411489 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f82d8103360 0x7f82d819f050 unknown :-1 s=CLOSED pgs=258 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:11.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.280+0000 7f82dfbee640 1 --2- 172.21.15.32:0/2844411489 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f82d81029a0 0x7f82d8197fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:11.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.280+0000 7f82dfbee640 1 --2- 172.21.15.32:0/2844411489 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f82d81017b0 0x7f82d8197aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:11.282 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.280+0000 7f82dfbee640 1 -- 172.21.15.32:0/2844411489 >> 172.21.15.32:0/2844411489 conn(0x7f82d80fcd40 msgr2=0x7f82d8105d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:11.282 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.280+0000 7f82dfbee640 1 -- 172.21.15.32:0/2844411489 shutdown_connections 2024-04-25T00:31:11.282 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.280+0000 7f82dfbee640 1 -- 172.21.15.32:0/2844411489 wait complete. 2024-04-25T00:31:11.292 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs standby_count_wanted 0 2024-04-25T00:31:11.398 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.396+0000 7f46319f4640 1 -- 172.21.15.32:0/3314706735 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f462c101890 msgr2=0x7f462c101d10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:11.399 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.396+0000 7f46319f4640 1 --2- 172.21.15.32:0/3314706735 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f462c101890 0x7f462c101d10 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f4620009900 tx=0x7f462002a410 comp rx=0 tx=0).stop 2024-04-25T00:31:11.399 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.396+0000 7f46319f4640 1 -- 172.21.15.32:0/3314706735 shutdown_connections 2024-04-25T00:31:11.399 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.396+0000 7f46319f4640 1 --2- 172.21.15.32:0/3314706735 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f462c102250 0x7f462c108ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:11.399 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.396+0000 7f46319f4640 1 --2- 172.21.15.32:0/3314706735 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f462c101890 0x7f462c101d10 unknown :-1 s=CLOSED pgs=259 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:11.399 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.396+0000 7f46319f4640 1 --2- 172.21.15.32:0/3314706735 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f462c1006a0 0x7f462c100ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:11.399 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.396+0000 7f46319f4640 1 -- 172.21.15.32:0/3314706735 >> 172.21.15.32:0/3314706735 conn(0x7f462c0fbc30 msgr2=0x7f462c0fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:11.399 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.396+0000 7f46319f4640 1 -- 172.21.15.32:0/3314706735 shutdown_connections 2024-04-25T00:31:11.399 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.396+0000 7f46319f4640 1 -- 172.21.15.32:0/3314706735 wait complete. 2024-04-25T00:31:11.399 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.397+0000 7f46319f4640 1 Processor -- start 2024-04-25T00:31:11.399 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.397+0000 7f46319f4640 1 -- start start 2024-04-25T00:31:11.399 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.397+0000 7f46319f4640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f462c1006a0 0x7f462c199c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:11.400 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.397+0000 7f46319f4640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f462c101890 0x7f462c19a1c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:11.400 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.397+0000 7f46319f4640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f462c102250 0x7f462c1a1220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:11.400 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.397+0000 7f46319f4640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f462c10d440 con 0x7f462c101890 2024-04-25T00:31:11.400 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.397+0000 7f46319f4640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f462c10d2c0 con 0x7f462c1006a0 2024-04-25T00:31:11.400 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.397+0000 7f46319f4640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f462c10d5c0 con 0x7f462c102250 2024-04-25T00:31:11.400 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.397+0000 7f462a7fc640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f462c101890 0x7f462c19a1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:11.400 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.397+0000 7f462a7fc640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f462c101890 0x7f462c19a1c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:60982/0 (socket says 172.21.15.32:60982) 2024-04-25T00:31:11.400 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.397+0000 7f462a7fc640 1 -- 172.21.15.32:0/461506386 learned_addr learned my addr 172.21.15.32:0/461506386 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:11.400 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.397+0000 7f462affd640 1 --2- 172.21.15.32:0/461506386 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f462c1006a0 0x7f462c199c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:11.400 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.397+0000 7f462b7fe640 1 --2- 172.21.15.32:0/461506386 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f462c102250 0x7f462c1a1220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:11.400 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.397+0000 7f462a7fc640 1 -- 172.21.15.32:0/461506386 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f462c102250 msgr2=0x7f462c1a1220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:11.400 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.397+0000 7f462a7fc640 1 --2- 172.21.15.32:0/461506386 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f462c102250 0x7f462c1a1220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:11.400 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.397+0000 7f462a7fc640 1 -- 172.21.15.32:0/461506386 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f462c1006a0 msgr2=0x7f462c199c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:11.400 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.397+0000 7f462a7fc640 1 --2- 172.21.15.32:0/461506386 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f462c1006a0 0x7f462c199c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:11.401 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.397+0000 7f462a7fc640 1 -- 172.21.15.32:0/461506386 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f462c1a18c0 con 0x7f462c101890 2024-04-25T00:31:11.401 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.397+0000 7f462affd640 1 --2- 172.21.15.32:0/461506386 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f462c1006a0 0x7f462c199c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:31:11.401 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.398+0000 7f462a7fc640 1 --2- 172.21.15.32:0/461506386 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f462c101890 0x7f462c19a1c0 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7f4620009230 tx=0x7f4620004360 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:11.401 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.398+0000 7f46309f2640 1 -- 172.21.15.32:0/461506386 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f462003f030 con 0x7f462c101890 2024-04-25T00:31:11.401 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.398+0000 7f46309f2640 1 -- 172.21.15.32:0/461506386 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4620003ce0 con 0x7f462c101890 2024-04-25T00:31:11.401 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.398+0000 7f46319f4640 1 -- 172.21.15.32:0/461506386 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f462c1a1b50 con 0x7f462c101890 2024-04-25T00:31:11.401 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.398+0000 7f46309f2640 1 -- 172.21.15.32:0/461506386 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4620035070 con 0x7f462c101890 2024-04-25T00:31:11.401 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.398+0000 7f46319f4640 1 -- 172.21.15.32:0/461506386 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f462c1a2040 con 0x7f462c101890 2024-04-25T00:31:11.402 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.399+0000 7f46319f4640 1 -- 172.21.15.32:0/461506386 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f462c1a2210 con 0x7f462c101890 2024-04-25T00:31:11.402 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.400+0000 7f46309f2640 1 -- 172.21.15.32:0/461506386 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f4620042030 con 0x7f462c101890 2024-04-25T00:31:11.402 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.400+0000 7f46309f2640 1 --2- 172.21.15.32:0/461506386 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f46140a2b90 0x7f46140a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:11.402 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.401+0000 7f462affd640 1 --2- 172.21.15.32:0/461506386 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f46140a2b90 0x7f46140a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:11.402 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.401+0000 7f46309f2640 1 -- 172.21.15.32:0/461506386 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f46200efdb0 con 0x7f462c101890 2024-04-25T00:31:11.402 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.401+0000 7f462affd640 1 --2- 172.21.15.32:0/461506386 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f46140a2b90 0x7f46140a5050 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f461c005fd0 tx=0x7f461c005850 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:11.405 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.405+0000 7f46309f2640 1 -- 172.21.15.32:0/461506386 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f46200f4070 con 0x7f462c101890 2024-04-25T00:31:11.630 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.630+0000 7f46319f4640 1 -- 172.21.15.32:0/461506386 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"} v 0) -- 0x7f462c1a2480 con 0x7f462c101890 2024-04-25T00:31:11.943 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.943+0000 7f46309f2640 1 -- 172.21.15.32:0/461506386 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]=0 v18) ==== 118+0+0 (secure 0 0 0) 0x7f46200bca70 con 0x7f462c101890 2024-04-25T00:31:11.947 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.946+0000 7f46319f4640 1 -- 172.21.15.32:0/461506386 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f46140a2b90 msgr2=0x7f46140a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:11.947 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.946+0000 7f46319f4640 1 --2- 172.21.15.32:0/461506386 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f46140a2b90 0x7f46140a5050 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f461c005fd0 tx=0x7f461c005850 comp rx=0 tx=0).stop 2024-04-25T00:31:11.947 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.946+0000 7f46319f4640 1 -- 172.21.15.32:0/461506386 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f462c101890 msgr2=0x7f462c19a1c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:11.947 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.946+0000 7f46319f4640 1 --2- 172.21.15.32:0/461506386 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f462c101890 0x7f462c19a1c0 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7f4620009230 tx=0x7f4620004360 comp rx=0 tx=0).stop 2024-04-25T00:31:11.947 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.946+0000 7f46319f4640 1 -- 172.21.15.32:0/461506386 shutdown_connections 2024-04-25T00:31:11.947 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.946+0000 7f46319f4640 1 --2- 172.21.15.32:0/461506386 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f46140a2b90 0x7f46140a5050 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:11.947 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.946+0000 7f46319f4640 1 --2- 172.21.15.32:0/461506386 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f462c102250 0x7f462c1a1220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:11.947 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.946+0000 7f46319f4640 1 --2- 172.21.15.32:0/461506386 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f462c101890 0x7f462c19a1c0 unknown :-1 s=CLOSED pgs=260 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:11.947 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.946+0000 7f46319f4640 1 --2- 172.21.15.32:0/461506386 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f462c1006a0 0x7f462c199c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:11.947 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.946+0000 7f46319f4640 1 -- 172.21.15.32:0/461506386 >> 172.21.15.32:0/461506386 conn(0x7f462c0fbc30 msgr2=0x7f462c1094a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:11.947 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.946+0000 7f46319f4640 1 -- 172.21.15.32:0/461506386 shutdown_connections 2024-04-25T00:31:11.947 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:11.946+0000 7f46319f4640 1 -- 172.21.15.32:0/461506386 wait complete. 2024-04-25T00:31:11.958 DEBUG:tasks.cephfs.filesystem:fs_config: {'ec_profile': ['m=2', 'k=2', 'crush-failure-domain=osd'], 'max_mds': 3, 'session_timeout': 300, 'standby_replay': True, 'subvols': {'create': 2}} 2024-04-25T00:31:11.959 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs max_mds 3 2024-04-25T00:31:12.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.065+0000 7f1fd6c9a640 1 -- 172.21.15.32:0/3583869528 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1fd0102280 msgr2=0x7f1fd0104670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:12.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.065+0000 7f1fd6c9a640 1 --2- 172.21.15.32:0/3583869528 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1fd0102280 0x7f1fd0104670 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f1fc0009900 tx=0x7f1fc002a410 comp rx=0 tx=0).stop 2024-04-25T00:31:12.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.065+0000 7f1fd6c9a640 1 -- 172.21.15.32:0/3583869528 shutdown_connections 2024-04-25T00:31:12.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.065+0000 7f1fd6c9a640 1 --2- 172.21.15.32:0/3583869528 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1fd01074e0 0x7f1fd01098d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:12.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.065+0000 7f1fd6c9a640 1 --2- 172.21.15.32:0/3583869528 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1fd0104bb0 0x7f1fd0106fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:12.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.065+0000 7f1fd6c9a640 1 --2- 172.21.15.32:0/3583869528 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1fd0102280 0x7f1fd0104670 unknown :-1 s=CLOSED pgs=261 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:12.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.065+0000 7f1fd6c9a640 1 -- 172.21.15.32:0/3583869528 >> 172.21.15.32:0/3583869528 conn(0x7f1fd00fbc10 msgr2=0x7f1fd00fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:12.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.065+0000 7f1fd6c9a640 1 -- 172.21.15.32:0/3583869528 shutdown_connections 2024-04-25T00:31:12.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.065+0000 7f1fd6c9a640 1 -- 172.21.15.32:0/3583869528 wait complete. 2024-04-25T00:31:12.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.066+0000 7f1fd6c9a640 1 Processor -- start 2024-04-25T00:31:12.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.066+0000 7f1fd6c9a640 1 -- start start 2024-04-25T00:31:12.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.066+0000 7f1fd6c9a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1fd0102280 0x7f1fd0101bf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:12.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.066+0000 7f1fd6c9a640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1fd0104bb0 0x7f1fd0100230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:12.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.066+0000 7f1fd6c9a640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1fd01074e0 0x7f1fd0100770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:12.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.066+0000 7f1fd6c9a640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f1fd010d440 con 0x7f1fd0102280 2024-04-25T00:31:12.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.066+0000 7f1fd6c9a640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f1fd010d2c0 con 0x7f1fd01074e0 2024-04-25T00:31:12.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.066+0000 7f1fd6c9a640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f1fd010d5c0 con 0x7f1fd0104bb0 2024-04-25T00:31:12.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.066+0000 7f1fd4a36640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1fd0102280 0x7f1fd0101bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:12.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.066+0000 7f1fd4a36640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1fd0102280 0x7f1fd0101bf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:57802/0 (socket says 172.21.15.32:57802) 2024-04-25T00:31:12.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.066+0000 7f1fd4a36640 1 -- 172.21.15.32:0/3622174171 learned_addr learned my addr 172.21.15.32:0/3622174171 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:12.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.066+0000 7f1fcffff640 1 --2- 172.21.15.32:0/3622174171 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1fd0104bb0 0x7f1fd0100230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:12.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.066+0000 7f1fd5237640 1 --2- 172.21.15.32:0/3622174171 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1fd01074e0 0x7f1fd0100770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:12.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.067+0000 7f1fd4a36640 1 -- 172.21.15.32:0/3622174171 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1fd0104bb0 msgr2=0x7f1fd0100230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:12.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.067+0000 7f1fd4a36640 1 --2- 172.21.15.32:0/3622174171 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1fd0104bb0 0x7f1fd0100230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:12.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.067+0000 7f1fd4a36640 1 -- 172.21.15.32:0/3622174171 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1fd01074e0 msgr2=0x7f1fd0100770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:12.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.067+0000 7f1fd4a36640 1 --2- 172.21.15.32:0/3622174171 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1fd01074e0 0x7f1fd0100770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:12.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.067+0000 7f1fd4a36640 1 -- 172.21.15.32:0/3622174171 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1fd0101000 con 0x7f1fd0102280 2024-04-25T00:31:12.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.067+0000 7f1fcffff640 1 --2- 172.21.15.32:0/3622174171 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1fd0104bb0 0x7f1fd0100230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:31:12.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.067+0000 7f1fd4a36640 1 --2- 172.21.15.32:0/3622174171 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1fd0102280 0x7f1fd0101bf0 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f1fc0009230 tx=0x7f1fc0004360 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:12.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.067+0000 7f1fcdffb640 1 -- 172.21.15.32:0/3622174171 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1fc003f030 con 0x7f1fd0102280 2024-04-25T00:31:12.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.067+0000 7f1fd6c9a640 1 -- 172.21.15.32:0/3622174171 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1fd01a1b80 con 0x7f1fd0102280 2024-04-25T00:31:12.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.067+0000 7f1fcdffb640 1 -- 172.21.15.32:0/3622174171 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1fc0003ce0 con 0x7f1fd0102280 2024-04-25T00:31:12.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.067+0000 7f1fcdffb640 1 -- 172.21.15.32:0/3622174171 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1fc0035070 con 0x7f1fd0102280 2024-04-25T00:31:12.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.067+0000 7f1fd6c9a640 1 -- 172.21.15.32:0/3622174171 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1fd01a20d0 con 0x7f1fd0102280 2024-04-25T00:31:12.071 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.068+0000 7f1fd6c9a640 1 -- 172.21.15.32:0/3622174171 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1fd00fd700 con 0x7f1fd0102280 2024-04-25T00:31:12.071 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.069+0000 7f1fcdffb640 1 -- 172.21.15.32:0/3622174171 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f1fc0042030 con 0x7f1fd0102280 2024-04-25T00:31:12.071 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.070+0000 7f1fcdffb640 1 --2- 172.21.15.32:0/3622174171 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f1fb80a2b90 0x7f1fb80a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:12.071 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.070+0000 7f1fcdffb640 1 -- 172.21.15.32:0/3622174171 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f1fc00efdb0 con 0x7f1fd0102280 2024-04-25T00:31:12.072 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.070+0000 7f1fcffff640 1 --2- 172.21.15.32:0/3622174171 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f1fb80a2b90 0x7f1fb80a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:12.072 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.070+0000 7f1fcffff640 1 --2- 172.21.15.32:0/3622174171 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f1fb80a2b90 0x7f1fb80a5050 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f1fd0101960 tx=0x7f1fc4005ec0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:12.074 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.074+0000 7f1fcdffb640 1 -- 172.21.15.32:0/3622174171 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f1fc00f4070 con 0x7f1fd0102280 2024-04-25T00:31:12.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:11 smithi032 ceph-mon[29357]: pgmap v303: 129 pgs: 129 active+clean; 581 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 426 B/s wr, 0 op/s 2024-04-25T00:31:12.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:11 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/178770114' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:31:12.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:11 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2844411489' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:31:12.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:11 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/461506386' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-04-25T00:31:12.299 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.299+0000 7f1fd6c9a640 1 -- 172.21.15.32:0/3622174171 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "3"} v 0) -- 0x7f1fd0062340 con 0x7f1fd0102280 2024-04-25T00:31:12.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:11 smithi121 ceph-mon[32231]: pgmap v303: 129 pgs: 129 active+clean; 581 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 426 B/s wr, 0 op/s 2024-04-25T00:31:12.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:11 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/178770114' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:31:12.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:11 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2844411489' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:31:12.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:11 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/461506386' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-04-25T00:31:12.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:11 smithi130 ceph-mon[32525]: pgmap v303: 129 pgs: 129 active+clean; 581 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 426 B/s wr, 0 op/s 2024-04-25T00:31:12.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:11 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/178770114' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:31:12.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:11 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2844411489' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:31:12.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:11 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/461506386' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-04-25T00:31:12.945 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.945+0000 7f1fcdffb640 1 -- 172.21.15.32:0/3622174171 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "3"}]=0 v19) ==== 105+0+0 (secure 0 0 0) 0x7f1fc00bca70 con 0x7f1fd0102280 2024-04-25T00:31:12.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.948+0000 7f1fd6c9a640 1 -- 172.21.15.32:0/3622174171 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f1fb80a2b90 msgr2=0x7f1fb80a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:12.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.948+0000 7f1fd6c9a640 1 --2- 172.21.15.32:0/3622174171 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f1fb80a2b90 0x7f1fb80a5050 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f1fd0101960 tx=0x7f1fc4005ec0 comp rx=0 tx=0).stop 2024-04-25T00:31:12.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.948+0000 7f1fd6c9a640 1 -- 172.21.15.32:0/3622174171 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1fd0102280 msgr2=0x7f1fd0101bf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:12.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.948+0000 7f1fd6c9a640 1 --2- 172.21.15.32:0/3622174171 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1fd0102280 0x7f1fd0101bf0 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f1fc0009230 tx=0x7f1fc0004360 comp rx=0 tx=0).stop 2024-04-25T00:31:12.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.948+0000 7f1fd6c9a640 1 -- 172.21.15.32:0/3622174171 shutdown_connections 2024-04-25T00:31:12.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.948+0000 7f1fd6c9a640 1 --2- 172.21.15.32:0/3622174171 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f1fb80a2b90 0x7f1fb80a5050 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:12.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.948+0000 7f1fd6c9a640 1 --2- 172.21.15.32:0/3622174171 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1fd01074e0 0x7f1fd0100770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:12.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.948+0000 7f1fd6c9a640 1 --2- 172.21.15.32:0/3622174171 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1fd0104bb0 0x7f1fd0100230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:12.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.948+0000 7f1fd6c9a640 1 --2- 172.21.15.32:0/3622174171 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1fd0102280 0x7f1fd0101bf0 unknown :-1 s=CLOSED pgs=262 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:12.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.948+0000 7f1fd6c9a640 1 -- 172.21.15.32:0/3622174171 >> 172.21.15.32:0/3622174171 conn(0x7f1fd00fbc10 msgr2=0x7f1fd0102d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:12.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.948+0000 7f1fd6c9a640 1 -- 172.21.15.32:0/3622174171 shutdown_connections 2024-04-25T00:31:12.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:12.948+0000 7f1fd6c9a640 1 -- 172.21.15.32:0/3622174171 wait complete. 2024-04-25T00:31:12.965 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs allow_standby_replay true 2024-04-25T00:31:13.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.075+0000 7f02bf64d640 1 -- 172.21.15.32:0/2441668069 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f02b8102250 msgr2=0x7f02b8108ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:13.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.075+0000 7f02bf64d640 1 --2- 172.21.15.32:0/2441668069 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f02b8102250 0x7f02b8108ae0 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7f02b0009ca0 tx=0x7f02b002a8e0 comp rx=0 tx=0).stop 2024-04-25T00:31:13.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.075+0000 7f02bf64d640 1 -- 172.21.15.32:0/2441668069 shutdown_connections 2024-04-25T00:31:13.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.075+0000 7f02bf64d640 1 --2- 172.21.15.32:0/2441668069 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f02b8102250 0x7f02b8108ae0 unknown :-1 s=CLOSED pgs=263 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:13.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.075+0000 7f02bf64d640 1 --2- 172.21.15.32:0/2441668069 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f02b8101890 0x7f02b8101d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:13.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.075+0000 7f02bf64d640 1 --2- 172.21.15.32:0/2441668069 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f02b81006a0 0x7f02b8100ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:13.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.075+0000 7f02bf64d640 1 -- 172.21.15.32:0/2441668069 >> 172.21.15.32:0/2441668069 conn(0x7f02b80fbc30 msgr2=0x7f02b80fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:13.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.075+0000 7f02bf64d640 1 -- 172.21.15.32:0/2441668069 shutdown_connections 2024-04-25T00:31:13.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.075+0000 7f02bf64d640 1 -- 172.21.15.32:0/2441668069 wait complete. 2024-04-25T00:31:13.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.076+0000 7f02bf64d640 1 Processor -- start 2024-04-25T00:31:13.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.076+0000 7f02bf64d640 1 -- start start 2024-04-25T00:31:13.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.076+0000 7f02bf64d640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f02b81006a0 0x7f02b8104630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:13.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.076+0000 7f02bf64d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f02b8101890 0x7f02b8102c70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:13.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.076+0000 7f02bf64d640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f02b8102250 0x7f02b81031b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:13.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.076+0000 7f02bf64d640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f02b810c3a0 con 0x7f02b8101890 2024-04-25T00:31:13.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.076+0000 7f02bf64d640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f02b810c220 con 0x7f02b81006a0 2024-04-25T00:31:13.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.076+0000 7f02bf64d640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f02b810c520 con 0x7f02b8102250 2024-04-25T00:31:13.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.076+0000 7f02bcbe8640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f02b8101890 0x7f02b8102c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:13.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.076+0000 7f02bcbe8640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f02b8101890 0x7f02b8102c70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:57826/0 (socket says 172.21.15.32:57826) 2024-04-25T00:31:13.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.076+0000 7f02bcbe8640 1 -- 172.21.15.32:0/4162244575 learned_addr learned my addr 172.21.15.32:0/4162244575 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:13.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.076+0000 7f02bd3e9640 1 --2- 172.21.15.32:0/4162244575 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f02b81006a0 0x7f02b8104630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:13.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.076+0000 7f02bdbea640 1 --2- 172.21.15.32:0/4162244575 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f02b8102250 0x7f02b81031b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:13.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.076+0000 7f02bcbe8640 1 -- 172.21.15.32:0/4162244575 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f02b8102250 msgr2=0x7f02b81031b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:13.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.076+0000 7f02bcbe8640 1 --2- 172.21.15.32:0/4162244575 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f02b8102250 0x7f02b81031b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:13.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.076+0000 7f02bcbe8640 1 -- 172.21.15.32:0/4162244575 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f02b81006a0 msgr2=0x7f02b8104630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:13.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.076+0000 7f02bcbe8640 1 --2- 172.21.15.32:0/4162244575 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f02b81006a0 0x7f02b8104630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:13.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.076+0000 7f02bcbe8640 1 -- 172.21.15.32:0/4162244575 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f02b8103a40 con 0x7f02b8101890 2024-04-25T00:31:13.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.076+0000 7f02bcbe8640 1 --2- 172.21.15.32:0/4162244575 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f02b8101890 0x7f02b8102c70 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7f02a800b450 tx=0x7f02a800b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:13.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.077+0000 7f02ae7fc640 1 -- 172.21.15.32:0/4162244575 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f02a8015020 con 0x7f02b8101890 2024-04-25T00:31:13.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.077+0000 7f02ae7fc640 1 -- 172.21.15.32:0/4162244575 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f02a8011d70 con 0x7f02b8101890 2024-04-25T00:31:13.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.077+0000 7f02bf64d640 1 -- 172.21.15.32:0/4162244575 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f02b81a1c20 con 0x7f02b8101890 2024-04-25T00:31:13.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.077+0000 7f02ae7fc640 1 -- 172.21.15.32:0/4162244575 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f02a800f340 con 0x7f02b8101890 2024-04-25T00:31:13.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.077+0000 7f02bf64d640 1 -- 172.21.15.32:0/4162244575 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f02b81a2170 con 0x7f02b8101890 2024-04-25T00:31:13.081 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.078+0000 7f02bf64d640 1 -- 172.21.15.32:0/4162244575 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f02b81a2340 con 0x7f02b8101890 2024-04-25T00:31:13.081 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.079+0000 7f02ae7fc640 1 -- 172.21.15.32:0/4162244575 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f02a8018020 con 0x7f02b8101890 2024-04-25T00:31:13.081 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.080+0000 7f02ae7fc640 1 --2- 172.21.15.32:0/4162244575 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f02a40a2c10 0x7f02a40a50d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:13.081 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.080+0000 7f02bd3e9640 1 --2- 172.21.15.32:0/4162244575 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f02a40a2c10 0x7f02a40a50d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:13.081 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.080+0000 7f02ae7fc640 1 -- 172.21.15.32:0/4162244575 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f02a80cede0 con 0x7f02b8101890 2024-04-25T00:31:13.081 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.080+0000 7f02bd3e9640 1 --2- 172.21.15.32:0/4162244575 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f02a40a2c10 0x7f02a40a50d0 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f02b81016f0 tx=0x7f02b400b390 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:13.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.084+0000 7f02ae7fc640 1 -- 172.21.15.32:0/4162244575 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f02a801f020 con 0x7f02b8101890 2024-04-25T00:31:13.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:12 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/461506386' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-04-25T00:31:13.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:12 smithi032 ceph-mon[29357]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-25T00:31:13.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:12 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3622174171' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "3"}]: dispatch 2024-04-25T00:31:13.307 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.306+0000 7f02bf64d640 1 -- 172.21.15.32:0/4162244575 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"} v 0) -- 0x7f02b81a2730 con 0x7f02b8101890 2024-04-25T00:31:13.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:12 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/461506386' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-04-25T00:31:13.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:12 smithi121 ceph-mon[32231]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-25T00:31:13.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:12 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3622174171' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "3"}]: dispatch 2024-04-25T00:31:13.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:12 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/461506386' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-04-25T00:31:13.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:12 smithi130 ceph-mon[32525]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-25T00:31:13.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:12 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3622174171' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "3"}]: dispatch 2024-04-25T00:31:13.959 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.958+0000 7f02ae7fc640 1 -- 172.21.15.32:0/4162244575 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]=0 v21) ==== 121+0+0 (secure 0 0 0) 0x7f02a800d040 con 0x7f02b8101890 2024-04-25T00:31:13.962 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.961+0000 7f02bf64d640 1 -- 172.21.15.32:0/4162244575 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f02a40a2c10 msgr2=0x7f02a40a50d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:13.962 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.961+0000 7f02bf64d640 1 --2- 172.21.15.32:0/4162244575 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f02a40a2c10 0x7f02a40a50d0 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f02b81016f0 tx=0x7f02b400b390 comp rx=0 tx=0).stop 2024-04-25T00:31:13.962 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.961+0000 7f02bf64d640 1 -- 172.21.15.32:0/4162244575 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f02b8101890 msgr2=0x7f02b8102c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:13.962 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.961+0000 7f02bf64d640 1 --2- 172.21.15.32:0/4162244575 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f02b8101890 0x7f02b8102c70 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7f02a800b450 tx=0x7f02a800b910 comp rx=0 tx=0).stop 2024-04-25T00:31:13.962 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.961+0000 7f02bf64d640 1 -- 172.21.15.32:0/4162244575 shutdown_connections 2024-04-25T00:31:13.962 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.961+0000 7f02bf64d640 1 --2- 172.21.15.32:0/4162244575 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f02a40a2c10 0x7f02a40a50d0 unknown :-1 s=CLOSED pgs=180 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:13.962 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.961+0000 7f02bf64d640 1 --2- 172.21.15.32:0/4162244575 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f02b8102250 0x7f02b81031b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:13.962 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.961+0000 7f02bf64d640 1 --2- 172.21.15.32:0/4162244575 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f02b8101890 0x7f02b8102c70 unknown :-1 s=CLOSED pgs=264 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:13.962 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.961+0000 7f02bf64d640 1 --2- 172.21.15.32:0/4162244575 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f02b81006a0 0x7f02b8104630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:13.962 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.961+0000 7f02bf64d640 1 -- 172.21.15.32:0/4162244575 >> 172.21.15.32:0/4162244575 conn(0x7f02b80fbc30 msgr2=0x7f02b80fd6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:13.962 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.961+0000 7f02bf64d640 1 -- 172.21.15.32:0/4162244575 shutdown_connections 2024-04-25T00:31:13.963 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:13.961+0000 7f02bf64d640 1 -- 172.21.15.32:0/4162244575 wait complete. 2024-04-25T00:31:13.979 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs session_timeout 300 2024-04-25T00:31:14.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.090+0000 7f4ed69d5640 1 -- 172.21.15.32:0/281938995 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4ed01006a0 msgr2=0x7f4ed0100ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:14.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.090+0000 7f4ed69d5640 1 --2- 172.21.15.32:0/281938995 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4ed01006a0 0x7f4ed0100ac0 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f4ec0009860 tx=0x7f4ec002a340 comp rx=0 tx=0).stop 2024-04-25T00:31:14.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.090+0000 7f4ed69d5640 1 -- 172.21.15.32:0/281938995 shutdown_connections 2024-04-25T00:31:14.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.090+0000 7f4ed69d5640 1 --2- 172.21.15.32:0/281938995 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ed0102250 0x7f4ed0108ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:14.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.090+0000 7f4ed69d5640 1 --2- 172.21.15.32:0/281938995 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ed0101890 0x7f4ed0101d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:14.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.090+0000 7f4ed69d5640 1 --2- 172.21.15.32:0/281938995 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4ed01006a0 0x7f4ed0100ac0 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:14.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.090+0000 7f4ed69d5640 1 -- 172.21.15.32:0/281938995 >> 172.21.15.32:0/281938995 conn(0x7f4ed00fbc30 msgr2=0x7f4ed00fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:14.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.090+0000 7f4ed69d5640 1 -- 172.21.15.32:0/281938995 shutdown_connections 2024-04-25T00:31:14.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.090+0000 7f4ed69d5640 1 -- 172.21.15.32:0/281938995 wait complete. 2024-04-25T00:31:14.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.090+0000 7f4ed69d5640 1 Processor -- start 2024-04-25T00:31:14.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.091+0000 7f4ed69d5640 1 -- start start 2024-04-25T00:31:14.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.091+0000 7f4ed69d5640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4ed01006a0 0x7f4ed0197b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:14.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.091+0000 7f4ed69d5640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ed0101890 0x7f4ed0198060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:14.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.091+0000 7f4ed69d5640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ed0102250 0x7f4ed019f080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:14.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.091+0000 7f4ed69d5640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f4ed010c2f0 con 0x7f4ed0102250 2024-04-25T00:31:14.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.091+0000 7f4ed69d5640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f4ed010c170 con 0x7f4ed01006a0 2024-04-25T00:31:14.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.091+0000 7f4ed69d5640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f4ed010c470 con 0x7f4ed0101890 2024-04-25T00:31:14.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.091+0000 7f4ed4f72640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ed0102250 0x7f4ed019f080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:14.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.091+0000 7f4ed4f72640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ed0102250 0x7f4ed019f080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:57836/0 (socket says 172.21.15.32:57836) 2024-04-25T00:31:14.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.091+0000 7f4ed4f72640 1 -- 172.21.15.32:0/1289060660 learned_addr learned my addr 172.21.15.32:0/1289060660 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:14.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.091+0000 7f4ecffff640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4ed01006a0 0x7f4ed0197b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:14.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.091+0000 7f4ecf7fe640 1 --2- 172.21.15.32:0/1289060660 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ed0101890 0x7f4ed0198060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:14.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.091+0000 7f4ed4f72640 1 -- 172.21.15.32:0/1289060660 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ed0101890 msgr2=0x7f4ed0198060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:14.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.091+0000 7f4ed4f72640 1 --2- 172.21.15.32:0/1289060660 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ed0101890 0x7f4ed0198060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:14.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.091+0000 7f4ed4f72640 1 -- 172.21.15.32:0/1289060660 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4ed01006a0 msgr2=0x7f4ed0197b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:14.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.091+0000 7f4ed4f72640 1 --2- 172.21.15.32:0/1289060660 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4ed01006a0 0x7f4ed0197b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:14.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.091+0000 7f4ed4f72640 1 -- 172.21.15.32:0/1289060660 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4ed019f6f0 con 0x7f4ed0102250 2024-04-25T00:31:14.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.091+0000 7f4ecffff640 1 --2- 172.21.15.32:0/1289060660 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4ed01006a0 0x7f4ed0197b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:31:14.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.091+0000 7f4ed4f72640 1 --2- 172.21.15.32:0/1289060660 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ed0102250 0x7f4ed019f080 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7f4ec800efc0 tx=0x7f4ec800c800 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:14.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.092+0000 7f4ecd7fa640 1 -- 172.21.15.32:0/1289060660 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4ec8003970 con 0x7f4ed0102250 2024-04-25T00:31:14.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.092+0000 7f4ed69d5640 1 -- 172.21.15.32:0/1289060660 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4ed019f9e0 con 0x7f4ed0102250 2024-04-25T00:31:14.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.092+0000 7f4ecd7fa640 1 -- 172.21.15.32:0/1289060660 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4ec8004020 con 0x7f4ed0102250 2024-04-25T00:31:14.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.092+0000 7f4ecd7fa640 1 -- 172.21.15.32:0/1289060660 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4ec800b9f0 con 0x7f4ed0102250 2024-04-25T00:31:14.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.092+0000 7f4ed69d5640 1 -- 172.21.15.32:0/1289060660 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4ed019ff30 con 0x7f4ed0102250 2024-04-25T00:31:14.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.093+0000 7f4ed69d5640 1 -- 172.21.15.32:0/1289060660 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4ed01a0100 con 0x7f4ed0102250 2024-04-25T00:31:14.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.094+0000 7f4ecd7fa640 1 -- 172.21.15.32:0/1289060660 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f4ec801d020 con 0x7f4ed0102250 2024-04-25T00:31:14.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.095+0000 7f4ecd7fa640 1 --2- 172.21.15.32:0/1289060660 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4ebc0a2b90 0x7f4ebc0a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:14.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.095+0000 7f4ecffff640 1 --2- 172.21.15.32:0/1289060660 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4ebc0a2b90 0x7f4ebc0a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:14.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.095+0000 7f4ecd7fa640 1 -- 172.21.15.32:0/1289060660 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f4ec80cf1a0 con 0x7f4ed0102250 2024-04-25T00:31:14.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.095+0000 7f4ecffff640 1 --2- 172.21.15.32:0/1289060660 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4ebc0a2b90 0x7f4ebc0a5050 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f4ec0009830 tx=0x7f4ec0030570 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:14.101 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.099+0000 7f4ecd7fa640 1 -- 172.21.15.32:0/1289060660 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f4ec809bec0 con 0x7f4ed0102250 2024-04-25T00:31:14.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:13 smithi032 ceph-mon[29357]: pgmap v304: 129 pgs: 129 active+clean; 581 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 426 B/s wr, 0 op/s 2024-04-25T00:31:14.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:13 smithi032 ceph-mon[29357]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-25T00:31:14.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:13 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3622174171' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "3"}]': finished 2024-04-25T00:31:14.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:13 smithi032 ceph-mon[29357]: daemon mds.i assigned to filesystem cephfs as rank 1 (now has 2 ranks) 2024-04-25T00:31:14.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:13 smithi032 ceph-mon[29357]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-25T00:31:14.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:13 smithi032 ceph-mon[29357]: fsmap cephfs:2 {0=b=up:active,1=i=up:creating} 10 up:standby 2024-04-25T00:31:14.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:13 smithi032 ceph-mon[29357]: daemon mds.i is now active in filesystem cephfs as rank 1 2024-04-25T00:31:14.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:13 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/4162244575' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]: dispatch 2024-04-25T00:31:14.331 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.331+0000 7f4ed69d5640 1 -- 172.21.15.32:0/1289060660 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"} v 0) -- 0x7f4ed01a02d0 con 0x7f4ed0102250 2024-04-25T00:31:14.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:13 smithi121 ceph-mon[32231]: pgmap v304: 129 pgs: 129 active+clean; 581 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 426 B/s wr, 0 op/s 2024-04-25T00:31:14.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:13 smithi121 ceph-mon[32231]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-25T00:31:14.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:13 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3622174171' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "3"}]': finished 2024-04-25T00:31:14.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:13 smithi121 ceph-mon[32231]: daemon mds.i assigned to filesystem cephfs as rank 1 (now has 2 ranks) 2024-04-25T00:31:14.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:13 smithi121 ceph-mon[32231]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-25T00:31:14.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:13 smithi121 ceph-mon[32231]: fsmap cephfs:2 {0=b=up:active,1=i=up:creating} 10 up:standby 2024-04-25T00:31:14.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:13 smithi121 ceph-mon[32231]: daemon mds.i is now active in filesystem cephfs as rank 1 2024-04-25T00:31:14.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:13 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/4162244575' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]: dispatch 2024-04-25T00:31:14.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:13 smithi130 ceph-mon[32525]: pgmap v304: 129 pgs: 129 active+clean; 581 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 426 B/s wr, 0 op/s 2024-04-25T00:31:14.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:13 smithi130 ceph-mon[32525]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-25T00:31:14.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:13 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3622174171' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "3"}]': finished 2024-04-25T00:31:14.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:13 smithi130 ceph-mon[32525]: daemon mds.i assigned to filesystem cephfs as rank 1 (now has 2 ranks) 2024-04-25T00:31:14.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:13 smithi130 ceph-mon[32525]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-25T00:31:14.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:13 smithi130 ceph-mon[32525]: fsmap cephfs:2 {0=b=up:active,1=i=up:creating} 10 up:standby 2024-04-25T00:31:14.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:13 smithi130 ceph-mon[32525]: daemon mds.i is now active in filesystem cephfs as rank 1 2024-04-25T00:31:14.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:13 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/4162244575' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]: dispatch 2024-04-25T00:31:14.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.968+0000 7f4ecd7fa640 1 -- 172.21.15.32:0/1289060660 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]=0 v23) ==== 115+0+0 (secure 0 0 0) 0x7f4ec809b7c0 con 0x7f4ed0102250 2024-04-25T00:31:14.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.971+0000 7f4ed69d5640 1 -- 172.21.15.32:0/1289060660 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4ebc0a2b90 msgr2=0x7f4ebc0a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:14.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.971+0000 7f4ed69d5640 1 --2- 172.21.15.32:0/1289060660 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4ebc0a2b90 0x7f4ebc0a5050 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f4ec0009830 tx=0x7f4ec0030570 comp rx=0 tx=0).stop 2024-04-25T00:31:14.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.971+0000 7f4ed69d5640 1 -- 172.21.15.32:0/1289060660 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ed0102250 msgr2=0x7f4ed019f080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:14.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.971+0000 7f4ed69d5640 1 --2- 172.21.15.32:0/1289060660 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ed0102250 0x7f4ed019f080 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7f4ec800efc0 tx=0x7f4ec800c800 comp rx=0 tx=0).stop 2024-04-25T00:31:14.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.971+0000 7f4ed69d5640 1 -- 172.21.15.32:0/1289060660 shutdown_connections 2024-04-25T00:31:14.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.971+0000 7f4ed69d5640 1 --2- 172.21.15.32:0/1289060660 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4ebc0a2b90 0x7f4ebc0a5050 unknown :-1 s=CLOSED pgs=181 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:14.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.971+0000 7f4ed69d5640 1 --2- 172.21.15.32:0/1289060660 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ed0102250 0x7f4ed019f080 unknown :-1 s=CLOSED pgs=265 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:14.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.971+0000 7f4ed69d5640 1 --2- 172.21.15.32:0/1289060660 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ed0101890 0x7f4ed0198060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:14.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.971+0000 7f4ed69d5640 1 --2- 172.21.15.32:0/1289060660 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4ed01006a0 0x7f4ed0197b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:14.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.971+0000 7f4ed69d5640 1 -- 172.21.15.32:0/1289060660 >> 172.21.15.32:0/1289060660 conn(0x7f4ed00fbc30 msgr2=0x7f4ed0069d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:14.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.971+0000 7f4ed69d5640 1 -- 172.21.15.32:0/1289060660 shutdown_connections 2024-04-25T00:31:14.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:14.971+0000 7f4ed69d5640 1 -- 172.21.15.32:0/1289060660 wait complete. 2024-04-25T00:31:14.986 DEBUG:tasks.cephfs.filesystem:Creating {'create': 2} subvols for filesystem 'cephfs' 2024-04-25T00:31:14.986 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs subvolumegroup create cephfs qa 2024-04-25T00:31:15.100 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.096+0000 7f7455495640 1 -- 172.21.15.32:0/2284494246 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7450103350 msgr2=0x7f7450109be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:15.100 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.096+0000 7f7455495640 1 --2- 172.21.15.32:0/2284494246 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7450103350 0x7f7450109be0 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7f7448009ca0 tx=0x7f744802a8c0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.100 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.096+0000 7f7455495640 1 -- 172.21.15.32:0/2284494246 shutdown_connections 2024-04-25T00:31:15.100 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.096+0000 7f7455495640 1 --2- 172.21.15.32:0/2284494246 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7450103350 0x7f7450109be0 unknown :-1 s=CLOSED pgs=266 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.100 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.096+0000 7f7455495640 1 --2- 172.21.15.32:0/2284494246 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7450102990 0x7f7450102e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.100 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.096+0000 7f7455495640 1 --2- 172.21.15.32:0/2284494246 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f74501017a0 0x7f7450101bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.100 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.096+0000 7f7455495640 1 -- 172.21.15.32:0/2284494246 >> 172.21.15.32:0/2284494246 conn(0x7f74500fcd10 msgr2=0x7f74500ff150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:15.100 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.096+0000 7f7455495640 1 -- 172.21.15.32:0/2284494246 shutdown_connections 2024-04-25T00:31:15.100 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.096+0000 7f7455495640 1 -- 172.21.15.32:0/2284494246 wait complete. 2024-04-25T00:31:15.100 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.097+0000 7f7455495640 1 Processor -- start 2024-04-25T00:31:15.101 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.097+0000 7f7455495640 1 -- start start 2024-04-25T00:31:15.101 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.097+0000 7f7455495640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f74501017a0 0x7f74501979c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:15.101 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.097+0000 7f7455495640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7450102990 0x7f7450197f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:15.101 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.097+0000 7f7455495640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7450103350 0x7f745019ef70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:15.101 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.097+0000 7f7455495640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f745010d400 con 0x7f7450102990 2024-04-25T00:31:15.101 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.097+0000 7f7455495640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f745010d280 con 0x7f7450103350 2024-04-25T00:31:15.101 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.097+0000 7f7455495640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f745010d580 con 0x7f74501017a0 2024-04-25T00:31:15.101 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.097+0000 7f744e7fc640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7450102990 0x7f7450197f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:15.102 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.097+0000 7f744e7fc640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7450102990 0x7f7450197f00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:57858/0 (socket says 172.21.15.32:57858) 2024-04-25T00:31:15.102 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.097+0000 7f744e7fc640 1 -- 172.21.15.32:0/2705576093 learned_addr learned my addr 172.21.15.32:0/2705576093 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:15.102 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.097+0000 7f744effd640 1 --2- 172.21.15.32:0/2705576093 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f74501017a0 0x7f74501979c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:15.102 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.097+0000 7f744f7fe640 1 --2- 172.21.15.32:0/2705576093 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7450103350 0x7f745019ef70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:15.102 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.098+0000 7f744e7fc640 1 -- 172.21.15.32:0/2705576093 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f74501017a0 msgr2=0x7f74501979c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:15.102 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.098+0000 7f744e7fc640 1 --2- 172.21.15.32:0/2705576093 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f74501017a0 0x7f74501979c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.102 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.098+0000 7f744e7fc640 1 -- 172.21.15.32:0/2705576093 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7450103350 msgr2=0x7f745019ef70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:15.102 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.098+0000 7f744e7fc640 1 --2- 172.21.15.32:0/2705576093 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7450103350 0x7f745019ef70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.102 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.098+0000 7f744e7fc640 1 -- 172.21.15.32:0/2705576093 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f745019f5e0 con 0x7f7450102990 2024-04-25T00:31:15.102 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.098+0000 7f744e7fc640 1 --2- 172.21.15.32:0/2705576093 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7450102990 0x7f7450197f00 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f744400c920 tx=0x7f744400cde0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:15.102 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.098+0000 7f7437fff640 1 -- 172.21.15.32:0/2705576093 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7444007c70 con 0x7f7450102990 2024-04-25T00:31:15.102 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.098+0000 7f7455495640 1 -- 172.21.15.32:0/2705576093 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f745019f8d0 con 0x7f7450102990 2024-04-25T00:31:15.103 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.098+0000 7f7437fff640 1 -- 172.21.15.32:0/2705576093 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7444007e10 con 0x7f7450102990 2024-04-25T00:31:15.103 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.098+0000 7f7437fff640 1 -- 172.21.15.32:0/2705576093 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7444018890 con 0x7f7450102990 2024-04-25T00:31:15.103 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.098+0000 7f7455495640 1 -- 172.21.15.32:0/2705576093 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f745019fe20 con 0x7f7450102990 2024-04-25T00:31:15.103 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.099+0000 7f7455495640 1 -- 172.21.15.32:0/2705576093 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7450108a60 con 0x7f7450102990 2024-04-25T00:31:15.104 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.100+0000 7f7437fff640 1 -- 172.21.15.32:0/2705576093 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f7444018a70 con 0x7f7450102990 2024-04-25T00:31:15.104 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.101+0000 7f7437fff640 1 --2- 172.21.15.32:0/2705576093 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f74380a2b90 0x7f74380a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:15.104 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.101+0000 7f7437fff640 1 -- 172.21.15.32:0/2705576093 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f74440cf380 con 0x7f7450102990 2024-04-25T00:31:15.104 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.101+0000 7f744effd640 1 --2- 172.21.15.32:0/2705576093 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f74380a2b90 0x7f74380a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:15.104 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.101+0000 7f744effd640 1 --2- 172.21.15.32:0/2705576093 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f74380a2b90 0x7f74380a5050 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f74501027f0 tx=0x7f7440005f90 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:15.105 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.105+0000 7f7437fff640 1 -- 172.21.15.32:0/2705576093 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f7444014030 con 0x7f7450102990 2024-04-25T00:31:15.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:14 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/4162244575' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]': finished 2024-04-25T00:31:15.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:14 smithi032 ceph-mon[29357]: mds.? [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] up:active 2024-04-25T00:31:15.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:14 smithi032 ceph-mon[29357]: daemon mds.h assigned to filesystem cephfs as rank 2 (now has 3 ranks) 2024-04-25T00:31:15.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:14 smithi032 ceph-mon[29357]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-25T00:31:15.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:14 smithi032 ceph-mon[29357]: Cluster is now healthy 2024-04-25T00:31:15.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:14 smithi032 ceph-mon[29357]: fsmap cephfs:2 {0=b=up:active,1=i=up:active} 10 up:standby 2024-04-25T00:31:15.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:14 smithi032 ceph-mon[29357]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:creating} 9 up:standby 2024-04-25T00:31:15.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:14 smithi032 ceph-mon[29357]: daemon mds.h is now active in filesystem cephfs as rank 2 2024-04-25T00:31:15.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:14 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1289060660' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]: dispatch 2024-04-25T00:31:15.317 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.316+0000 7f7455495640 1 -- 172.21.15.32:0/2705576093 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "fs subvolumegroup create", "vol_name": "cephfs", "group_name": "qa", "target": ["mon-mgr", ""]}) -- 0x7f7450061b80 con 0x7f74380a2b90 2024-04-25T00:31:15.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:14 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/4162244575' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]': finished 2024-04-25T00:31:15.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:14 smithi121 ceph-mon[32231]: mds.? [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] up:active 2024-04-25T00:31:15.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:14 smithi121 ceph-mon[32231]: daemon mds.h assigned to filesystem cephfs as rank 2 (now has 3 ranks) 2024-04-25T00:31:15.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:14 smithi121 ceph-mon[32231]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-25T00:31:15.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:14 smithi121 ceph-mon[32231]: Cluster is now healthy 2024-04-25T00:31:15.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:14 smithi121 ceph-mon[32231]: fsmap cephfs:2 {0=b=up:active,1=i=up:active} 10 up:standby 2024-04-25T00:31:15.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:14 smithi121 ceph-mon[32231]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:creating} 9 up:standby 2024-04-25T00:31:15.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:14 smithi121 ceph-mon[32231]: daemon mds.h is now active in filesystem cephfs as rank 2 2024-04-25T00:31:15.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:14 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1289060660' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]: dispatch 2024-04-25T00:31:15.339 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.339+0000 7f7437fff640 1 -- 172.21.15.32:0/2705576093 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+0 (secure 0 0 0) 0x7f7450061b80 con 0x7f74380a2b90 2024-04-25T00:31:15.341 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.341+0000 7f7455495640 1 -- 172.21.15.32:0/2705576093 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f74380a2b90 msgr2=0x7f74380a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:15.341 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.341+0000 7f7455495640 1 --2- 172.21.15.32:0/2705576093 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f74380a2b90 0x7f74380a5050 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f74501027f0 tx=0x7f7440005f90 comp rx=0 tx=0).stop 2024-04-25T00:31:15.341 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.341+0000 7f7455495640 1 -- 172.21.15.32:0/2705576093 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7450102990 msgr2=0x7f7450197f00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:15.342 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.341+0000 7f7455495640 1 --2- 172.21.15.32:0/2705576093 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7450102990 0x7f7450197f00 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f744400c920 tx=0x7f744400cde0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.342 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.341+0000 7f7455495640 1 -- 172.21.15.32:0/2705576093 shutdown_connections 2024-04-25T00:31:15.342 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.341+0000 7f7455495640 1 --2- 172.21.15.32:0/2705576093 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f74380a2b90 0x7f74380a5050 unknown :-1 s=CLOSED pgs=182 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.342 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.341+0000 7f7455495640 1 --2- 172.21.15.32:0/2705576093 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7450103350 0x7f745019ef70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.342 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.341+0000 7f7455495640 1 --2- 172.21.15.32:0/2705576093 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7450102990 0x7f7450197f00 unknown :-1 s=CLOSED pgs=267 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.342 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.341+0000 7f7455495640 1 --2- 172.21.15.32:0/2705576093 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f74501017a0 0x7f74501979c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.342 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.341+0000 7f7455495640 1 -- 172.21.15.32:0/2705576093 >> 172.21.15.32:0/2705576093 conn(0x7f74500fcd10 msgr2=0x7f7450105c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:15.342 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.341+0000 7f7455495640 1 -- 172.21.15.32:0/2705576093 shutdown_connections 2024-04-25T00:31:15.342 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.341+0000 7f7455495640 1 -- 172.21.15.32:0/2705576093 wait complete. 2024-04-25T00:31:15.352 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs subvolume create cephfs sv_0 --group_name qa 2024-04-25T00:31:15.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:14 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/4162244575' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]': finished 2024-04-25T00:31:15.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:14 smithi130 ceph-mon[32525]: mds.? [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] up:active 2024-04-25T00:31:15.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:14 smithi130 ceph-mon[32525]: daemon mds.h assigned to filesystem cephfs as rank 2 (now has 3 ranks) 2024-04-25T00:31:15.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:14 smithi130 ceph-mon[32525]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-25T00:31:15.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:14 smithi130 ceph-mon[32525]: Cluster is now healthy 2024-04-25T00:31:15.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:14 smithi130 ceph-mon[32525]: fsmap cephfs:2 {0=b=up:active,1=i=up:active} 10 up:standby 2024-04-25T00:31:15.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:14 smithi130 ceph-mon[32525]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:creating} 9 up:standby 2024-04-25T00:31:15.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:14 smithi130 ceph-mon[32525]: daemon mds.h is now active in filesystem cephfs as rank 2 2024-04-25T00:31:15.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:14 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1289060660' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]: dispatch 2024-04-25T00:31:15.460 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.458+0000 7ff886a7d640 1 -- 172.21.15.32:0/143830535 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff880102250 msgr2=0x7ff880108ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:15.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.458+0000 7ff886a7d640 1 --2- 172.21.15.32:0/143830535 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff880102250 0x7ff880108ae0 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7ff878009ca0 tx=0x7ff87802a8c0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.458+0000 7ff886a7d640 1 -- 172.21.15.32:0/143830535 shutdown_connections 2024-04-25T00:31:15.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.458+0000 7ff886a7d640 1 --2- 172.21.15.32:0/143830535 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff880102250 0x7ff880108ae0 unknown :-1 s=CLOSED pgs=270 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.458+0000 7ff886a7d640 1 --2- 172.21.15.32:0/143830535 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff880101890 0x7ff880101d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.458+0000 7ff886a7d640 1 --2- 172.21.15.32:0/143830535 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff8801006a0 0x7ff880100ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.458+0000 7ff886a7d640 1 -- 172.21.15.32:0/143830535 >> 172.21.15.32:0/143830535 conn(0x7ff8800fbc30 msgr2=0x7ff8800fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:15.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.458+0000 7ff886a7d640 1 -- 172.21.15.32:0/143830535 shutdown_connections 2024-04-25T00:31:15.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.458+0000 7ff886a7d640 1 -- 172.21.15.32:0/143830535 wait complete. 2024-04-25T00:31:15.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.459+0000 7ff886a7d640 1 Processor -- start 2024-04-25T00:31:15.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.459+0000 7ff886a7d640 1 -- start start 2024-04-25T00:31:15.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.459+0000 7ff886a7d640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff8801006a0 0x7ff880197a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:15.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.459+0000 7ff886a7d640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff880101890 0x7ff880197f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:15.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.459+0000 7ff886a7d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff880102250 0x7ff88019eff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:15.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.459+0000 7ff886a7d640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7ff88010c300 con 0x7ff880102250 2024-04-25T00:31:15.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.459+0000 7ff886a7d640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7ff88010c180 con 0x7ff880101890 2024-04-25T00:31:15.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.459+0000 7ff886a7d640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7ff88010c480 con 0x7ff8801006a0 2024-04-25T00:31:15.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.459+0000 7ff88501a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff880102250 0x7ff88019eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:15.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.459+0000 7ff88501a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff880102250 0x7ff88019eff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:57898/0 (socket says 172.21.15.32:57898) 2024-04-25T00:31:15.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.459+0000 7ff88501a640 1 -- 172.21.15.32:0/2590997164 learned_addr learned my addr 172.21.15.32:0/2590997164 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:15.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.459+0000 7ff87ffff640 1 --2- 172.21.15.32:0/2590997164 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff880101890 0x7ff880197f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:15.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.459+0000 7ff884819640 1 --2- 172.21.15.32:0/2590997164 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff8801006a0 0x7ff880197a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:15.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.459+0000 7ff88501a640 1 -- 172.21.15.32:0/2590997164 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff8801006a0 msgr2=0x7ff880197a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:15.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.459+0000 7ff88501a640 1 --2- 172.21.15.32:0/2590997164 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff8801006a0 0x7ff880197a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.459+0000 7ff88501a640 1 -- 172.21.15.32:0/2590997164 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff880101890 msgr2=0x7ff880197f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:15.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.459+0000 7ff88501a640 1 --2- 172.21.15.32:0/2590997164 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff880101890 0x7ff880197f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.459+0000 7ff88501a640 1 -- 172.21.15.32:0/2590997164 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff88019f660 con 0x7ff880102250 2024-04-25T00:31:15.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.459+0000 7ff884819640 1 --2- 172.21.15.32:0/2590997164 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff8801006a0 0x7ff880197a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:31:15.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.459+0000 7ff88501a640 1 --2- 172.21.15.32:0/2590997164 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff880102250 0x7ff88019eff0 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7ff878031ae0 tx=0x7ff878031ee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:15.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.460+0000 7ff87dffb640 1 -- 172.21.15.32:0/2590997164 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff878003c30 con 0x7ff880102250 2024-04-25T00:31:15.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.460+0000 7ff87dffb640 1 -- 172.21.15.32:0/2590997164 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff878003dd0 con 0x7ff880102250 2024-04-25T00:31:15.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.460+0000 7ff886a7d640 1 -- 172.21.15.32:0/2590997164 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff88019f8f0 con 0x7ff880102250 2024-04-25T00:31:15.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.460+0000 7ff87dffb640 1 -- 172.21.15.32:0/2590997164 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff87802fcb0 con 0x7ff880102250 2024-04-25T00:31:15.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.460+0000 7ff886a7d640 1 -- 172.21.15.32:0/2590997164 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff88019fe90 con 0x7ff880102250 2024-04-25T00:31:15.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.461+0000 7ff886a7d640 1 -- 172.21.15.32:0/2590997164 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff8801a00c0 con 0x7ff880102250 2024-04-25T00:31:15.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.462+0000 7ff87dffb640 1 -- 172.21.15.32:0/2590997164 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7ff878004050 con 0x7ff880102250 2024-04-25T00:31:15.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.462+0000 7ff87dffb640 1 --2- 172.21.15.32:0/2590997164 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff8700a2c10 0x7ff8700a50d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:15.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.463+0000 7ff884819640 1 --2- 172.21.15.32:0/2590997164 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff8700a2c10 0x7ff8700a50d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:15.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.463+0000 7ff87dffb640 1 -- 172.21.15.32:0/2590997164 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7ff8780f04d0 con 0x7ff880102250 2024-04-25T00:31:15.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.463+0000 7ff884819640 1 --2- 172.21.15.32:0/2590997164 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff8700a2c10 0x7ff8700a50d0 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7ff8801016f0 tx=0x7ff874005f90 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:15.467 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.467+0000 7ff87dffb640 1 -- 172.21.15.32:0/2590997164 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7ff8780bd310 con 0x7ff880102250 2024-04-25T00:31:15.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.678+0000 7ff886a7d640 1 -- 172.21.15.32:0/2590997164 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_0", "group_name": "qa", "target": ["mon-mgr", ""]}) -- 0x7ff8800fddc0 con 0x7ff8700a2c10 2024-04-25T00:31:15.702 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.702+0000 7ff87dffb640 1 -- 172.21.15.32:0/2590997164 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+0 (secure 0 0 0) 0x7ff8800fddc0 con 0x7ff8700a2c10 2024-04-25T00:31:15.705 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.704+0000 7ff886a7d640 1 -- 172.21.15.32:0/2590997164 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff8700a2c10 msgr2=0x7ff8700a50d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:15.705 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.704+0000 7ff886a7d640 1 --2- 172.21.15.32:0/2590997164 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff8700a2c10 0x7ff8700a50d0 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7ff8801016f0 tx=0x7ff874005f90 comp rx=0 tx=0).stop 2024-04-25T00:31:15.705 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.704+0000 7ff886a7d640 1 -- 172.21.15.32:0/2590997164 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff880102250 msgr2=0x7ff88019eff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:15.705 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.704+0000 7ff886a7d640 1 --2- 172.21.15.32:0/2590997164 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff880102250 0x7ff88019eff0 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7ff878031ae0 tx=0x7ff878031ee0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.705 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.704+0000 7ff886a7d640 1 -- 172.21.15.32:0/2590997164 shutdown_connections 2024-04-25T00:31:15.705 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.704+0000 7ff886a7d640 1 --2- 172.21.15.32:0/2590997164 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff8700a2c10 0x7ff8700a50d0 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.706 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.704+0000 7ff886a7d640 1 --2- 172.21.15.32:0/2590997164 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff880102250 0x7ff88019eff0 unknown :-1 s=CLOSED pgs=271 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.706 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.704+0000 7ff886a7d640 1 --2- 172.21.15.32:0/2590997164 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff880101890 0x7ff880197f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.706 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.704+0000 7ff886a7d640 1 --2- 172.21.15.32:0/2590997164 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff8801006a0 0x7ff880197a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.706 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.704+0000 7ff886a7d640 1 -- 172.21.15.32:0/2590997164 >> 172.21.15.32:0/2590997164 conn(0x7ff8800fbc30 msgr2=0x7ff8800fd6b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:15.706 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.704+0000 7ff886a7d640 1 -- 172.21.15.32:0/2590997164 shutdown_connections 2024-04-25T00:31:15.706 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.704+0000 7ff886a7d640 1 -- 172.21.15.32:0/2590997164 wait complete. 2024-04-25T00:31:15.714 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:31:15 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:31:15.321+0000 7f9706d21700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-25T00:31:15.714 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:31:15 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:31:15.321+0000 7f9706d21700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-25T00:31:15.715 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:31:15 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:31:15.321+0000 7f9706d21700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-25T00:31:15.715 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:31:15 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:31:15.321+0000 7f9706d21700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-25T00:31:15.715 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:31:15 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:31:15.321+0000 7f9706d21700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-25T00:31:15.716 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs subvolume getpath cephfs --group_name qa sv_0 2024-04-25T00:31:15.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.822+0000 7f1daf4fc640 1 -- 172.21.15.32:0/3502375682 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1da8102980 msgr2=0x7f1da8102e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:15.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.822+0000 7f1daf4fc640 1 --2- 172.21.15.32:0/3502375682 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1da8102980 0x7f1da8102e00 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7f1d98009900 tx=0x7f1d9802a410 comp rx=0 tx=0).stop 2024-04-25T00:31:15.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.823+0000 7f1daf4fc640 1 -- 172.21.15.32:0/3502375682 shutdown_connections 2024-04-25T00:31:15.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.823+0000 7f1daf4fc640 1 --2- 172.21.15.32:0/3502375682 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1da8103340 0x7f1da8109bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.823+0000 7f1daf4fc640 1 --2- 172.21.15.32:0/3502375682 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1da8102980 0x7f1da8102e00 unknown :-1 s=CLOSED pgs=272 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.823+0000 7f1daf4fc640 1 --2- 172.21.15.32:0/3502375682 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1da8101790 0x7f1da8101bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.823+0000 7f1daf4fc640 1 -- 172.21.15.32:0/3502375682 >> 172.21.15.32:0/3502375682 conn(0x7f1da80fcd20 msgr2=0x7f1da80ff140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:15.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.823+0000 7f1daf4fc640 1 -- 172.21.15.32:0/3502375682 shutdown_connections 2024-04-25T00:31:15.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.823+0000 7f1daf4fc640 1 -- 172.21.15.32:0/3502375682 wait complete. 2024-04-25T00:31:15.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.823+0000 7f1daf4fc640 1 Processor -- start 2024-04-25T00:31:15.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.823+0000 7f1daf4fc640 1 -- start start 2024-04-25T00:31:15.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.823+0000 7f1daf4fc640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1da8101790 0x7f1da80754f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:15.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.823+0000 7f1daf4fc640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1da8102980 0x7f1da8073b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:15.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.823+0000 7f1daf4fc640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1da8103340 0x7f1da8074070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:15.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.823+0000 7f1daf4fc640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f1da810d3f0 con 0x7f1da8103340 2024-04-25T00:31:15.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.823+0000 7f1daf4fc640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f1da810d270 con 0x7f1da8101790 2024-04-25T00:31:15.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.823+0000 7f1daf4fc640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f1da810d570 con 0x7f1da8102980 2024-04-25T00:31:15.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.823+0000 7f1dada99640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1da8103340 0x7f1da8074070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:15.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.824+0000 7f1dada99640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1da8103340 0x7f1da8074070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:57924/0 (socket says 172.21.15.32:57924) 2024-04-25T00:31:15.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.824+0000 7f1dada99640 1 -- 172.21.15.32:0/144278104 learned_addr learned my addr 172.21.15.32:0/144278104 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:15.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.824+0000 7f1dad298640 1 --2- 172.21.15.32:0/144278104 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1da8101790 0x7f1da80754f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:15.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.824+0000 7f1daca97640 1 --2- 172.21.15.32:0/144278104 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1da8102980 0x7f1da8073b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:15.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.824+0000 7f1dada99640 1 -- 172.21.15.32:0/144278104 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1da8102980 msgr2=0x7f1da8073b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:15.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.824+0000 7f1dada99640 1 --2- 172.21.15.32:0/144278104 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1da8102980 0x7f1da8073b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.824+0000 7f1dada99640 1 -- 172.21.15.32:0/144278104 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1da8101790 msgr2=0x7f1da80754f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:15.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.824+0000 7f1dada99640 1 --2- 172.21.15.32:0/144278104 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1da8101790 0x7f1da80754f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:15.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.824+0000 7f1dada99640 1 -- 172.21.15.32:0/144278104 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1da8074900 con 0x7f1da8103340 2024-04-25T00:31:15.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.824+0000 7f1daca97640 1 --2- 172.21.15.32:0/144278104 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1da8102980 0x7f1da8073b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:31:15.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.824+0000 7f1dada99640 1 --2- 172.21.15.32:0/144278104 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1da8103340 0x7f1da8074070 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7f1da000eb10 tx=0x7f1da000c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:15.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.824+0000 7f1d9e7fc640 1 -- 172.21.15.32:0/144278104 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1da0004b20 con 0x7f1da8103340 2024-04-25T00:31:15.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.824+0000 7f1d9e7fc640 1 -- 172.21.15.32:0/144278104 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1da0004cc0 con 0x7f1da8103340 2024-04-25T00:31:15.828 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.824+0000 7f1d9e7fc640 1 -- 172.21.15.32:0/144278104 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1da0010760 con 0x7f1da8103340 2024-04-25T00:31:15.828 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.824+0000 7f1daf4fc640 1 -- 172.21.15.32:0/144278104 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1da81a1bb0 con 0x7f1da8103340 2024-04-25T00:31:15.828 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.824+0000 7f1daf4fc640 1 -- 172.21.15.32:0/144278104 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1da81a2080 con 0x7f1da8103340 2024-04-25T00:31:15.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.825+0000 7f1daf4fc640 1 -- 172.21.15.32:0/144278104 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1da8068560 con 0x7f1da8103340 2024-04-25T00:31:15.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.826+0000 7f1d9e7fc640 1 -- 172.21.15.32:0/144278104 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f1da0005020 con 0x7f1da8103340 2024-04-25T00:31:15.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.827+0000 7f1d9e7fc640 1 --2- 172.21.15.32:0/144278104 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f1d940a2b90 0x7f1d940a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:15.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.827+0000 7f1dad298640 1 --2- 172.21.15.32:0/144278104 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f1d940a2b90 0x7f1d940a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:15.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.827+0000 7f1d9e7fc640 1 -- 172.21.15.32:0/144278104 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f1da00cf980 con 0x7f1da8103340 2024-04-25T00:31:15.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.827+0000 7f1dad298640 1 --2- 172.21.15.32:0/144278104 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f1d940a2b90 0x7f1d940a5050 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f1da81027e0 tx=0x7f1da400a2d0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:15.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:15.831+0000 7f1d9e7fc640 1 -- 172.21.15.32:0/144278104 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f1da009c640 con 0x7f1da8103340 2024-04-25T00:31:16.047 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.046+0000 7f1daf4fc640 1 -- 172.21.15.32:0/144278104 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_0", "target": ["mon-mgr", ""]}) -- 0x7f1da8061c10 con 0x7f1d940a2b90 2024-04-25T00:31:16.052 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.051+0000 7f1d9e7fc640 1 -- 172.21.15.32:0/144278104 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+54 (secure 0 0 0) 0x7f1da8061c10 con 0x7f1d940a2b90 2024-04-25T00:31:16.052 INFO:teuthology.orchestra.run.smithi032.stdout:/volumes/qa/sv_0/d23367ad-0ac2-4422-ae7b-65f85c49244e 2024-04-25T00:31:16.054 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.053+0000 7f1daf4fc640 1 -- 172.21.15.32:0/144278104 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f1d940a2b90 msgr2=0x7f1d940a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:16.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.053+0000 7f1daf4fc640 1 --2- 172.21.15.32:0/144278104 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f1d940a2b90 0x7f1d940a5050 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f1da81027e0 tx=0x7f1da400a2d0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.053+0000 7f1daf4fc640 1 -- 172.21.15.32:0/144278104 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1da8103340 msgr2=0x7f1da8074070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:16.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.053+0000 7f1daf4fc640 1 --2- 172.21.15.32:0/144278104 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1da8103340 0x7f1da8074070 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7f1da000eb10 tx=0x7f1da000c2d0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.053+0000 7f1daf4fc640 1 -- 172.21.15.32:0/144278104 shutdown_connections 2024-04-25T00:31:16.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.053+0000 7f1daf4fc640 1 --2- 172.21.15.32:0/144278104 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f1d940a2b90 0x7f1d940a5050 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.053+0000 7f1daf4fc640 1 --2- 172.21.15.32:0/144278104 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1da8103340 0x7f1da8074070 unknown :-1 s=CLOSED pgs=273 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.053+0000 7f1daf4fc640 1 --2- 172.21.15.32:0/144278104 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1da8102980 0x7f1da8073b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.053+0000 7f1daf4fc640 1 --2- 172.21.15.32:0/144278104 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1da8101790 0x7f1da80754f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.053+0000 7f1daf4fc640 1 -- 172.21.15.32:0/144278104 >> 172.21.15.32:0/144278104 conn(0x7f1da80fcd20 msgr2=0x7f1da8105c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:16.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.054+0000 7f1daf4fc640 1 -- 172.21.15.32:0/144278104 shutdown_connections 2024-04-25T00:31:16.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.054+0000 7f1daf4fc640 1 -- 172.21.15.32:0/144278104 wait complete. 2024-04-25T00:31:16.065 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs subvolume create cephfs sv_1 --group_name qa 2024-04-25T00:31:16.175 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.171+0000 7fbadd142640 1 -- 172.21.15.32:0/4084608049 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbad8104b30 msgr2=0x7fbad8106f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:16.175 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.171+0000 7fbadd142640 1 --2- 172.21.15.32:0/4084608049 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbad8104b30 0x7fbad8106f20 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7fbac0009900 tx=0x7fbac002a410 comp rx=0 tx=0).stop 2024-04-25T00:31:16.175 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.171+0000 7fbadd142640 1 -- 172.21.15.32:0/4084608049 shutdown_connections 2024-04-25T00:31:16.175 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.171+0000 7fbadd142640 1 --2- 172.21.15.32:0/4084608049 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbad8107460 0x7fbad81098b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.175 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.171+0000 7fbadd142640 1 --2- 172.21.15.32:0/4084608049 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbad8104b30 0x7fbad8106f20 unknown :-1 s=CLOSED pgs=274 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.175 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.171+0000 7fbadd142640 1 --2- 172.21.15.32:0/4084608049 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbad8102200 0x7fbad81045f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.175 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.171+0000 7fbadd142640 1 -- 172.21.15.32:0/4084608049 >> 172.21.15.32:0/4084608049 conn(0x7fbad80fbbf0 msgr2=0x7fbad80fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:16.175 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.171+0000 7fbadd142640 1 -- 172.21.15.32:0/4084608049 shutdown_connections 2024-04-25T00:31:16.175 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.171+0000 7fbadd142640 1 -- 172.21.15.32:0/4084608049 wait complete. 2024-04-25T00:31:16.175 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.172+0000 7fbadd142640 1 Processor -- start 2024-04-25T00:31:16.176 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.172+0000 7fbadd142640 1 -- start start 2024-04-25T00:31:16.176 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.172+0000 7fbadd142640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbad8102200 0x7fbad8197ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:16.176 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.172+0000 7fbadd142640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbad8104b30 0x7fbad8198000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:16.176 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.172+0000 7fbadd142640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbad8107460 0x7fbad819f070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:16.176 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.172+0000 7fbadd142640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fbad810d3f0 con 0x7fbad8102200 2024-04-25T00:31:16.176 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.172+0000 7fbadd142640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fbad810d270 con 0x7fbad8107460 2024-04-25T00:31:16.176 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.172+0000 7fbadd142640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fbad810d570 con 0x7fbad8104b30 2024-04-25T00:31:16.176 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.172+0000 7fbad6d9d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbad8102200 0x7fbad8197ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:16.176 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.172+0000 7fbad6d9d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbad8102200 0x7fbad8197ac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:57936/0 (socket says 172.21.15.32:57936) 2024-04-25T00:31:16.176 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.172+0000 7fbad6d9d640 1 -- 172.21.15.32:0/1370642405 learned_addr learned my addr 172.21.15.32:0/1370642405 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:16.176 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.173+0000 7fbad759e640 1 --2- 172.21.15.32:0/1370642405 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbad8107460 0x7fbad819f070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:16.176 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.173+0000 7fbacffff640 1 --2- 172.21.15.32:0/1370642405 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbad8104b30 0x7fbad8198000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:16.176 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.173+0000 7fbad6d9d640 1 -- 172.21.15.32:0/1370642405 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbad8104b30 msgr2=0x7fbad8198000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:16.177 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.173+0000 7fbad6d9d640 1 --2- 172.21.15.32:0/1370642405 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbad8104b30 0x7fbad8198000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.177 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.173+0000 7fbad6d9d640 1 -- 172.21.15.32:0/1370642405 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbad8107460 msgr2=0x7fbad819f070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:16.177 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.173+0000 7fbad6d9d640 1 --2- 172.21.15.32:0/1370642405 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbad8107460 0x7fbad819f070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.177 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.173+0000 7fbad6d9d640 1 -- 172.21.15.32:0/1370642405 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbad819f6e0 con 0x7fbad8102200 2024-04-25T00:31:16.177 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.173+0000 7fbad759e640 1 --2- 172.21.15.32:0/1370642405 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbad8107460 0x7fbad819f070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:31:16.177 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.173+0000 7fbad6d9d640 1 --2- 172.21.15.32:0/1370642405 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbad8102200 0x7fbad8197ac0 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7fbac800e9f0 tx=0x7fbac800eeb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:16.177 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.173+0000 7fbad4d99640 1 -- 172.21.15.32:0/1370642405 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbac800cd80 con 0x7fbad8102200 2024-04-25T00:31:16.177 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.173+0000 7fbadd142640 1 -- 172.21.15.32:0/1370642405 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbad819f9d0 con 0x7fbad8102200 2024-04-25T00:31:16.177 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.173+0000 7fbad4d99640 1 -- 172.21.15.32:0/1370642405 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbac8010910 con 0x7fbad8102200 2024-04-25T00:31:16.177 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.173+0000 7fbad4d99640 1 -- 172.21.15.32:0/1370642405 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbac80188f0 con 0x7fbad8102200 2024-04-25T00:31:16.177 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.173+0000 7fbadd142640 1 -- 172.21.15.32:0/1370642405 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fbad819ff40 con 0x7fbad8102200 2024-04-25T00:31:16.177 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.174+0000 7fbadd142640 1 -- 172.21.15.32:0/1370642405 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fbad80fd6e0 con 0x7fbad8102200 2024-04-25T00:31:16.178 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.175+0000 7fbad4d99640 1 -- 172.21.15.32:0/1370642405 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7fbac8010ac0 con 0x7fbad8102200 2024-04-25T00:31:16.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.176+0000 7fbad4d99640 1 --2- 172.21.15.32:0/1370642405 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fbac40a2b90 0x7fbac40a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:16.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.176+0000 7fbacffff640 1 --2- 172.21.15.32:0/1370642405 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fbac40a2b90 0x7fbac40a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:16.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.176+0000 7fbad4d99640 1 -- 172.21.15.32:0/1370642405 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7fbac80cf7d0 con 0x7fbad8102200 2024-04-25T00:31:16.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.176+0000 7fbacffff640 1 --2- 172.21.15.32:0/1370642405 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fbac40a2b90 0x7fbac40a5050 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7fbac0006010 tx=0x7fbac0000bc0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:16.180 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.180+0000 7fbad4d99640 1 -- 172.21.15.32:0/1370642405 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fbac809c490 con 0x7fbad8102200 2024-04-25T00:31:16.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:15 smithi032 ceph-mon[29357]: pgmap v305: 129 pgs: 129 active+clean; 581 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 938 B/s wr, 1 op/s 2024-04-25T00:31:16.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:15 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1289060660' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]': finished 2024-04-25T00:31:16.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:15 smithi032 ceph-mon[29357]: mds.? [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] up:active 2024-04-25T00:31:16.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:15 smithi032 ceph-mon[29357]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 9 up:standby 2024-04-25T00:31:16.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:15 smithi032 ceph-mon[29357]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:31:16.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:15 smithi032 ceph-mon[29357]: from='client.14913 -' entity='client.admin' cmd=[{"prefix": "fs subvolumegroup create", "vol_name": "cephfs", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:31:16.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:15 smithi032 ceph-mon[29357]: from='client.14925 -' entity='client.admin' cmd=[{"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_0", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:31:16.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:15 smithi121 ceph-mon[32231]: pgmap v305: 129 pgs: 129 active+clean; 581 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 938 B/s wr, 1 op/s 2024-04-25T00:31:16.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:15 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1289060660' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]': finished 2024-04-25T00:31:16.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:15 smithi121 ceph-mon[32231]: mds.? [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] up:active 2024-04-25T00:31:16.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:15 smithi121 ceph-mon[32231]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 9 up:standby 2024-04-25T00:31:16.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:15 smithi121 ceph-mon[32231]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:31:16.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:15 smithi121 ceph-mon[32231]: from='client.14913 -' entity='client.admin' cmd=[{"prefix": "fs subvolumegroup create", "vol_name": "cephfs", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:31:16.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:15 smithi121 ceph-mon[32231]: from='client.14925 -' entity='client.admin' cmd=[{"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_0", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:31:16.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:15 smithi130 ceph-mon[32525]: pgmap v305: 129 pgs: 129 active+clean; 581 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 938 B/s wr, 1 op/s 2024-04-25T00:31:16.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:15 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1289060660' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]': finished 2024-04-25T00:31:16.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:15 smithi130 ceph-mon[32525]: mds.? [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] up:active 2024-04-25T00:31:16.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:15 smithi130 ceph-mon[32525]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 9 up:standby 2024-04-25T00:31:16.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:15 smithi130 ceph-mon[32525]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:31:16.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:15 smithi130 ceph-mon[32525]: from='client.14913 -' entity='client.admin' cmd=[{"prefix": "fs subvolumegroup create", "vol_name": "cephfs", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:31:16.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:15 smithi130 ceph-mon[32525]: from='client.14925 -' entity='client.admin' cmd=[{"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_0", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:31:16.396 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.396+0000 7fbadd142640 1 -- 172.21.15.32:0/1370642405 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_1", "group_name": "qa", "target": ["mon-mgr", ""]}) -- 0x7fbad8061c10 con 0x7fbac40a2b90 2024-04-25T00:31:16.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.419+0000 7fbad4d99640 1 -- 172.21.15.32:0/1370642405 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+0 (secure 0 0 0) 0x7fbad8061c10 con 0x7fbac40a2b90 2024-04-25T00:31:16.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.421+0000 7fbadd142640 1 -- 172.21.15.32:0/1370642405 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fbac40a2b90 msgr2=0x7fbac40a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:16.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.421+0000 7fbadd142640 1 --2- 172.21.15.32:0/1370642405 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fbac40a2b90 0x7fbac40a5050 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7fbac0006010 tx=0x7fbac0000bc0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.421+0000 7fbadd142640 1 -- 172.21.15.32:0/1370642405 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbad8102200 msgr2=0x7fbad8197ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:16.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.421+0000 7fbadd142640 1 --2- 172.21.15.32:0/1370642405 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbad8102200 0x7fbad8197ac0 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7fbac800e9f0 tx=0x7fbac800eeb0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.421+0000 7fbadd142640 1 -- 172.21.15.32:0/1370642405 shutdown_connections 2024-04-25T00:31:16.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.421+0000 7fbadd142640 1 --2- 172.21.15.32:0/1370642405 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fbac40a2b90 0x7fbac40a5050 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.421+0000 7fbadd142640 1 --2- 172.21.15.32:0/1370642405 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbad8107460 0x7fbad819f070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.421+0000 7fbadd142640 1 --2- 172.21.15.32:0/1370642405 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbad8104b30 0x7fbad8198000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.421+0000 7fbadd142640 1 --2- 172.21.15.32:0/1370642405 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbad8102200 0x7fbad8197ac0 unknown :-1 s=CLOSED pgs=275 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.421+0000 7fbadd142640 1 -- 172.21.15.32:0/1370642405 >> 172.21.15.32:0/1370642405 conn(0x7fbad80fbbf0 msgr2=0x7fbad8102cc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:16.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.421+0000 7fbadd142640 1 -- 172.21.15.32:0/1370642405 shutdown_connections 2024-04-25T00:31:16.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.421+0000 7fbadd142640 1 -- 172.21.15.32:0/1370642405 wait complete. 2024-04-25T00:31:16.432 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs subvolume getpath cephfs --group_name qa sv_1 2024-04-25T00:31:16.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.538+0000 7f894ac9a640 1 -- 172.21.15.32:0/4286651601 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8944100680 msgr2=0x7f8944100aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:16.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.538+0000 7f894ac9a640 1 --2- 172.21.15.32:0/4286651601 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8944100680 0x7f8944100aa0 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7f8938009900 tx=0x7f893802a410 comp rx=0 tx=0).stop 2024-04-25T00:31:16.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.538+0000 7f894ac9a640 1 -- 172.21.15.32:0/4286651601 shutdown_connections 2024-04-25T00:31:16.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.538+0000 7f894ac9a640 1 --2- 172.21.15.32:0/4286651601 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8944102230 0x7f8944108ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.538+0000 7f894ac9a640 1 --2- 172.21.15.32:0/4286651601 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8944101870 0x7f8944101cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.538+0000 7f894ac9a640 1 --2- 172.21.15.32:0/4286651601 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8944100680 0x7f8944100aa0 unknown :-1 s=CLOSED pgs=276 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.538+0000 7f894ac9a640 1 -- 172.21.15.32:0/4286651601 >> 172.21.15.32:0/4286651601 conn(0x7f89440fbbf0 msgr2=0x7f89440fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:16.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.538+0000 7f894ac9a640 1 -- 172.21.15.32:0/4286651601 shutdown_connections 2024-04-25T00:31:16.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.538+0000 7f894ac9a640 1 -- 172.21.15.32:0/4286651601 wait complete. 2024-04-25T00:31:16.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.539+0000 7f894ac9a640 1 Processor -- start 2024-04-25T00:31:16.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.539+0000 7f894ac9a640 1 -- start start 2024-04-25T00:31:16.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.539+0000 7f894ac9a640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8944100680 0x7f8944069440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:16.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.539+0000 7f894ac9a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8944101870 0x7f894419c810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:16.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.539+0000 7f894ac9a640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8944102230 0x7f894419ee30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:16.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.539+0000 7f894ac9a640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f894410c2d0 con 0x7f8944101870 2024-04-25T00:31:16.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.539+0000 7f894ac9a640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f894410c150 con 0x7f8944100680 2024-04-25T00:31:16.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.539+0000 7f894ac9a640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f894410c450 con 0x7f8944102230 2024-04-25T00:31:16.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.539+0000 7f8943fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8944101870 0x7f894419c810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:16.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.539+0000 7f8943fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8944101870 0x7f894419c810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:57954/0 (socket says 172.21.15.32:57954) 2024-04-25T00:31:16.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.539+0000 7f8943fff640 1 -- 172.21.15.32:0/3816894896 learned_addr learned my addr 172.21.15.32:0/3816894896 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:16.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.539+0000 7f8948a36640 1 --2- 172.21.15.32:0/3816894896 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8944100680 0x7f8944069440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:16.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.539+0000 7f8949237640 1 --2- 172.21.15.32:0/3816894896 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8944102230 0x7f894419ee30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:16.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.540+0000 7f8943fff640 1 -- 172.21.15.32:0/3816894896 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8944102230 msgr2=0x7f894419ee30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:16.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.540+0000 7f8943fff640 1 --2- 172.21.15.32:0/3816894896 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8944102230 0x7f894419ee30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.540+0000 7f8943fff640 1 -- 172.21.15.32:0/3816894896 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8944100680 msgr2=0x7f8944069440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:16.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.540+0000 7f8943fff640 1 --2- 172.21.15.32:0/3816894896 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8944100680 0x7f8944069440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.540+0000 7f8943fff640 1 -- 172.21.15.32:0/3816894896 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f894419f740 con 0x7f8944101870 2024-04-25T00:31:16.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.540+0000 7f8943fff640 1 --2- 172.21.15.32:0/3816894896 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8944101870 0x7f894419c810 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7f893000d950 tx=0x7f893000de10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:16.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.540+0000 7f8941ffb640 1 -- 172.21.15.32:0/3816894896 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8930017070 con 0x7f8944101870 2024-04-25T00:31:16.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.540+0000 7f8941ffb640 1 -- 172.21.15.32:0/3816894896 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8930009c30 con 0x7f8944101870 2024-04-25T00:31:16.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.540+0000 7f8941ffb640 1 -- 172.21.15.32:0/3816894896 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f893000f6b0 con 0x7f8944101870 2024-04-25T00:31:16.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.540+0000 7f894ac9a640 1 -- 172.21.15.32:0/3816894896 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f894419f9d0 con 0x7f8944101870 2024-04-25T00:31:16.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.540+0000 7f894ac9a640 1 -- 172.21.15.32:0/3816894896 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f894419fed0 con 0x7f8944101870 2024-04-25T00:31:16.544 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.541+0000 7f894ac9a640 1 -- 172.21.15.32:0/3816894896 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8944107a90 con 0x7f8944101870 2024-04-25T00:31:16.544 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.542+0000 7f8941ffb640 1 -- 172.21.15.32:0/3816894896 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7f89300109d0 con 0x7f8944101870 2024-04-25T00:31:16.544 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.543+0000 7f8941ffb640 1 --2- 172.21.15.32:0/3816894896 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f89340a2c10 0x7f89340a50d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:16.545 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.543+0000 7f8948a36640 1 --2- 172.21.15.32:0/3816894896 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f89340a2c10 0x7f89340a50d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:16.545 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.543+0000 7f8941ffb640 1 -- 172.21.15.32:0/3816894896 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f89300cee20 con 0x7f8944101870 2024-04-25T00:31:16.545 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.543+0000 7f8948a36640 1 --2- 172.21.15.32:0/3816894896 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f89340a2c10 0x7f89340a50d0 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f8938005ab0 tx=0x7f8938000bc0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:16.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.547+0000 7f8941ffb640 1 -- 172.21.15.32:0/3816894896 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f8930009de0 con 0x7f8944101870 2024-04-25T00:31:16.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.762+0000 7f894ac9a640 1 -- 172.21.15.32:0/3816894896 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_1", "target": ["mon-mgr", ""]}) -- 0x7f8944061b80 con 0x7f89340a2c10 2024-04-25T00:31:16.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.767+0000 7f8941ffb640 1 -- 172.21.15.32:0/3816894896 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+54 (secure 0 0 0) 0x7f8944061b80 con 0x7f89340a2c10 2024-04-25T00:31:16.767 INFO:teuthology.orchestra.run.smithi032.stdout:/volumes/qa/sv_1/13b120f4-666f-4737-942b-13f01a95a021 2024-04-25T00:31:16.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.769+0000 7f894ac9a640 1 -- 172.21.15.32:0/3816894896 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f89340a2c10 msgr2=0x7f89340a50d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:16.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.769+0000 7f894ac9a640 1 --2- 172.21.15.32:0/3816894896 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f89340a2c10 0x7f89340a50d0 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f8938005ab0 tx=0x7f8938000bc0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.769+0000 7f894ac9a640 1 -- 172.21.15.32:0/3816894896 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8944101870 msgr2=0x7f894419c810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:16.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.769+0000 7f894ac9a640 1 --2- 172.21.15.32:0/3816894896 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8944101870 0x7f894419c810 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7f893000d950 tx=0x7f893000de10 comp rx=0 tx=0).stop 2024-04-25T00:31:16.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.769+0000 7f894ac9a640 1 -- 172.21.15.32:0/3816894896 shutdown_connections 2024-04-25T00:31:16.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.769+0000 7f894ac9a640 1 --2- 172.21.15.32:0/3816894896 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f89340a2c10 0x7f89340a50d0 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.769+0000 7f894ac9a640 1 --2- 172.21.15.32:0/3816894896 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8944102230 0x7f894419ee30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.769+0000 7f894ac9a640 1 --2- 172.21.15.32:0/3816894896 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8944101870 0x7f894419c810 unknown :-1 s=CLOSED pgs=277 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.769+0000 7f894ac9a640 1 --2- 172.21.15.32:0/3816894896 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8944100680 0x7f8944069440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.769+0000 7f894ac9a640 1 -- 172.21.15.32:0/3816894896 >> 172.21.15.32:0/3816894896 conn(0x7f89440fbbf0 msgr2=0x7f8944104c60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:16.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.769+0000 7f894ac9a640 1 -- 172.21.15.32:0/3816894896 shutdown_connections 2024-04-25T00:31:16.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.769+0000 7f894ac9a640 1 -- 172.21.15.32:0/3816894896 wait complete. 2024-04-25T00:31:16.781 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:31:16.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.883+0000 7fdadbfce640 1 -- 172.21.15.32:0/3228714447 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdad4102280 msgr2=0x7fdad4104670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:16.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.883+0000 7fdadbfce640 1 --2- 172.21.15.32:0/3228714447 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdad4102280 0x7fdad4104670 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7fdad0009900 tx=0x7fdad002a410 comp rx=0 tx=0).stop 2024-04-25T00:31:16.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.883+0000 7fdadbfce640 1 -- 172.21.15.32:0/3228714447 shutdown_connections 2024-04-25T00:31:16.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.883+0000 7fdadbfce640 1 --2- 172.21.15.32:0/3228714447 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdad41074e0 0x7fdad41098d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.883+0000 7fdadbfce640 1 --2- 172.21.15.32:0/3228714447 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdad4104bb0 0x7fdad4106fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.883+0000 7fdadbfce640 1 --2- 172.21.15.32:0/3228714447 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdad4102280 0x7fdad4104670 unknown :-1 s=CLOSED pgs=278 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.883+0000 7fdadbfce640 1 -- 172.21.15.32:0/3228714447 >> 172.21.15.32:0/3228714447 conn(0x7fdad40fbc30 msgr2=0x7fdad40fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:16.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.883+0000 7fdadbfce640 1 -- 172.21.15.32:0/3228714447 shutdown_connections 2024-04-25T00:31:16.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.883+0000 7fdadbfce640 1 -- 172.21.15.32:0/3228714447 wait complete. 2024-04-25T00:31:16.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.884+0000 7fdadbfce640 1 Processor -- start 2024-04-25T00:31:16.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.884+0000 7fdadbfce640 1 -- start start 2024-04-25T00:31:16.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.884+0000 7fdadbfce640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdad4102280 0x7fdad4101bf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:16.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.884+0000 7fdadbfce640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdad4104bb0 0x7fdad4100230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:16.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.884+0000 7fdadbfce640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdad41074e0 0x7fdad4100770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:16.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.884+0000 7fdadbfce640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fdad410d510 con 0x7fdad4102280 2024-04-25T00:31:16.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.884+0000 7fdadbfce640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fdad410d390 con 0x7fdad4104bb0 2024-04-25T00:31:16.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.884+0000 7fdadbfce640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fdad410d690 con 0x7fdad41074e0 2024-04-25T00:31:16.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.884+0000 7fdad9d6a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdad4102280 0x7fdad4101bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:16.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.884+0000 7fdad9d6a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdad4102280 0x7fdad4101bf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:57982/0 (socket says 172.21.15.32:57982) 2024-04-25T00:31:16.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.884+0000 7fdad9d6a640 1 -- 172.21.15.32:0/2555249369 learned_addr learned my addr 172.21.15.32:0/2555249369 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:16.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.884+0000 7fdad9d6a640 1 -- 172.21.15.32:0/2555249369 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdad41074e0 msgr2=0x7fdad4100770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:16.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.884+0000 7fdad9569640 1 --2- 172.21.15.32:0/2555249369 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdad4104bb0 0x7fdad4100230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:16.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.884+0000 7fdad9d6a640 1 --2- 172.21.15.32:0/2555249369 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdad41074e0 0x7fdad4100770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.884+0000 7fdad9d6a640 1 -- 172.21.15.32:0/2555249369 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdad4104bb0 msgr2=0x7fdad4100230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:16.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.884+0000 7fdad9d6a640 1 --2- 172.21.15.32:0/2555249369 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdad4104bb0 0x7fdad4100230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:16.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.884+0000 7fdad9d6a640 1 -- 172.21.15.32:0/2555249369 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fdad4101000 con 0x7fdad4102280 2024-04-25T00:31:16.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.884+0000 7fdad9569640 1 --2- 172.21.15.32:0/2555249369 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdad4104bb0 0x7fdad4100230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:31:16.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.885+0000 7fdad9d6a640 1 --2- 172.21.15.32:0/2555249369 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdad4102280 0x7fdad4101bf0 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7fdad0009230 tx=0x7fdad0004430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:16.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.885+0000 7fdacaffd640 1 -- 172.21.15.32:0/2555249369 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdad003f030 con 0x7fdad4102280 2024-04-25T00:31:16.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.885+0000 7fdacaffd640 1 -- 172.21.15.32:0/2555249369 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fdad0003ce0 con 0x7fdad4102280 2024-04-25T00:31:16.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.885+0000 7fdacaffd640 1 -- 172.21.15.32:0/2555249369 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdad0035070 con 0x7fdad4102280 2024-04-25T00:31:16.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.885+0000 7fdadbfce640 1 -- 172.21.15.32:0/2555249369 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fdad41a1d00 con 0x7fdad4102280 2024-04-25T00:31:16.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.885+0000 7fdadbfce640 1 -- 172.21.15.32:0/2555249369 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fdad41a21a0 con 0x7fdad4102280 2024-04-25T00:31:16.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.886+0000 7fdadbfce640 1 -- 172.21.15.32:0/2555249369 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fdad40fd720 con 0x7fdad4102280 2024-04-25T00:31:16.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.887+0000 7fdacaffd640 1 -- 172.21.15.32:0/2555249369 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 16) ==== 137833+0+0 (secure 0 0 0) 0x7fdad0042030 con 0x7fdad4102280 2024-04-25T00:31:16.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.887+0000 7fdacaffd640 1 --2- 172.21.15.32:0/2555249369 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdac00a2c10 0x7fdac00a50d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:16.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.888+0000 7fdad9569640 1 --2- 172.21.15.32:0/2555249369 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdac00a2c10 0x7fdac00a50d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:16.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.888+0000 7fdacaffd640 1 -- 172.21.15.32:0/2555249369 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7fdad00efaa0 con 0x7fdad4102280 2024-04-25T00:31:16.890 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.888+0000 7fdad9569640 1 --2- 172.21.15.32:0/2555249369 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdac00a2c10 0x7fdac00a50d0 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7fdac4006fd0 tx=0x7fdac4008040 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:16.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.892+0000 7fdacaffd640 1 -- 172.21.15.32:0/2555249369 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fdad00bc760 con 0x7fdad4102280 2024-04-25T00:31:16.986 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:16.985+0000 7fdacaffd640 1 -- 172.21.15.32:0/2555249369 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7fdad00b86d0 con 0x7fdad4102280 2024-04-25T00:31:17.131 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.130+0000 7fdadbfce640 1 -- 172.21.15.32:0/2555249369 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fdad402d170 con 0x7fdad4102280 2024-04-25T00:31:17.131 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.131+0000 7fdacaffd640 1 -- 172.21.15.32:0/2555249369 <== mon.0 v2:172.21.15.32:3300/0 8 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) ==== 94+0+11876 (secure 0 0 0) 0x7fdad00a7cf0 con 0x7fdad4102280 2024-04-25T00:31:17.132 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:31:17.132 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14643,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6839/187492674","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6838","nonce":187492674},{"type":"v1","addr":"172.21.15.32:6839","nonce":187492674}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14655,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6841/1948785545","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6840","nonce":1948785545},{"type":"v1","addr":"172.21.15.32:6841","nonce":1948785545}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-25T00:30:47.866875+0000","modified":"2024-04-25T00:31:14.972514+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24506,"mds_1":24481,"mds_2":24476},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24443":{"gid":24443,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24445":{"gid":24445,"name":"f","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24457":{"gid":24457,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24476":{"gid":24476,"name":"h","rank":2,"incarnation":22,"state":"up:active","state_seq":12,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24481":{"gid":24481,"name":"i","rank":1,"incarnation":20,"state":"up:active","state_seq":11,"addr":"172.21.15.130:6839/4267125809","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":4267125809},{"type":"v1","addr":"172.21.15.130:6839","nonce":4267125809}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24506":{"gid":24506,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.121:6839/258444853","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6838","nonce":258444853},{"type":"v1","addr":"172.21.15.121:6839","nonce":258444853}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24506,"qdb_cluster":[24476,24481,24506]},"id":1}]} 2024-04-25T00:31:17.132 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 24 2024-04-25T00:31:17.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.133+0000 7fdadbfce640 1 -- 172.21.15.32:0/2555249369 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdac00a2c10 msgr2=0x7fdac00a50d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:17.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.133+0000 7fdadbfce640 1 --2- 172.21.15.32:0/2555249369 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdac00a2c10 0x7fdac00a50d0 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7fdac4006fd0 tx=0x7fdac4008040 comp rx=0 tx=0).stop 2024-04-25T00:31:17.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.133+0000 7fdadbfce640 1 -- 172.21.15.32:0/2555249369 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdad4102280 msgr2=0x7fdad4101bf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:17.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.133+0000 7fdadbfce640 1 --2- 172.21.15.32:0/2555249369 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdad4102280 0x7fdad4101bf0 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7fdad0009230 tx=0x7fdad0004430 comp rx=0 tx=0).stop 2024-04-25T00:31:17.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.133+0000 7fdadbfce640 1 -- 172.21.15.32:0/2555249369 shutdown_connections 2024-04-25T00:31:17.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.133+0000 7fdadbfce640 1 --2- 172.21.15.32:0/2555249369 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdac00a2c10 0x7fdac00a50d0 unknown :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.133+0000 7fdadbfce640 1 --2- 172.21.15.32:0/2555249369 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdad41074e0 0x7fdad4100770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.133+0000 7fdadbfce640 1 --2- 172.21.15.32:0/2555249369 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdad4104bb0 0x7fdad4100230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.135 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.133+0000 7fdadbfce640 1 --2- 172.21.15.32:0/2555249369 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdad4102280 0x7fdad4101bf0 unknown :-1 s=CLOSED pgs=279 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.135 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.133+0000 7fdadbfce640 1 -- 172.21.15.32:0/2555249369 >> 172.21.15.32:0/2555249369 conn(0x7fdad40fbc30 msgr2=0x7fdad4102bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:17.135 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.133+0000 7fdadbfce640 1 -- 172.21.15.32:0/2555249369 shutdown_connections 2024-04-25T00:31:17.135 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.133+0000 7fdadbfce640 1 -- 172.21.15.32:0/2555249369 wait complete. 2024-04-25T00:31:17.143 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:16 smithi032 ceph-mon[29357]: from='client.14931 -' entity='client.admin' cmd=[{"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_0", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:31:17.143 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:16 smithi032 ceph-mon[29357]: from='client.14937 -' entity='client.admin' cmd=[{"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_1", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:31:17.145 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-25T00:31:17.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.246+0000 7f4aee27d640 1 -- 172.21.15.32:0/2375755029 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ae8102250 msgr2=0x7f4ae8108ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:17.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.246+0000 7f4aee27d640 1 --2- 172.21.15.32:0/2375755029 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ae8102250 0x7f4ae8108ae0 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7f4ae0009ca0 tx=0x7f4ae002a8c0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.246+0000 7f4aee27d640 1 -- 172.21.15.32:0/2375755029 shutdown_connections 2024-04-25T00:31:17.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.246+0000 7f4aee27d640 1 --2- 172.21.15.32:0/2375755029 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ae8102250 0x7f4ae8108ae0 unknown :-1 s=CLOSED pgs=280 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.246+0000 7f4aee27d640 1 --2- 172.21.15.32:0/2375755029 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4ae8101890 0x7f4ae8101d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.246+0000 7f4aee27d640 1 --2- 172.21.15.32:0/2375755029 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ae81006a0 0x7f4ae8100ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.246+0000 7f4aee27d640 1 -- 172.21.15.32:0/2375755029 >> 172.21.15.32:0/2375755029 conn(0x7f4ae80fbc30 msgr2=0x7f4ae80fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:17.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.246+0000 7f4aee27d640 1 -- 172.21.15.32:0/2375755029 shutdown_connections 2024-04-25T00:31:17.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.246+0000 7f4aee27d640 1 -- 172.21.15.32:0/2375755029 wait complete. 2024-04-25T00:31:17.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.247+0000 7f4aee27d640 1 Processor -- start 2024-04-25T00:31:17.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.247+0000 7f4aee27d640 1 -- start start 2024-04-25T00:31:17.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.247+0000 7f4aee27d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ae81006a0 0x7f4ae8197a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:17.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.247+0000 7f4aee27d640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ae8101890 0x7f4ae8197f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:17.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.247+0000 7f4aee27d640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4ae8102250 0x7f4ae819eff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:17.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.247+0000 7f4aee27d640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f4ae810d440 con 0x7f4ae81006a0 2024-04-25T00:31:17.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.247+0000 7f4aee27d640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f4ae810d2c0 con 0x7f4ae8102250 2024-04-25T00:31:17.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.247+0000 7f4aee27d640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f4ae810d5c0 con 0x7f4ae8101890 2024-04-25T00:31:17.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.247+0000 7f4ae7fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ae81006a0 0x7f4ae8197a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:17.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.247+0000 7f4ae7fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ae81006a0 0x7f4ae8197a40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:57992/0 (socket says 172.21.15.32:57992) 2024-04-25T00:31:17.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.247+0000 7f4ae7fff640 1 -- 172.21.15.32:0/1358938102 learned_addr learned my addr 172.21.15.32:0/1358938102 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:17.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.247+0000 7f4aec81a640 1 --2- 172.21.15.32:0/1358938102 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4ae8102250 0x7f4ae819eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:17.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.247+0000 7f4ae77fe640 1 --2- 172.21.15.32:0/1358938102 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ae8101890 0x7f4ae8197f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:17.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.248+0000 7f4ae7fff640 1 -- 172.21.15.32:0/1358938102 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ae8101890 msgr2=0x7f4ae8197f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:17.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.248+0000 7f4ae7fff640 1 --2- 172.21.15.32:0/1358938102 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ae8101890 0x7f4ae8197f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.248+0000 7f4ae7fff640 1 -- 172.21.15.32:0/1358938102 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4ae8102250 msgr2=0x7f4ae819eff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:17.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.248+0000 7f4ae7fff640 1 --2- 172.21.15.32:0/1358938102 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4ae8102250 0x7f4ae819eff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.248+0000 7f4ae7fff640 1 -- 172.21.15.32:0/1358938102 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4ae819f660 con 0x7f4ae81006a0 2024-04-25T00:31:17.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.248+0000 7f4ae77fe640 1 --2- 172.21.15.32:0/1358938102 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ae8101890 0x7f4ae8197f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:31:17.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.248+0000 7f4ae7fff640 1 --2- 172.21.15.32:0/1358938102 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ae81006a0 0x7f4ae8197a40 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7f4ad800c970 tx=0x7f4ad800ce30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:17.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.248+0000 7f4ae57fa640 1 -- 172.21.15.32:0/1358938102 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4ad8004220 con 0x7f4ae81006a0 2024-04-25T00:31:17.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.248+0000 7f4ae57fa640 1 -- 172.21.15.32:0/1358938102 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4ad80043c0 con 0x7f4ae81006a0 2024-04-25T00:31:17.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.248+0000 7f4aee27d640 1 -- 172.21.15.32:0/1358938102 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4ae819f950 con 0x7f4ae81006a0 2024-04-25T00:31:17.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.248+0000 7f4ae57fa640 1 -- 172.21.15.32:0/1358938102 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4ad8004a00 con 0x7f4ae81006a0 2024-04-25T00:31:17.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.248+0000 7f4aee27d640 1 -- 172.21.15.32:0/1358938102 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4ae819fe50 con 0x7f4ae81006a0 2024-04-25T00:31:17.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.249+0000 7f4aee27d640 1 -- 172.21.15.32:0/1358938102 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4ae8107960 con 0x7f4ae81006a0 2024-04-25T00:31:17.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.250+0000 7f4ae57fa640 1 -- 172.21.15.32:0/1358938102 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f4ad80079b0 con 0x7f4ae81006a0 2024-04-25T00:31:17.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.250+0000 7f4ae57fa640 1 --2- 172.21.15.32:0/1358938102 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4ad00a2c20 0x7f4ad00a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:17.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.251+0000 7f4ae77fe640 1 --2- 172.21.15.32:0/1358938102 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4ad00a2c20 0x7f4ad00a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:17.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.251+0000 7f4ae57fa640 1 -- 172.21.15.32:0/1358938102 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f4ad80cf230 con 0x7f4ae81006a0 2024-04-25T00:31:17.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.251+0000 7f4ae77fe640 1 --2- 172.21.15.32:0/1358938102 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4ad00a2c20 0x7f4ad00a50e0 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7f4ae8198f50 tx=0x7f4adc00b5f0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:17.255 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.255+0000 7f4ae57fa640 1 -- 172.21.15.32:0/1358938102 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f4ad8014030 con 0x7f4ae81006a0 2024-04-25T00:31:17.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:16 smithi121 ceph-mon[32231]: from='client.14931 -' entity='client.admin' cmd=[{"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_0", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:31:17.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:16 smithi121 ceph-mon[32231]: from='client.14937 -' entity='client.admin' cmd=[{"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_1", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:31:17.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:16 smithi130 ceph-mon[32525]: from='client.14931 -' entity='client.admin' cmd=[{"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_0", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:31:17.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:16 smithi130 ceph-mon[32525]: from='client.14937 -' entity='client.admin' cmd=[{"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_1", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:31:17.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.442+0000 7f4aee27d640 1 -- 172.21.15.32:0/1358938102 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f4ae802d170 con 0x7f4ae81006a0 2024-04-25T00:31:17.444 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.443+0000 7f4ae57fa640 1 -- 172.21.15.32:0/1358938102 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v89) ==== 74+0+24128 (secure 0 0 0) 0x7f4ad809bd90 con 0x7f4ae81006a0 2024-04-25T00:31:17.444 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:31:17.444 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":89,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:30:53.134605+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-25T00:30:46.239556+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-25T00:30:47.257182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2024-04-25T00:30:51.193975+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":83,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-num-failure-domains":"0","crush-osds-per-failure-domain":"0","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:31:17.446 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.445+0000 7f4aee27d640 1 -- 172.21.15.32:0/1358938102 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4ad00a2c20 msgr2=0x7f4ad00a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:17.446 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.445+0000 7f4aee27d640 1 --2- 172.21.15.32:0/1358938102 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4ad00a2c20 0x7f4ad00a50e0 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7f4ae8198f50 tx=0x7f4adc00b5f0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.445+0000 7f4aee27d640 1 -- 172.21.15.32:0/1358938102 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ae81006a0 msgr2=0x7f4ae8197a40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:17.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.445+0000 7f4aee27d640 1 --2- 172.21.15.32:0/1358938102 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ae81006a0 0x7f4ae8197a40 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7f4ad800c970 tx=0x7f4ad800ce30 comp rx=0 tx=0).stop 2024-04-25T00:31:17.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.446+0000 7f4aee27d640 1 -- 172.21.15.32:0/1358938102 shutdown_connections 2024-04-25T00:31:17.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.446+0000 7f4aee27d640 1 --2- 172.21.15.32:0/1358938102 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f4ad00a2c20 0x7f4ad00a50e0 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.446+0000 7f4aee27d640 1 --2- 172.21.15.32:0/1358938102 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4ae8102250 0x7f4ae819eff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.446+0000 7f4aee27d640 1 --2- 172.21.15.32:0/1358938102 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ae8101890 0x7f4ae8197f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.446+0000 7f4aee27d640 1 --2- 172.21.15.32:0/1358938102 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4ae81006a0 0x7f4ae8197a40 unknown :-1 s=CLOSED pgs=281 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.446+0000 7f4aee27d640 1 -- 172.21.15.32:0/1358938102 >> 172.21.15.32:0/1358938102 conn(0x7f4ae80fbc30 msgr2=0x7f4ae8104b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:17.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.446+0000 7f4aee27d640 1 -- 172.21.15.32:0/1358938102 shutdown_connections 2024-04-25T00:31:17.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.446+0000 7f4aee27d640 1 -- 172.21.15.32:0/1358938102 wait complete. 2024-04-25T00:31:17.457 INFO:tasks.cephfs.filesystem.ceph_manager:waiting for clean 2024-04-25T00:31:17.458 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-04-25T00:31:17.562 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.559+0000 7fa692714640 1 -- 172.21.15.32:0/871176555 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa68c1074e0 msgr2=0x7fa68c1098d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:17.562 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.559+0000 7fa692714640 1 --2- 172.21.15.32:0/871176555 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa68c1074e0 0x7fa68c1098d0 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7fa684009ca0 tx=0x7fa68402a8c0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.562 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.559+0000 7fa692714640 1 -- 172.21.15.32:0/871176555 shutdown_connections 2024-04-25T00:31:17.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.559+0000 7fa692714640 1 --2- 172.21.15.32:0/871176555 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa68c1074e0 0x7fa68c1098d0 unknown :-1 s=CLOSED pgs=282 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.559+0000 7fa692714640 1 --2- 172.21.15.32:0/871176555 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa68c104bb0 0x7fa68c106fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.559+0000 7fa692714640 1 --2- 172.21.15.32:0/871176555 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa68c102280 0x7fa68c104670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.559+0000 7fa692714640 1 -- 172.21.15.32:0/871176555 >> 172.21.15.32:0/871176555 conn(0x7fa68c0fbc30 msgr2=0x7fa68c0fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:17.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.559+0000 7fa692714640 1 -- 172.21.15.32:0/871176555 shutdown_connections 2024-04-25T00:31:17.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.559+0000 7fa692714640 1 -- 172.21.15.32:0/871176555 wait complete. 2024-04-25T00:31:17.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.560+0000 7fa692714640 1 Processor -- start 2024-04-25T00:31:17.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.560+0000 7fa692714640 1 -- start start 2024-04-25T00:31:17.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.560+0000 7fa692714640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa68c102280 0x7fa68c197a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:17.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.560+0000 7fa692714640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa68c104bb0 0x7fa68c197fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:17.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.560+0000 7fa692714640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa68c1074e0 0x7fa68c19f030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:17.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.560+0000 7fa692714640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fa68c10d440 con 0x7fa68c1074e0 2024-04-25T00:31:17.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.560+0000 7fa692714640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fa68c10d2c0 con 0x7fa68c102280 2024-04-25T00:31:17.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.560+0000 7fa692714640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fa68c10d5c0 con 0x7fa68c104bb0 2024-04-25T00:31:17.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.560+0000 7fa690cb1640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa68c1074e0 0x7fa68c19f030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:17.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.560+0000 7fa690cb1640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa68c1074e0 0x7fa68c19f030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:58006/0 (socket says 172.21.15.32:58006) 2024-04-25T00:31:17.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.560+0000 7fa690cb1640 1 -- 172.21.15.32:0/2138683591 learned_addr learned my addr 172.21.15.32:0/2138683591 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:17.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.561+0000 7fa68bfff640 1 --2- 172.21.15.32:0/2138683591 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa68c102280 0x7fa68c197a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:17.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.561+0000 7fa68b7fe640 1 --2- 172.21.15.32:0/2138683591 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa68c104bb0 0x7fa68c197fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:17.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.561+0000 7fa690cb1640 1 -- 172.21.15.32:0/2138683591 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa68c104bb0 msgr2=0x7fa68c197fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:17.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.561+0000 7fa690cb1640 1 --2- 172.21.15.32:0/2138683591 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa68c104bb0 0x7fa68c197fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.561+0000 7fa690cb1640 1 -- 172.21.15.32:0/2138683591 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa68c102280 msgr2=0x7fa68c197a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:17.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.561+0000 7fa690cb1640 1 --2- 172.21.15.32:0/2138683591 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa68c102280 0x7fa68c197a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.561+0000 7fa690cb1640 1 -- 172.21.15.32:0/2138683591 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa68c19f6a0 con 0x7fa68c1074e0 2024-04-25T00:31:17.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.561+0000 7fa690cb1640 1 --2- 172.21.15.32:0/2138683591 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa68c1074e0 0x7fa68c19f030 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7fa684031ae0 tx=0x7fa684031ee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:17.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.561+0000 7fa6897fa640 1 -- 172.21.15.32:0/2138683591 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa684003c30 con 0x7fa68c1074e0 2024-04-25T00:31:17.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.561+0000 7fa6897fa640 1 -- 172.21.15.32:0/2138683591 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa684003dd0 con 0x7fa68c1074e0 2024-04-25T00:31:17.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.561+0000 7fa692714640 1 -- 172.21.15.32:0/2138683591 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa68c19f930 con 0x7fa68c1074e0 2024-04-25T00:31:17.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.561+0000 7fa6897fa640 1 -- 172.21.15.32:0/2138683591 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa68402fcb0 con 0x7fa68c1074e0 2024-04-25T00:31:17.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.561+0000 7fa692714640 1 -- 172.21.15.32:0/2138683591 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa68c19fed0 con 0x7fa68c1074e0 2024-04-25T00:31:17.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.562+0000 7fa692714640 1 -- 172.21.15.32:0/2138683591 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa68c0fd720 con 0x7fa68c1074e0 2024-04-25T00:31:17.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.563+0000 7fa6897fa640 1 -- 172.21.15.32:0/2138683591 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7fa684004050 con 0x7fa68c1074e0 2024-04-25T00:31:17.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.564+0000 7fa6897fa640 1 --2- 172.21.15.32:0/2138683591 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa6780a2ca0 0x7fa6780a5160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:17.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.564+0000 7fa68bfff640 1 --2- 172.21.15.32:0/2138683591 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa6780a2ca0 0x7fa6780a5160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:17.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.564+0000 7fa6897fa640 1 -- 172.21.15.32:0/2138683591 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7fa6840f04a0 con 0x7fa68c1074e0 2024-04-25T00:31:17.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.564+0000 7fa68bfff640 1 --2- 172.21.15.32:0/2138683591 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa6780a2ca0 0x7fa6780a5160 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7fa674005fd0 tx=0x7fa67400b390 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:17.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.568+0000 7fa6897fa640 1 -- 172.21.15.32:0/2138683591 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fa6840bd290 con 0x7fa68c1074e0 2024-04-25T00:31:17.750 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.749+0000 7fa692714640 1 -- 172.21.15.32:0/2138683591 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7fa68c061b80 con 0x7fa6780a2ca0 2024-04-25T00:31:17.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.760+0000 7fa6897fa640 1 -- 172.21.15.32:0/2138683591 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+340269 (secure 0 0 0) 0x7fa68c061b80 con 0x7fa6780a2ca0 2024-04-25T00:31:17.765 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:31:17.767 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-04-25T00:31:17.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.763+0000 7fa692714640 1 -- 172.21.15.32:0/2138683591 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa6780a2ca0 msgr2=0x7fa6780a5160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:17.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.763+0000 7fa692714640 1 --2- 172.21.15.32:0/2138683591 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa6780a2ca0 0x7fa6780a5160 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7fa674005fd0 tx=0x7fa67400b390 comp rx=0 tx=0).stop 2024-04-25T00:31:17.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.763+0000 7fa692714640 1 -- 172.21.15.32:0/2138683591 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa68c1074e0 msgr2=0x7fa68c19f030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:17.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.763+0000 7fa692714640 1 --2- 172.21.15.32:0/2138683591 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa68c1074e0 0x7fa68c19f030 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7fa684031ae0 tx=0x7fa684031ee0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.763+0000 7fa692714640 1 -- 172.21.15.32:0/2138683591 shutdown_connections 2024-04-25T00:31:17.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.763+0000 7fa692714640 1 --2- 172.21.15.32:0/2138683591 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa6780a2ca0 0x7fa6780a5160 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.763+0000 7fa692714640 1 --2- 172.21.15.32:0/2138683591 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa68c1074e0 0x7fa68c19f030 unknown :-1 s=CLOSED pgs=283 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.763+0000 7fa692714640 1 --2- 172.21.15.32:0/2138683591 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa68c104bb0 0x7fa68c197fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.763+0000 7fa692714640 1 --2- 172.21.15.32:0/2138683591 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa68c102280 0x7fa68c197a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.763+0000 7fa692714640 1 -- 172.21.15.32:0/2138683591 >> 172.21.15.32:0/2138683591 conn(0x7fa68c0fbc30 msgr2=0x7fa68c102bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:17.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.763+0000 7fa692714640 1 -- 172.21.15.32:0/2138683591 shutdown_connections 2024-04-25T00:31:17.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.763+0000 7fa692714640 1 -- 172.21.15.32:0/2138683591 wait complete. 2024-04-25T00:31:17.775 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":306,"stamp":"2024-04-25T00:31:16.866000+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595142,"num_objects":35,"num_object_clones":0,"num_object_copies":105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":35,"num_whiteouts":0,"num_read":137,"num_read_kb":120,"num_write":258,"num_write_kb":2967,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":22,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":204,"ondisk_log_size":204,"up":419,"acting":419,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":419,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":375064,"kb_used_data":13404,"kb_used_omap":18,"kb_used_meta":361517,"kb_avail":1124566760,"statfs":{"total":1151940427776,"available":1151556362240,"internally_reserved":0,"allocated":13725696,"data_stored":7517346,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19142,"internal_metadata":370193722},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":275,"num_objects":11,"num_object_clones":0,"num_object_copies":33,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":30,"num_write_kb":12,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":8,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003557"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.232735+0000","last_change":"2024-04-25T00:30:48.538719+0000","last_active":"2024-04-25T00:30:54.232735+0000","last_peered":"2024-04-25T00:30:54.232735+0000","last_clean":"2024-04-25T00:30:54.232735+0000","last_became_active":"2024-04-25T00:30:48.538356+0000","last_became_peered":"2024-04-25T00:30:48.538356+0000","last_unstale":"2024-04-25T00:30:54.232735+0000","last_undegraded":"2024-04-25T00:30:54.232735+0000","last_fullsized":"2024-04-25T00:30:54.232735+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:39:08.974838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.677681+0000","last_change":"2024-04-25T00:30:48.532262+0000","last_active":"2024-04-25T00:30:54.677681+0000","last_peered":"2024-04-25T00:30:54.677681+0000","last_clean":"2024-04-25T00:30:54.677681+0000","last_became_active":"2024-04-25T00:30:48.531926+0000","last_became_peered":"2024-04-25T00:30:48.531926+0000","last_unstale":"2024-04-25T00:30:54.677681+0000","last_undegraded":"2024-04-25T00:30:54.677681+0000","last_fullsized":"2024-04-25T00:30:54.677681+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:12:06.527649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.677572+0000","last_change":"2024-04-25T00:30:48.536383+0000","last_active":"2024-04-25T00:30:54.677572+0000","last_peered":"2024-04-25T00:30:54.677572+0000","last_clean":"2024-04-25T00:30:54.677572+0000","last_became_active":"2024-04-25T00:30:48.536048+0000","last_became_peered":"2024-04-25T00:30:48.536048+0000","last_unstale":"2024-04-25T00:30:54.677572+0000","last_undegraded":"2024-04-25T00:30:54.677572+0000","last_fullsized":"2024-04-25T00:30:54.677572+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:32:26.728441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.632025+0000","last_change":"2024-04-25T00:30:48.532195+0000","last_active":"2024-04-25T00:30:54.632025+0000","last_peered":"2024-04-25T00:30:54.632025+0000","last_clean":"2024-04-25T00:30:54.632025+0000","last_became_active":"2024-04-25T00:30:48.531851+0000","last_became_peered":"2024-04-25T00:30:48.531851+0000","last_unstale":"2024-04-25T00:30:54.632025+0000","last_undegraded":"2024-04-25T00:30:54.632025+0000","last_fullsized":"2024-04-25T00:30:54.632025+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:15:34.704958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.630613+0000","last_change":"2024-04-25T00:30:48.534024+0000","last_active":"2024-04-25T00:30:54.630613+0000","last_peered":"2024-04-25T00:30:54.630613+0000","last_clean":"2024-04-25T00:30:54.630613+0000","last_became_active":"2024-04-25T00:30:48.533681+0000","last_became_peered":"2024-04-25T00:30:48.533681+0000","last_unstale":"2024-04-25T00:30:54.630613+0000","last_undegraded":"2024-04-25T00:30:54.630613+0000","last_fullsized":"2024-04-25T00:30:54.630613+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:07:35.816816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.769396+0000","last_change":"2024-04-25T00:30:48.541263+0000","last_active":"2024-04-25T00:30:53.769396+0000","last_peered":"2024-04-25T00:30:53.769396+0000","last_clean":"2024-04-25T00:30:53.769396+0000","last_became_active":"2024-04-25T00:30:48.540889+0000","last_became_peered":"2024-04-25T00:30:48.540889+0000","last_unstale":"2024-04-25T00:30:53.769396+0000","last_undegraded":"2024-04-25T00:30:53.769396+0000","last_fullsized":"2024-04-25T00:30:53.769396+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:31:42.067004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,4],"acting":[10,9,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.130274+0000","last_change":"2024-04-25T00:30:48.533194+0000","last_active":"2024-04-25T00:30:54.130274+0000","last_peered":"2024-04-25T00:30:54.130274+0000","last_clean":"2024-04-25T00:30:54.130274+0000","last_became_active":"2024-04-25T00:30:48.532424+0000","last_became_peered":"2024-04-25T00:30:48.532424+0000","last_unstale":"2024-04-25T00:30:54.130274+0000","last_undegraded":"2024-04-25T00:30:54.130274+0000","last_fullsized":"2024-04-25T00:30:54.130274+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:06:50.187796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.630936+0000","last_change":"2024-04-25T00:30:48.542829+0000","last_active":"2024-04-25T00:30:54.630936+0000","last_peered":"2024-04-25T00:30:54.630936+0000","last_clean":"2024-04-25T00:30:54.630936+0000","last_became_active":"2024-04-25T00:30:48.542204+0000","last_became_peered":"2024-04-25T00:30:48.542204+0000","last_unstale":"2024-04-25T00:30:54.630936+0000","last_undegraded":"2024-04-25T00:30:54.630936+0000","last_fullsized":"2024-04-25T00:30:54.630936+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:56:47.869075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631732+0000","last_change":"2024-04-25T00:30:48.537924+0000","last_active":"2024-04-25T00:30:54.631732+0000","last_peered":"2024-04-25T00:30:54.631732+0000","last_clean":"2024-04-25T00:30:54.631732+0000","last_became_active":"2024-04-25T00:30:48.537547+0000","last_became_peered":"2024-04-25T00:30:48.537547+0000","last_unstale":"2024-04-25T00:30:54.631732+0000","last_undegraded":"2024-04-25T00:30:54.631732+0000","last_fullsized":"2024-04-25T00:30:54.631732+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:26:45.590923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.769787+0000","last_change":"2024-04-25T00:30:48.539717+0000","last_active":"2024-04-25T00:30:53.769787+0000","last_peered":"2024-04-25T00:30:53.769787+0000","last_clean":"2024-04-25T00:30:53.769787+0000","last_became_active":"2024-04-25T00:30:48.539242+0000","last_became_peered":"2024-04-25T00:30:48.539242+0000","last_unstale":"2024-04-25T00:30:53.769787+0000","last_undegraded":"2024-04-25T00:30:53.769787+0000","last_fullsized":"2024-04-25T00:30:53.769787+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:58:15.353959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.130730+0000","last_change":"2024-04-25T00:30:48.542767+0000","last_active":"2024-04-25T00:30:54.130730+0000","last_peered":"2024-04-25T00:30:54.130730+0000","last_clean":"2024-04-25T00:30:54.130730+0000","last_became_active":"2024-04-25T00:30:48.542252+0000","last_became_peered":"2024-04-25T00:30:48.542252+0000","last_unstale":"2024-04-25T00:30:54.130730+0000","last_undegraded":"2024-04-25T00:30:54.130730+0000","last_fullsized":"2024-04-25T00:30:54.130730+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:07:49.414660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631970+0000","last_change":"2024-04-25T00:30:48.532248+0000","last_active":"2024-04-25T00:30:54.631970+0000","last_peered":"2024-04-25T00:30:54.631970+0000","last_clean":"2024-04-25T00:30:54.631970+0000","last_became_active":"2024-04-25T00:30:48.531876+0000","last_became_peered":"2024-04-25T00:30:48.531876+0000","last_unstale":"2024-04-25T00:30:54.631970+0000","last_undegraded":"2024-04-25T00:30:54.631970+0000","last_fullsized":"2024-04-25T00:30:54.631970+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:00:35.711693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631287+0000","last_change":"2024-04-25T00:30:48.526871+0000","last_active":"2024-04-25T00:30:54.631287+0000","last_peered":"2024-04-25T00:30:54.631287+0000","last_clean":"2024-04-25T00:30:54.631287+0000","last_became_active":"2024-04-25T00:30:48.526477+0000","last_became_peered":"2024-04-25T00:30:48.526477+0000","last_unstale":"2024-04-25T00:30:54.631287+0000","last_undegraded":"2024-04-25T00:30:54.631287+0000","last_fullsized":"2024-04-25T00:30:54.631287+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:17:59.447932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.677352+0000","last_change":"2024-04-25T00:30:48.530297+0000","last_active":"2024-04-25T00:30:54.677352+0000","last_peered":"2024-04-25T00:30:54.677352+0000","last_clean":"2024-04-25T00:30:54.677352+0000","last_became_active":"2024-04-25T00:30:48.529091+0000","last_became_peered":"2024-04-25T00:30:48.529091+0000","last_unstale":"2024-04-25T00:30:54.677352+0000","last_undegraded":"2024-04-25T00:30:54.677352+0000","last_fullsized":"2024-04-25T00:30:54.677352+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:54:52.206881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.677314+0000","last_change":"2024-04-25T00:30:48.535629+0000","last_active":"2024-04-25T00:30:54.677314+0000","last_peered":"2024-04-25T00:30:54.677314+0000","last_clean":"2024-04-25T00:30:54.677314+0000","last_became_active":"2024-04-25T00:30:48.535190+0000","last_became_peered":"2024-04-25T00:30:48.535190+0000","last_unstale":"2024-04-25T00:30:54.677314+0000","last_undegraded":"2024-04-25T00:30:54.677314+0000","last_fullsized":"2024-04-25T00:30:54.677314+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:11:27.127964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.630736+0000","last_change":"2024-04-25T00:30:48.532988+0000","last_active":"2024-04-25T00:30:54.630736+0000","last_peered":"2024-04-25T00:30:54.630736+0000","last_clean":"2024-04-25T00:30:54.630736+0000","last_became_active":"2024-04-25T00:30:48.532670+0000","last_became_peered":"2024-04-25T00:30:48.532670+0000","last_unstale":"2024-04-25T00:30:54.630736+0000","last_undegraded":"2024-04-25T00:30:54.630736+0000","last_fullsized":"2024-04-25T00:30:54.630736+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:03:35.336175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.232626+0000","last_change":"2024-04-25T00:30:48.539499+0000","last_active":"2024-04-25T00:30:54.232626+0000","last_peered":"2024-04-25T00:30:54.232626+0000","last_clean":"2024-04-25T00:30:54.232626+0000","last_became_active":"2024-04-25T00:30:48.538516+0000","last_became_peered":"2024-04-25T00:30:48.538516+0000","last_unstale":"2024-04-25T00:30:54.232626+0000","last_undegraded":"2024-04-25T00:30:54.232626+0000","last_fullsized":"2024-04-25T00:30:54.232626+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:03:16.781091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.767084+0000","last_change":"2024-04-25T00:30:48.534204+0000","last_active":"2024-04-25T00:30:53.767084+0000","last_peered":"2024-04-25T00:30:53.767084+0000","last_clean":"2024-04-25T00:30:53.767084+0000","last_became_active":"2024-04-25T00:30:48.533818+0000","last_became_peered":"2024-04-25T00:30:48.533818+0000","last_unstale":"2024-04-25T00:30:53.767084+0000","last_undegraded":"2024-04-25T00:30:53.767084+0000","last_fullsized":"2024-04-25T00:30:53.767084+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:08:42.964056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.767341+0000","last_change":"2024-04-25T00:30:48.542756+0000","last_active":"2024-04-25T00:30:53.767341+0000","last_peered":"2024-04-25T00:30:53.767341+0000","last_clean":"2024-04-25T00:30:53.767341+0000","last_became_active":"2024-04-25T00:30:48.541824+0000","last_became_peered":"2024-04-25T00:30:48.541824+0000","last_unstale":"2024-04-25T00:30:53.767341+0000","last_undegraded":"2024-04-25T00:30:53.767341+0000","last_fullsized":"2024-04-25T00:30:53.767341+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:31:40.996474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.766876+0000","last_change":"2024-04-25T00:30:52.788477+0000","last_active":"2024-04-25T00:30:53.766876+0000","last_peered":"2024-04-25T00:30:53.766876+0000","last_clean":"2024-04-25T00:30:53.766876+0000","last_became_active":"2024-04-25T00:30:52.788073+0000","last_became_peered":"2024-04-25T00:30:52.788073+0000","last_unstale":"2024-04-25T00:30:53.766876+0000","last_undegraded":"2024-04-25T00:30:53.766876+0000","last_fullsized":"2024-04-25T00:30:53.766876+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:29:34.781119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,9,0],"acting":[2,3,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"89'9","reported_seq":33,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:31:13.975508+0000","last_change":"2024-04-25T00:30:47.524580+0000","last_active":"2024-04-25T00:31:13.975508+0000","last_peered":"2024-04-25T00:31:13.975508+0000","last_clean":"2024-04-25T00:31:13.975508+0000","last_became_active":"2024-04-25T00:30:47.524170+0000","last_became_peered":"2024-04-25T00:30:47.524170+0000","last_unstale":"2024-04-25T00:31:13.975508+0000","last_undegraded":"2024-04-25T00:31:13.975508+0000","last_fullsized":"2024-04-25T00:31:13.975508+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:53:08.734842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.632256+0000","last_change":"2024-04-25T00:30:48.542077+0000","last_active":"2024-04-25T00:30:54.632256+0000","last_peered":"2024-04-25T00:30:54.632256+0000","last_clean":"2024-04-25T00:30:54.632256+0000","last_became_active":"2024-04-25T00:30:48.539043+0000","last_became_peered":"2024-04-25T00:30:48.539043+0000","last_unstale":"2024-04-25T00:30:54.632256+0000","last_undegraded":"2024-04-25T00:30:54.632256+0000","last_fullsized":"2024-04-25T00:30:54.632256+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:48:52.254800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.630870+0000","last_change":"2024-04-25T00:30:52.786091+0000","last_active":"2024-04-25T00:30:54.630870+0000","last_peered":"2024-04-25T00:30:54.630870+0000","last_clean":"2024-04-25T00:30:54.630870+0000","last_became_active":"2024-04-25T00:30:52.785511+0000","last_became_peered":"2024-04-25T00:30:52.785511+0000","last_unstale":"2024-04-25T00:30:54.630870+0000","last_undegraded":"2024-04-25T00:30:54.630870+0000","last_fullsized":"2024-04-25T00:30:54.630870+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:19:17.992033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,9,3],"acting":[4,6,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1e","version":"84'1","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.632356+0000","last_change":"2024-04-25T00:30:47.529293+0000","last_active":"2024-04-25T00:30:54.632356+0000","last_peered":"2024-04-25T00:30:54.632356+0000","last_clean":"2024-04-25T00:30:54.632356+0000","last_became_active":"2024-04-25T00:30:47.527975+0000","last_became_peered":"2024-04-25T00:30:47.527975+0000","last_unstale":"2024-04-25T00:30:54.632356+0000","last_undegraded":"2024-04-25T00:30:54.632356+0000","last_fullsized":"2024-04-25T00:30:54.632356+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:17:40.057962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.677687+0000","last_change":"2024-04-25T00:30:48.530993+0000","last_active":"2024-04-25T00:30:54.677687+0000","last_peered":"2024-04-25T00:30:54.677687+0000","last_clean":"2024-04-25T00:30:54.677687+0000","last_became_active":"2024-04-25T00:30:48.530550+0000","last_became_peered":"2024-04-25T00:30:48.530550+0000","last_unstale":"2024-04-25T00:30:54.677687+0000","last_undegraded":"2024-04-25T00:30:54.677687+0000","last_fullsized":"2024-04-25T00:30:54.677687+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:11:35.308456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631715+0000","last_change":"2024-04-25T00:30:52.787774+0000","last_active":"2024-04-25T00:30:54.631715+0000","last_peered":"2024-04-25T00:30:54.631715+0000","last_clean":"2024-04-25T00:30:54.631715+0000","last_became_active":"2024-04-25T00:30:52.786988+0000","last_became_peered":"2024-04-25T00:30:52.786988+0000","last_unstale":"2024-04-25T00:30:54.631715+0000","last_undegraded":"2024-04-25T00:30:54.631715+0000","last_fullsized":"2024-04-25T00:30:54.631715+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:58:28.081767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,9,5],"acting":[4,3,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1d","version":"89'4","reported_seq":27,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:31:13.974259+0000","last_change":"2024-04-25T00:30:47.523477+0000","last_active":"2024-04-25T00:31:13.974259+0000","last_peered":"2024-04-25T00:31:13.974259+0000","last_clean":"2024-04-25T00:31:13.974259+0000","last_became_active":"2024-04-25T00:30:47.522961+0000","last_became_peered":"2024-04-25T00:30:47.522961+0000","last_unstale":"2024-04-25T00:31:13.974259+0000","last_undegraded":"2024-04-25T00:31:13.974259+0000","last_fullsized":"2024-04-25T00:31:13.974259+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:39:39.793477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631367+0000","last_change":"2024-04-25T00:30:48.527746+0000","last_active":"2024-04-25T00:30:54.631367+0000","last_peered":"2024-04-25T00:30:54.631367+0000","last_clean":"2024-04-25T00:30:54.631367+0000","last_became_active":"2024-04-25T00:30:48.527374+0000","last_became_peered":"2024-04-25T00:30:48.527374+0000","last_unstale":"2024-04-25T00:30:54.631367+0000","last_undegraded":"2024-04-25T00:30:54.631367+0000","last_fullsized":"2024-04-25T00:30:54.631367+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:28:36.822220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631458+0000","last_change":"2024-04-25T00:30:52.782368+0000","last_active":"2024-04-25T00:30:54.631458+0000","last_peered":"2024-04-25T00:30:54.631458+0000","last_clean":"2024-04-25T00:30:54.631458+0000","last_became_active":"2024-04-25T00:30:52.781875+0000","last_became_peered":"2024-04-25T00:30:52.781875+0000","last_unstale":"2024-04-25T00:30:54.631458+0000","last_undegraded":"2024-04-25T00:30:54.631458+0000","last_fullsized":"2024-04-25T00:30:54.631458+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:07:55.509131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5,3],"acting":[4,6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1c","version":"84'2","reported_seq":25,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631104+0000","last_change":"2024-04-25T00:30:47.523707+0000","last_active":"2024-04-25T00:30:54.631104+0000","last_peered":"2024-04-25T00:30:54.631104+0000","last_clean":"2024-04-25T00:30:54.631104+0000","last_became_active":"2024-04-25T00:30:47.523216+0000","last_became_peered":"2024-04-25T00:30:47.523216+0000","last_unstale":"2024-04-25T00:30:54.631104+0000","last_undegraded":"2024-04-25T00:30:54.631104+0000","last_fullsized":"2024-04-25T00:30:54.631104+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:21:21.006368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631375+0000","last_change":"2024-04-25T00:30:48.529235+0000","last_active":"2024-04-25T00:30:54.631375+0000","last_peered":"2024-04-25T00:30:54.631375+0000","last_clean":"2024-04-25T00:30:54.631375+0000","last_became_active":"2024-04-25T00:30:48.528887+0000","last_became_peered":"2024-04-25T00:30:48.528887+0000","last_unstale":"2024-04-25T00:30:54.631375+0000","last_undegraded":"2024-04-25T00:30:54.631375+0000","last_fullsized":"2024-04-25T00:30:54.631375+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:18:05.290578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.767234+0000","last_change":"2024-04-25T00:30:52.784576+0000","last_active":"2024-04-25T00:30:53.767234+0000","last_peered":"2024-04-25T00:30:53.767234+0000","last_clean":"2024-04-25T00:30:53.767234+0000","last_became_active":"2024-04-25T00:30:52.784027+0000","last_became_peered":"2024-04-25T00:30:52.784027+0000","last_unstale":"2024-04-25T00:30:53.767234+0000","last_undegraded":"2024-04-25T00:30:53.767234+0000","last_fullsized":"2024-04-25T00:30:53.767234+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:31:18.798072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7,11],"acting":[2,5,7,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1b","version":"84'1","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631112+0000","last_change":"2024-04-25T00:30:47.518472+0000","last_active":"2024-04-25T00:30:54.631112+0000","last_peered":"2024-04-25T00:30:54.631112+0000","last_clean":"2024-04-25T00:30:54.631112+0000","last_became_active":"2024-04-25T00:30:47.518140+0000","last_became_peered":"2024-04-25T00:30:47.518140+0000","last_unstale":"2024-04-25T00:30:54.631112+0000","last_undegraded":"2024-04-25T00:30:54.631112+0000","last_fullsized":"2024-04-25T00:30:54.631112+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:19:07.780949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.630861+0000","last_change":"2024-04-25T00:30:48.533757+0000","last_active":"2024-04-25T00:30:54.630861+0000","last_peered":"2024-04-25T00:30:54.630861+0000","last_clean":"2024-04-25T00:30:54.630861+0000","last_became_active":"2024-04-25T00:30:48.533421+0000","last_became_peered":"2024-04-25T00:30:48.533421+0000","last_unstale":"2024-04-25T00:30:54.630861+0000","last_undegraded":"2024-04-25T00:30:54.630861+0000","last_fullsized":"2024-04-25T00:30:54.630861+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:11:38.190783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.767144+0000","last_change":"2024-04-25T00:30:52.785179+0000","last_active":"2024-04-25T00:30:53.767144+0000","last_peered":"2024-04-25T00:30:53.767144+0000","last_clean":"2024-04-25T00:30:53.767144+0000","last_became_active":"2024-04-25T00:30:52.784761+0000","last_became_peered":"2024-04-25T00:30:52.784761+0000","last_unstale":"2024-04-25T00:30:53.767144+0000","last_undegraded":"2024-04-25T00:30:53.767144+0000","last_fullsized":"2024-04-25T00:30:53.767144+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:57:37.449915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,9,5],"acting":[2,6,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1a","version":"84'1","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.769354+0000","last_change":"2024-04-25T00:30:47.527306+0000","last_active":"2024-04-25T00:30:53.769354+0000","last_peered":"2024-04-25T00:30:53.769354+0000","last_clean":"2024-04-25T00:30:53.769354+0000","last_became_active":"2024-04-25T00:30:47.526935+0000","last_became_peered":"2024-04-25T00:30:47.526935+0000","last_unstale":"2024-04-25T00:30:53.769354+0000","last_undegraded":"2024-04-25T00:30:53.769354+0000","last_fullsized":"2024-04-25T00:30:53.769354+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:41:37.951143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631197+0000","last_change":"2024-04-25T00:30:48.536040+0000","last_active":"2024-04-25T00:30:54.631197+0000","last_peered":"2024-04-25T00:30:54.631197+0000","last_clean":"2024-04-25T00:30:54.631197+0000","last_became_active":"2024-04-25T00:30:48.535399+0000","last_became_peered":"2024-04-25T00:30:48.535399+0000","last_unstale":"2024-04-25T00:30:54.631197+0000","last_undegraded":"2024-04-25T00:30:54.631197+0000","last_fullsized":"2024-04-25T00:30:54.631197+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:44:42.906907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.232625+0000","last_change":"2024-04-25T00:30:52.784986+0000","last_active":"2024-04-25T00:30:54.232625+0000","last_peered":"2024-04-25T00:30:54.232625+0000","last_clean":"2024-04-25T00:30:54.232625+0000","last_became_active":"2024-04-25T00:30:52.784593+0000","last_became_peered":"2024-04-25T00:30:52.784593+0000","last_unstale":"2024-04-25T00:30:54.232625+0000","last_undegraded":"2024-04-25T00:30:54.232625+0000","last_fullsized":"2024-04-25T00:30:54.232625+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:04:05.833730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,9,0],"acting":[11,4,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631512+0000","last_change":"2024-04-25T00:30:47.523663+0000","last_active":"2024-04-25T00:30:54.631512+0000","last_peered":"2024-04-25T00:30:54.631512+0000","last_clean":"2024-04-25T00:30:54.631512+0000","last_became_active":"2024-04-25T00:30:47.523324+0000","last_became_peered":"2024-04-25T00:30:47.523324+0000","last_unstale":"2024-04-25T00:30:54.631512+0000","last_undegraded":"2024-04-25T00:30:54.631512+0000","last_fullsized":"2024-04-25T00:30:54.631512+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:51:23.593579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.630759+0000","last_change":"2024-04-25T00:30:48.529738+0000","last_active":"2024-04-25T00:30:54.630759+0000","last_peered":"2024-04-25T00:30:54.630759+0000","last_clean":"2024-04-25T00:30:54.630759+0000","last_became_active":"2024-04-25T00:30:48.529350+0000","last_became_peered":"2024-04-25T00:30:48.529350+0000","last_unstale":"2024-04-25T00:30:54.630759+0000","last_undegraded":"2024-04-25T00:30:54.630759+0000","last_fullsized":"2024-04-25T00:30:54.630759+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:51:29.238293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.677788+0000","last_change":"2024-04-25T00:30:52.788480+0000","last_active":"2024-04-25T00:30:54.677788+0000","last_peered":"2024-04-25T00:30:54.677788+0000","last_clean":"2024-04-25T00:30:54.677788+0000","last_became_active":"2024-04-25T00:30:52.788065+0000","last_became_peered":"2024-04-25T00:30:52.788065+0000","last_unstale":"2024-04-25T00:30:54.677788+0000","last_undegraded":"2024-04-25T00:30:54.677788+0000","last_fullsized":"2024-04-25T00:30:54.677788+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:16:50.179177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6,1],"acting":[0,7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.630894+0000","last_change":"2024-04-25T00:30:47.524706+0000","last_active":"2024-04-25T00:30:54.630894+0000","last_peered":"2024-04-25T00:30:54.630894+0000","last_clean":"2024-04-25T00:30:54.630894+0000","last_became_active":"2024-04-25T00:30:47.524355+0000","last_became_peered":"2024-04-25T00:30:47.524355+0000","last_unstale":"2024-04-25T00:30:54.630894+0000","last_undegraded":"2024-04-25T00:30:54.630894+0000","last_fullsized":"2024-04-25T00:30:54.630894+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:22:38.093362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.766423+0000","last_change":"2024-04-25T00:30:48.533266+0000","last_active":"2024-04-25T00:30:53.766423+0000","last_peered":"2024-04-25T00:30:53.766423+0000","last_clean":"2024-04-25T00:30:53.766423+0000","last_became_active":"2024-04-25T00:30:48.532932+0000","last_became_peered":"2024-04-25T00:30:48.532932+0000","last_unstale":"2024-04-25T00:30:53.766423+0000","last_undegraded":"2024-04-25T00:30:53.766423+0000","last_fullsized":"2024-04-25T00:30:53.766423+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:00:54.147725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631874+0000","last_change":"2024-04-25T00:30:52.780317+0000","last_active":"2024-04-25T00:30:54.631874+0000","last_peered":"2024-04-25T00:30:54.631874+0000","last_clean":"2024-04-25T00:30:54.631874+0000","last_became_active":"2024-04-25T00:30:52.778539+0000","last_became_peered":"2024-04-25T00:30:52.778539+0000","last_unstale":"2024-04-25T00:30:54.631874+0000","last_undegraded":"2024-04-25T00:30:54.631874+0000","last_fullsized":"2024-04-25T00:30:54.631874+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:34:49.527307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1,5],"acting":[3,7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.769733+0000","last_change":"2024-04-25T00:30:47.525647+0000","last_active":"2024-04-25T00:30:53.769733+0000","last_peered":"2024-04-25T00:30:53.769733+0000","last_clean":"2024-04-25T00:30:53.769733+0000","last_became_active":"2024-04-25T00:30:47.525280+0000","last_became_peered":"2024-04-25T00:30:47.525280+0000","last_unstale":"2024-04-25T00:30:53.769733+0000","last_undegraded":"2024-04-25T00:30:53.769733+0000","last_fullsized":"2024-04-25T00:30:53.769733+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:56:15.761173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631401+0000","last_change":"2024-04-25T00:30:48.527913+0000","last_active":"2024-04-25T00:30:54.631401+0000","last_peered":"2024-04-25T00:30:54.631401+0000","last_clean":"2024-04-25T00:30:54.631401+0000","last_became_active":"2024-04-25T00:30:48.527538+0000","last_became_peered":"2024-04-25T00:30:48.527538+0000","last_unstale":"2024-04-25T00:30:54.631401+0000","last_undegraded":"2024-04-25T00:30:54.631401+0000","last_fullsized":"2024-04-25T00:30:54.631401+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:48:38.646867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.766316+0000","last_change":"2024-04-25T00:30:52.788267+0000","last_active":"2024-04-25T00:30:53.766316+0000","last_peered":"2024-04-25T00:30:53.766316+0000","last_clean":"2024-04-25T00:30:53.766316+0000","last_became_active":"2024-04-25T00:30:52.785188+0000","last_became_peered":"2024-04-25T00:30:52.785188+0000","last_unstale":"2024-04-25T00:30:53.766316+0000","last_undegraded":"2024-04-25T00:30:53.766316+0000","last_fullsized":"2024-04-25T00:30:53.766316+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:38:25.194673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,0,2,4],"acting":[9,0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.16","version":"89'2","reported_seq":25,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:31:12.963192+0000","last_change":"2024-04-25T00:30:47.522213+0000","last_active":"2024-04-25T00:31:12.963192+0000","last_peered":"2024-04-25T00:31:12.963192+0000","last_clean":"2024-04-25T00:31:12.963192+0000","last_became_active":"2024-04-25T00:30:47.521519+0000","last_became_peered":"2024-04-25T00:30:47.521519+0000","last_unstale":"2024-04-25T00:31:12.963192+0000","last_undegraded":"2024-04-25T00:31:12.963192+0000","last_fullsized":"2024-04-25T00:31:12.963192+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:16:10.001222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.677800+0000","last_change":"2024-04-25T00:30:48.540784+0000","last_active":"2024-04-25T00:30:54.677800+0000","last_peered":"2024-04-25T00:30:54.677800+0000","last_clean":"2024-04-25T00:30:54.677800+0000","last_became_active":"2024-04-25T00:30:48.539782+0000","last_became_peered":"2024-04-25T00:30:48.539782+0000","last_unstale":"2024-04-25T00:30:54.677800+0000","last_undegraded":"2024-04-25T00:30:54.677800+0000","last_fullsized":"2024-04-25T00:30:54.677800+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:45:10.701113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631980+0000","last_change":"2024-04-25T00:30:52.786818+0000","last_active":"2024-04-25T00:30:54.631980+0000","last_peered":"2024-04-25T00:30:54.631980+0000","last_clean":"2024-04-25T00:30:54.631980+0000","last_became_active":"2024-04-25T00:30:52.786293+0000","last_became_peered":"2024-04-25T00:30:52.786293+0000","last_unstale":"2024-04-25T00:30:54.631980+0000","last_undegraded":"2024-04-25T00:30:54.631980+0000","last_fullsized":"2024-04-25T00:30:54.631980+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:27:02.751296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2,9],"acting":[4,1,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.15","version":"84'3","reported_seq":26,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631242+0000","last_change":"2024-04-25T00:30:47.520882+0000","last_active":"2024-04-25T00:30:54.631242+0000","last_peered":"2024-04-25T00:30:54.631242+0000","last_clean":"2024-04-25T00:30:54.631242+0000","last_became_active":"2024-04-25T00:30:47.520545+0000","last_became_peered":"2024-04-25T00:30:47.520545+0000","last_unstale":"2024-04-25T00:30:54.631242+0000","last_undegraded":"2024-04-25T00:30:54.631242+0000","last_fullsized":"2024-04-25T00:30:54.631242+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:46:04.715580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631561+0000","last_change":"2024-04-25T00:30:48.536814+0000","last_active":"2024-04-25T00:30:54.631561+0000","last_peered":"2024-04-25T00:30:54.631561+0000","last_clean":"2024-04-25T00:30:54.631561+0000","last_became_active":"2024-04-25T00:30:48.536486+0000","last_became_peered":"2024-04-25T00:30:48.536486+0000","last_unstale":"2024-04-25T00:30:54.631561+0000","last_undegraded":"2024-04-25T00:30:54.631561+0000","last_fullsized":"2024-04-25T00:30:54.631561+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:58:23.861959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.766639+0000","last_change":"2024-04-25T00:30:52.786070+0000","last_active":"2024-04-25T00:30:53.766639+0000","last_peered":"2024-04-25T00:30:53.766639+0000","last_clean":"2024-04-25T00:30:53.766639+0000","last_became_active":"2024-04-25T00:30:52.785582+0000","last_became_peered":"2024-04-25T00:30:52.785582+0000","last_unstale":"2024-04-25T00:30:53.766639+0000","last_undegraded":"2024-04-25T00:30:53.766639+0000","last_fullsized":"2024-04-25T00:30:53.766639+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:41:59.584597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,8,11,5],"acting":[9,8,11,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.14","version":"89'10","reported_seq":33,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:31:04.470078+0000","last_change":"2024-04-25T00:30:47.526899+0000","last_active":"2024-04-25T00:31:04.470078+0000","last_peered":"2024-04-25T00:31:04.470078+0000","last_clean":"2024-04-25T00:31:04.470078+0000","last_became_active":"2024-04-25T00:30:47.526528+0000","last_became_peered":"2024-04-25T00:30:47.526528+0000","last_unstale":"2024-04-25T00:31:04.470078+0000","last_undegraded":"2024-04-25T00:31:04.470078+0000","last_fullsized":"2024-04-25T00:31:04.470078+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:13:34.004584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3842,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631381+0000","last_change":"2024-04-25T00:30:48.538225+0000","last_active":"2024-04-25T00:30:54.631381+0000","last_peered":"2024-04-25T00:30:54.631381+0000","last_clean":"2024-04-25T00:30:54.631381+0000","last_became_active":"2024-04-25T00:30:48.537807+0000","last_became_peered":"2024-04-25T00:30:48.537807+0000","last_unstale":"2024-04-25T00:30:54.631381+0000","last_undegraded":"2024-04-25T00:30:54.631381+0000","last_fullsized":"2024-04-25T00:30:54.631381+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:00:46.275584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631725+0000","last_change":"2024-04-25T00:30:52.782266+0000","last_active":"2024-04-25T00:30:54.631725+0000","last_peered":"2024-04-25T00:30:54.631725+0000","last_clean":"2024-04-25T00:30:54.631725+0000","last_became_active":"2024-04-25T00:30:52.781133+0000","last_became_peered":"2024-04-25T00:30:52.781133+0000","last_unstale":"2024-04-25T00:30:54.631725+0000","last_undegraded":"2024-04-25T00:30:54.631725+0000","last_fullsized":"2024-04-25T00:30:54.631725+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:19:03.818560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,8,3,6],"acting":[5,8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.13","version":"84'2","reported_seq":25,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.767022+0000","last_change":"2024-04-25T00:30:47.520909+0000","last_active":"2024-04-25T00:30:53.767022+0000","last_peered":"2024-04-25T00:30:53.767022+0000","last_clean":"2024-04-25T00:30:53.767022+0000","last_became_active":"2024-04-25T00:30:47.520287+0000","last_became_peered":"2024-04-25T00:30:47.520287+0000","last_unstale":"2024-04-25T00:30:53.767022+0000","last_undegraded":"2024-04-25T00:30:53.767022+0000","last_fullsized":"2024-04-25T00:30:53.767022+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:49:43.795115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631201+0000","last_change":"2024-04-25T00:30:48.532270+0000","last_active":"2024-04-25T00:30:54.631201+0000","last_peered":"2024-04-25T00:30:54.631201+0000","last_clean":"2024-04-25T00:30:54.631201+0000","last_became_active":"2024-04-25T00:30:48.531882+0000","last_became_peered":"2024-04-25T00:30:48.531882+0000","last_unstale":"2024-04-25T00:30:54.631201+0000","last_undegraded":"2024-04-25T00:30:54.631201+0000","last_fullsized":"2024-04-25T00:30:54.631201+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:58:17.865635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.232374+0000","last_change":"2024-04-25T00:30:52.777708+0000","last_active":"2024-04-25T00:30:54.232374+0000","last_peered":"2024-04-25T00:30:54.232374+0000","last_clean":"2024-04-25T00:30:54.232374+0000","last_became_active":"2024-04-25T00:30:52.777254+0000","last_became_peered":"2024-04-25T00:30:52.777254+0000","last_unstale":"2024-04-25T00:30:54.232374+0000","last_undegraded":"2024-04-25T00:30:54.232374+0000","last_fullsized":"2024-04-25T00:30:54.232374+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:29:47.427190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,9,7,6],"acting":[11,9,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631506+0000","last_change":"2024-04-25T00:30:47.524667+0000","last_active":"2024-04-25T00:30:54.631506+0000","last_peered":"2024-04-25T00:30:54.631506+0000","last_clean":"2024-04-25T00:30:54.631506+0000","last_became_active":"2024-04-25T00:30:47.524361+0000","last_became_peered":"2024-04-25T00:30:47.524361+0000","last_unstale":"2024-04-25T00:30:54.631506+0000","last_undegraded":"2024-04-25T00:30:54.631506+0000","last_fullsized":"2024-04-25T00:30:54.631506+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:16:08.805720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.130366+0000","last_change":"2024-04-25T00:30:48.538658+0000","last_active":"2024-04-25T00:30:54.130366+0000","last_peered":"2024-04-25T00:30:54.130366+0000","last_clean":"2024-04-25T00:30:54.130366+0000","last_became_active":"2024-04-25T00:30:48.538316+0000","last_became_peered":"2024-04-25T00:30:48.538316+0000","last_unstale":"2024-04-25T00:30:54.130366+0000","last_undegraded":"2024-04-25T00:30:54.130366+0000","last_fullsized":"2024-04-25T00:30:54.130366+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:24:47.303559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.767144+0000","last_change":"2024-04-25T00:30:52.789512+0000","last_active":"2024-04-25T00:30:53.767144+0000","last_peered":"2024-04-25T00:30:53.767144+0000","last_clean":"2024-04-25T00:30:53.767144+0000","last_became_active":"2024-04-25T00:30:52.788929+0000","last_became_peered":"2024-04-25T00:30:52.788929+0000","last_unstale":"2024-04-25T00:30:53.767144+0000","last_undegraded":"2024-04-25T00:30:53.767144+0000","last_fullsized":"2024-04-25T00:30:53.767144+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:21:05.195296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,0,3],"acting":[9,7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631352+0000","last_change":"2024-04-25T00:30:47.513627+0000","last_active":"2024-04-25T00:30:54.631352+0000","last_peered":"2024-04-25T00:30:54.631352+0000","last_clean":"2024-04-25T00:30:54.631352+0000","last_became_active":"2024-04-25T00:30:47.513215+0000","last_became_peered":"2024-04-25T00:30:47.513215+0000","last_unstale":"2024-04-25T00:30:54.631352+0000","last_undegraded":"2024-04-25T00:30:54.631352+0000","last_fullsized":"2024-04-25T00:30:54.631352+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:32:16.111590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.130421+0000","last_change":"2024-04-25T00:30:48.533056+0000","last_active":"2024-04-25T00:30:54.130421+0000","last_peered":"2024-04-25T00:30:54.130421+0000","last_clean":"2024-04-25T00:30:54.130421+0000","last_became_active":"2024-04-25T00:30:48.532698+0000","last_became_peered":"2024-04-25T00:30:48.532698+0000","last_unstale":"2024-04-25T00:30:54.130421+0000","last_undegraded":"2024-04-25T00:30:54.130421+0000","last_fullsized":"2024-04-25T00:30:54.130421+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:36:24.987784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631231+0000","last_change":"2024-04-25T00:30:52.780669+0000","last_active":"2024-04-25T00:30:54.631231+0000","last_peered":"2024-04-25T00:30:54.631231+0000","last_clean":"2024-04-25T00:30:54.631231+0000","last_became_active":"2024-04-25T00:30:52.780222+0000","last_became_peered":"2024-04-25T00:30:52.780222+0000","last_unstale":"2024-04-25T00:30:54.631231+0000","last_undegraded":"2024-04-25T00:30:54.631231+0000","last_fullsized":"2024-04-25T00:30:54.631231+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:50:27.415077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,9,11],"acting":[6,3,9,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"89'15","reported_seq":38,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:31:13.976458+0000","last_change":"2024-04-25T00:30:47.522547+0000","last_active":"2024-04-25T00:31:13.976458+0000","last_peered":"2024-04-25T00:31:13.976458+0000","last_clean":"2024-04-25T00:31:13.976458+0000","last_became_active":"2024-04-25T00:30:47.522193+0000","last_became_peered":"2024-04-25T00:30:47.522193+0000","last_unstale":"2024-04-25T00:31:13.976458+0000","last_undegraded":"2024-04-25T00:31:13.976458+0000","last_fullsized":"2024-04-25T00:31:13.976458+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:10:10.417552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,2],"acting":[8,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631325+0000","last_change":"2024-04-25T00:30:48.534363+0000","last_active":"2024-04-25T00:30:54.631325+0000","last_peered":"2024-04-25T00:30:54.631325+0000","last_clean":"2024-04-25T00:30:54.631325+0000","last_became_active":"2024-04-25T00:30:48.533855+0000","last_became_peered":"2024-04-25T00:30:48.533855+0000","last_unstale":"2024-04-25T00:30:54.631325+0000","last_undegraded":"2024-04-25T00:30:54.631325+0000","last_fullsized":"2024-04-25T00:30:54.631325+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:28:29.374501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631290+0000","last_change":"2024-04-25T00:30:52.790323+0000","last_active":"2024-04-25T00:30:54.631290+0000","last_peered":"2024-04-25T00:30:54.631290+0000","last_clean":"2024-04-25T00:30:54.631290+0000","last_became_active":"2024-04-25T00:30:52.789905+0000","last_became_peered":"2024-04-25T00:30:52.789905+0000","last_unstale":"2024-04-25T00:30:54.631290+0000","last_undegraded":"2024-04-25T00:30:54.631290+0000","last_fullsized":"2024-04-25T00:30:54.631290+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:19:03.687318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3,1],"acting":[5,0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"84'1","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631456+0000","last_change":"2024-04-25T00:30:47.516468+0000","last_active":"2024-04-25T00:30:54.631456+0000","last_peered":"2024-04-25T00:30:54.631456+0000","last_clean":"2024-04-25T00:30:54.631456+0000","last_became_active":"2024-04-25T00:30:47.515672+0000","last_became_peered":"2024-04-25T00:30:47.515672+0000","last_unstale":"2024-04-25T00:30:54.631456+0000","last_undegraded":"2024-04-25T00:30:54.631456+0000","last_fullsized":"2024-04-25T00:30:54.631456+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:43:38.525018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.232539+0000","last_change":"2024-04-25T00:30:48.539679+0000","last_active":"2024-04-25T00:30:54.232539+0000","last_peered":"2024-04-25T00:30:54.232539+0000","last_clean":"2024-04-25T00:30:54.232539+0000","last_became_active":"2024-04-25T00:30:48.538065+0000","last_became_peered":"2024-04-25T00:30:48.538065+0000","last_unstale":"2024-04-25T00:30:54.232539+0000","last_undegraded":"2024-04-25T00:30:54.232539+0000","last_fullsized":"2024-04-25T00:30:54.232539+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:05:16.435010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.630856+0000","last_change":"2024-04-25T00:30:52.804824+0000","last_active":"2024-04-25T00:30:54.630856+0000","last_peered":"2024-04-25T00:30:54.630856+0000","last_clean":"2024-04-25T00:30:54.630856+0000","last_became_active":"2024-04-25T00:30:52.804322+0000","last_became_peered":"2024-04-25T00:30:52.804322+0000","last_unstale":"2024-04-25T00:30:54.630856+0000","last_undegraded":"2024-04-25T00:30:54.630856+0000","last_fullsized":"2024-04-25T00:30:54.630856+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:38:08.385170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,10,11,9],"acting":[8,10,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.766709+0000","last_change":"2024-04-25T00:30:47.516916+0000","last_active":"2024-04-25T00:30:53.766709+0000","last_peered":"2024-04-25T00:30:53.766709+0000","last_clean":"2024-04-25T00:30:53.766709+0000","last_became_active":"2024-04-25T00:30:47.516520+0000","last_became_peered":"2024-04-25T00:30:47.516520+0000","last_unstale":"2024-04-25T00:30:53.766709+0000","last_undegraded":"2024-04-25T00:30:53.766709+0000","last_fullsized":"2024-04-25T00:30:53.766709+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:29:19.099764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.769568+0000","last_change":"2024-04-25T00:30:48.537584+0000","last_active":"2024-04-25T00:30:53.769568+0000","last_peered":"2024-04-25T00:30:53.769568+0000","last_clean":"2024-04-25T00:30:53.769568+0000","last_became_active":"2024-04-25T00:30:48.537193+0000","last_became_peered":"2024-04-25T00:30:48.537193+0000","last_unstale":"2024-04-25T00:30:53.769568+0000","last_undegraded":"2024-04-25T00:30:53.769568+0000","last_fullsized":"2024-04-25T00:30:53.769568+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:15:22.075323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.632203+0000","last_change":"2024-04-25T00:30:52.784802+0000","last_active":"2024-04-25T00:30:54.632203+0000","last_peered":"2024-04-25T00:30:54.632203+0000","last_clean":"2024-04-25T00:30:54.632203+0000","last_became_active":"2024-04-25T00:30:52.784359+0000","last_became_peered":"2024-04-25T00:30:52.784359+0000","last_unstale":"2024-04-25T00:30:54.632203+0000","last_undegraded":"2024-04-25T00:30:54.632203+0000","last_fullsized":"2024-04-25T00:30:54.632203+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:19:57.723542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1,8],"acting":[6,0,1,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.130525+0000","last_change":"2024-04-25T00:30:47.525494+0000","last_active":"2024-04-25T00:30:54.130525+0000","last_peered":"2024-04-25T00:30:54.130525+0000","last_clean":"2024-04-25T00:30:54.130525+0000","last_became_active":"2024-04-25T00:30:47.525158+0000","last_became_peered":"2024-04-25T00:30:47.525158+0000","last_unstale":"2024-04-25T00:30:54.130525+0000","last_undegraded":"2024-04-25T00:30:54.130525+0000","last_fullsized":"2024-04-25T00:30:54.130525+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:51:18.543352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.769260+0000","last_change":"2024-04-25T00:30:48.540926+0000","last_active":"2024-04-25T00:30:53.769260+0000","last_peered":"2024-04-25T00:30:53.769260+0000","last_clean":"2024-04-25T00:30:53.769260+0000","last_became_active":"2024-04-25T00:30:48.540204+0000","last_became_peered":"2024-04-25T00:30:48.540204+0000","last_unstale":"2024-04-25T00:30:53.769260+0000","last_undegraded":"2024-04-25T00:30:53.769260+0000","last_fullsized":"2024-04-25T00:30:53.769260+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:02:45.530266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631372+0000","last_change":"2024-04-25T00:30:52.805780+0000","last_active":"2024-04-25T00:30:54.631372+0000","last_peered":"2024-04-25T00:30:54.631372+0000","last_clean":"2024-04-25T00:30:54.631372+0000","last_became_active":"2024-04-25T00:30:52.805222+0000","last_became_peered":"2024-04-25T00:30:52.805222+0000","last_unstale":"2024-04-25T00:30:54.631372+0000","last_undegraded":"2024-04-25T00:30:54.631372+0000","last_fullsized":"2024-04-25T00:30:54.631372+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:32:17.034630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,10,0],"acting":[6,5,10,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631237+0000","last_change":"2024-04-25T00:30:48.534335+0000","last_active":"2024-04-25T00:30:54.631237+0000","last_peered":"2024-04-25T00:30:54.631237+0000","last_clean":"2024-04-25T00:30:54.631237+0000","last_became_active":"2024-04-25T00:30:48.532670+0000","last_became_peered":"2024-04-25T00:30:48.532670+0000","last_unstale":"2024-04-25T00:30:54.631237+0000","last_undegraded":"2024-04-25T00:30:54.631237+0000","last_fullsized":"2024-04-25T00:30:54.631237+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:26:23.145918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"89'1","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631506+0000","last_change":"2024-04-25T00:30:47.527695+0000","last_active":"2024-04-25T00:30:54.631506+0000","last_peered":"2024-04-25T00:30:54.631506+0000","last_clean":"2024-04-25T00:30:54.631506+0000","last_became_active":"2024-04-25T00:30:47.527239+0000","last_became_peered":"2024-04-25T00:30:47.527239+0000","last_unstale":"2024-04-25T00:30:54.631506+0000","last_undegraded":"2024-04-25T00:30:54.631506+0000","last_fullsized":"2024-04-25T00:30:54.631506+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:54:30.911284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.130333+0000","last_change":"2024-04-25T00:30:52.781091+0000","last_active":"2024-04-25T00:30:54.130333+0000","last_peered":"2024-04-25T00:30:54.130333+0000","last_clean":"2024-04-25T00:30:54.130333+0000","last_became_active":"2024-04-25T00:30:52.780538+0000","last_became_peered":"2024-04-25T00:30:52.780538+0000","last_unstale":"2024-04-25T00:30:54.130333+0000","last_undegraded":"2024-04-25T00:30:54.130333+0000","last_fullsized":"2024-04-25T00:30:54.130333+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:49:18.141802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,11,9],"acting":[7,5,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631440+0000","last_change":"2024-04-25T00:30:48.538730+0000","last_active":"2024-04-25T00:30:54.631440+0000","last_peered":"2024-04-25T00:30:54.631440+0000","last_clean":"2024-04-25T00:30:54.631440+0000","last_became_active":"2024-04-25T00:30:48.538336+0000","last_became_peered":"2024-04-25T00:30:48.538336+0000","last_unstale":"2024-04-25T00:30:54.631440+0000","last_undegraded":"2024-04-25T00:30:54.631440+0000","last_fullsized":"2024-04-25T00:30:54.631440+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:44:22.370448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631739+0000","last_change":"2024-04-25T00:30:47.520450+0000","last_active":"2024-04-25T00:30:54.631739+0000","last_peered":"2024-04-25T00:30:54.631739+0000","last_clean":"2024-04-25T00:30:54.631739+0000","last_became_active":"2024-04-25T00:30:47.520073+0000","last_became_peered":"2024-04-25T00:30:47.520073+0000","last_unstale":"2024-04-25T00:30:54.631739+0000","last_undegraded":"2024-04-25T00:30:54.631739+0000","last_fullsized":"2024-04-25T00:30:54.631739+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:49:53.570421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"22'137","reported_seq":356,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.130117+0000","last_change":"2024-04-25T00:29:34.720552+0000","last_active":"2024-04-25T00:30:54.130117+0000","last_peered":"2024-04-25T00:30:54.130117+0000","last_clean":"2024-04-25T00:30:54.130117+0000","last_became_active":"2024-04-25T00:29:34.720041+0000","last_became_peered":"2024-04-25T00:29:34.720041+0000","last_unstale":"2024-04-25T00:30:54.130117+0000","last_undegraded":"2024-04-25T00:30:54.130117+0000","last_fullsized":"2024-04-25T00:30:54.130117+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:26:07.691582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:26:07.691582+0000","last_clean_scrub_stamp":"2024-04-25T00:26:07.691582+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:15:27.995162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631112+0000","last_change":"2024-04-25T00:30:52.805489+0000","last_active":"2024-04-25T00:30:54.631112+0000","last_peered":"2024-04-25T00:30:54.631112+0000","last_clean":"2024-04-25T00:30:54.631112+0000","last_became_active":"2024-04-25T00:30:52.804758+0000","last_became_peered":"2024-04-25T00:30:52.804758+0000","last_unstale":"2024-04-25T00:30:54.631112+0000","last_undegraded":"2024-04-25T00:30:54.631112+0000","last_fullsized":"2024-04-25T00:30:54.631112+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:17:31.325548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,6,10],"acting":[5,9,6,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631348+0000","last_change":"2024-04-25T00:30:48.537699+0000","last_active":"2024-04-25T00:30:54.631348+0000","last_peered":"2024-04-25T00:30:54.631348+0000","last_clean":"2024-04-25T00:30:54.631348+0000","last_became_active":"2024-04-25T00:30:48.537291+0000","last_became_peered":"2024-04-25T00:30:48.537291+0000","last_unstale":"2024-04-25T00:30:54.631348+0000","last_undegraded":"2024-04-25T00:30:54.631348+0000","last_fullsized":"2024-04-25T00:30:54.631348+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:25:36.816499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"89'1","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.130635+0000","last_change":"2024-04-25T00:30:47.524694+0000","last_active":"2024-04-25T00:30:54.130635+0000","last_peered":"2024-04-25T00:30:54.130635+0000","last_clean":"2024-04-25T00:30:54.130635+0000","last_became_active":"2024-04-25T00:30:47.524359+0000","last_became_peered":"2024-04-25T00:30:47.524359+0000","last_unstale":"2024-04-25T00:30:54.130635+0000","last_undegraded":"2024-04-25T00:30:54.130635+0000","last_fullsized":"2024-04-25T00:30:54.130635+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:55:16.677234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.232415+0000","last_change":"2024-04-25T00:30:52.780364+0000","last_active":"2024-04-25T00:30:54.232415+0000","last_peered":"2024-04-25T00:30:54.232415+0000","last_clean":"2024-04-25T00:30:54.232415+0000","last_became_active":"2024-04-25T00:30:52.779904+0000","last_became_peered":"2024-04-25T00:30:52.779904+0000","last_unstale":"2024-04-25T00:30:54.232415+0000","last_undegraded":"2024-04-25T00:30:54.232415+0000","last_fullsized":"2024-04-25T00:30:54.232415+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:11:26.738526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,7,6],"acting":[11,5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.5","version":"89'1","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631522+0000","last_change":"2024-04-25T00:30:47.525205+0000","last_active":"2024-04-25T00:30:54.631522+0000","last_peered":"2024-04-25T00:30:54.631522+0000","last_clean":"2024-04-25T00:30:54.631522+0000","last_became_active":"2024-04-25T00:30:47.517729+0000","last_became_peered":"2024-04-25T00:30:47.517729+0000","last_unstale":"2024-04-25T00:30:54.631522+0000","last_undegraded":"2024-04-25T00:30:54.631522+0000","last_fullsized":"2024-04-25T00:30:54.631522+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:21:14.810119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.632230+0000","last_change":"2024-04-25T00:30:48.533822+0000","last_active":"2024-04-25T00:30:54.632230+0000","last_peered":"2024-04-25T00:30:54.632230+0000","last_clean":"2024-04-25T00:30:54.632230+0000","last_became_active":"2024-04-25T00:30:48.533479+0000","last_became_peered":"2024-04-25T00:30:48.533479+0000","last_unstale":"2024-04-25T00:30:54.632230+0000","last_undegraded":"2024-04-25T00:30:54.632230+0000","last_fullsized":"2024-04-25T00:30:54.632230+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:49:01.028124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.630925+0000","last_change":"2024-04-25T00:30:52.804770+0000","last_active":"2024-04-25T00:30:54.630925+0000","last_peered":"2024-04-25T00:30:54.630925+0000","last_clean":"2024-04-25T00:30:54.630925+0000","last_became_active":"2024-04-25T00:30:52.804242+0000","last_became_peered":"2024-04-25T00:30:52.804242+0000","last_unstale":"2024-04-25T00:30:54.630925+0000","last_undegraded":"2024-04-25T00:30:54.630925+0000","last_fullsized":"2024-04-25T00:30:54.630925+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:34:11.761478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,10,0,7],"acting":[3,10,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"89'4","reported_seq":27,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:31:13.975840+0000","last_change":"2024-04-25T00:30:47.521364+0000","last_active":"2024-04-25T00:31:13.975840+0000","last_peered":"2024-04-25T00:31:13.975840+0000","last_clean":"2024-04-25T00:31:13.975840+0000","last_became_active":"2024-04-25T00:30:47.521032+0000","last_became_peered":"2024-04-25T00:30:47.521032+0000","last_unstale":"2024-04-25T00:31:13.975840+0000","last_undegraded":"2024-04-25T00:31:13.975840+0000","last_fullsized":"2024-04-25T00:31:13.975840+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:33:26.156568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,8,3],"acting":[9,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631867+0000","last_change":"2024-04-25T00:30:48.532942+0000","last_active":"2024-04-25T00:30:54.631867+0000","last_peered":"2024-04-25T00:30:54.631867+0000","last_clean":"2024-04-25T00:30:54.631867+0000","last_became_active":"2024-04-25T00:30:48.532580+0000","last_became_peered":"2024-04-25T00:30:48.532580+0000","last_unstale":"2024-04-25T00:30:54.631867+0000","last_undegraded":"2024-04-25T00:30:54.631867+0000","last_fullsized":"2024-04-25T00:30:54.631867+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:42:22.887920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.766806+0000","last_change":"2024-04-25T00:30:52.779757+0000","last_active":"2024-04-25T00:30:53.766806+0000","last_peered":"2024-04-25T00:30:53.766806+0000","last_clean":"2024-04-25T00:30:53.766806+0000","last_became_active":"2024-04-25T00:30:52.779324+0000","last_became_peered":"2024-04-25T00:30:52.779324+0000","last_unstale":"2024-04-25T00:30:53.766806+0000","last_undegraded":"2024-04-25T00:30:53.766806+0000","last_fullsized":"2024-04-25T00:30:53.766806+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:22:39.062000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4,7],"acting":[2,1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"84'2","reported_seq":25,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.632132+0000","last_change":"2024-04-25T00:30:47.518212+0000","last_active":"2024-04-25T00:30:54.632132+0000","last_peered":"2024-04-25T00:30:54.632132+0000","last_clean":"2024-04-25T00:30:54.632132+0000","last_became_active":"2024-04-25T00:30:47.517610+0000","last_became_peered":"2024-04-25T00:30:47.517610+0000","last_unstale":"2024-04-25T00:30:54.632132+0000","last_undegraded":"2024-04-25T00:30:54.632132+0000","last_fullsized":"2024-04-25T00:30:54.632132+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:13:58.904020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.677796+0000","last_change":"2024-04-25T00:30:48.539808+0000","last_active":"2024-04-25T00:30:54.677796+0000","last_peered":"2024-04-25T00:30:54.677796+0000","last_clean":"2024-04-25T00:30:54.677796+0000","last_became_active":"2024-04-25T00:30:48.539168+0000","last_became_peered":"2024-04-25T00:30:48.539168+0000","last_unstale":"2024-04-25T00:30:54.677796+0000","last_undegraded":"2024-04-25T00:30:54.677796+0000","last_fullsized":"2024-04-25T00:30:54.677796+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:33:07.445896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.632161+0000","last_change":"2024-04-25T00:30:52.804915+0000","last_active":"2024-04-25T00:30:54.632161+0000","last_peered":"2024-04-25T00:30:54.632161+0000","last_clean":"2024-04-25T00:30:54.632161+0000","last_became_active":"2024-04-25T00:30:52.804491+0000","last_became_peered":"2024-04-25T00:30:52.804491+0000","last_unstale":"2024-04-25T00:30:54.632161+0000","last_undegraded":"2024-04-25T00:30:54.632161+0000","last_fullsized":"2024-04-25T00:30:54.632161+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:51:50.219556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,10,9],"acting":[4,5,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.630747+0000","last_change":"2024-04-25T00:30:47.521909+0000","last_active":"2024-04-25T00:30:54.630747+0000","last_peered":"2024-04-25T00:30:54.630747+0000","last_clean":"2024-04-25T00:30:54.630747+0000","last_became_active":"2024-04-25T00:30:47.521551+0000","last_became_peered":"2024-04-25T00:30:47.521551+0000","last_unstale":"2024-04-25T00:30:54.630747+0000","last_undegraded":"2024-04-25T00:30:54.630747+0000","last_fullsized":"2024-04-25T00:30:54.630747+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:51:16.368609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.630574+0000","last_change":"2024-04-25T00:30:48.530514+0000","last_active":"2024-04-25T00:30:54.630574+0000","last_peered":"2024-04-25T00:30:54.630574+0000","last_clean":"2024-04-25T00:30:54.630574+0000","last_became_active":"2024-04-25T00:30:48.530129+0000","last_became_peered":"2024-04-25T00:30:48.530129+0000","last_unstale":"2024-04-25T00:30:54.630574+0000","last_undegraded":"2024-04-25T00:30:54.630574+0000","last_fullsized":"2024-04-25T00:30:54.630574+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:51:25.623186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.767244+0000","last_change":"2024-04-25T00:30:52.785767+0000","last_active":"2024-04-25T00:30:53.767244+0000","last_peered":"2024-04-25T00:30:53.767244+0000","last_clean":"2024-04-25T00:30:53.767244+0000","last_became_active":"2024-04-25T00:30:52.785301+0000","last_became_peered":"2024-04-25T00:30:52.785301+0000","last_unstale":"2024-04-25T00:30:53.767244+0000","last_undegraded":"2024-04-25T00:30:53.767244+0000","last_fullsized":"2024-04-25T00:30:53.767244+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:10:14.071733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,2,5],"acting":[9,3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631562+0000","last_change":"2024-04-25T00:30:47.526364+0000","last_active":"2024-04-25T00:30:54.631562+0000","last_peered":"2024-04-25T00:30:54.631562+0000","last_clean":"2024-04-25T00:30:54.631562+0000","last_became_active":"2024-04-25T00:30:47.526009+0000","last_became_peered":"2024-04-25T00:30:47.526009+0000","last_unstale":"2024-04-25T00:30:54.631562+0000","last_undegraded":"2024-04-25T00:30:54.631562+0000","last_fullsized":"2024-04-25T00:30:54.631562+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:17:38.471878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.232186+0000","last_change":"2024-04-25T00:30:48.532283+0000","last_active":"2024-04-25T00:30:54.232186+0000","last_peered":"2024-04-25T00:30:54.232186+0000","last_clean":"2024-04-25T00:30:54.232186+0000","last_became_active":"2024-04-25T00:30:48.531935+0000","last_became_peered":"2024-04-25T00:30:48.531935+0000","last_unstale":"2024-04-25T00:30:54.232186+0000","last_undegraded":"2024-04-25T00:30:54.232186+0000","last_fullsized":"2024-04-25T00:30:54.232186+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:01:15.965611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631657+0000","last_change":"2024-04-25T00:30:52.786529+0000","last_active":"2024-04-25T00:30:54.631657+0000","last_peered":"2024-04-25T00:30:54.631657+0000","last_clean":"2024-04-25T00:30:54.631657+0000","last_became_active":"2024-04-25T00:30:52.785993+0000","last_became_peered":"2024-04-25T00:30:52.785993+0000","last_unstale":"2024-04-25T00:30:54.631657+0000","last_undegraded":"2024-04-25T00:30:54.631657+0000","last_fullsized":"2024-04-25T00:30:54.631657+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:59:24.360756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,9,7,8],"acting":[4,9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.766939+0000","last_change":"2024-04-25T00:30:47.520608+0000","last_active":"2024-04-25T00:30:53.766939+0000","last_peered":"2024-04-25T00:30:53.766939+0000","last_clean":"2024-04-25T00:30:53.766939+0000","last_became_active":"2024-04-25T00:30:47.520063+0000","last_became_peered":"2024-04-25T00:30:47.520063+0000","last_unstale":"2024-04-25T00:30:53.766939+0000","last_undegraded":"2024-04-25T00:30:53.766939+0000","last_fullsized":"2024-04-25T00:30:53.766939+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:58:47.823712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.632035+0000","last_change":"2024-04-25T00:30:48.535073+0000","last_active":"2024-04-25T00:30:54.632035+0000","last_peered":"2024-04-25T00:30:54.632035+0000","last_clean":"2024-04-25T00:30:54.632035+0000","last_became_active":"2024-04-25T00:30:48.534247+0000","last_became_peered":"2024-04-25T00:30:48.534247+0000","last_unstale":"2024-04-25T00:30:54.632035+0000","last_undegraded":"2024-04-25T00:30:54.632035+0000","last_fullsized":"2024-04-25T00:30:54.632035+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:46:18.613705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631695+0000","last_change":"2024-04-25T00:30:52.804519+0000","last_active":"2024-04-25T00:30:54.631695+0000","last_peered":"2024-04-25T00:30:54.631695+0000","last_clean":"2024-04-25T00:30:54.631695+0000","last_became_active":"2024-04-25T00:30:52.804074+0000","last_became_peered":"2024-04-25T00:30:52.804074+0000","last_unstale":"2024-04-25T00:30:54.631695+0000","last_undegraded":"2024-04-25T00:30:54.631695+0000","last_fullsized":"2024-04-25T00:30:54.631695+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:48:45.058873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,10,3],"acting":[4,2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.b","version":"89'4","reported_seq":27,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:31:12.960859+0000","last_change":"2024-04-25T00:30:47.521054+0000","last_active":"2024-04-25T00:31:12.960859+0000","last_peered":"2024-04-25T00:31:12.960859+0000","last_clean":"2024-04-25T00:31:12.960859+0000","last_became_active":"2024-04-25T00:30:47.520716+0000","last_became_peered":"2024-04-25T00:30:47.520716+0000","last_unstale":"2024-04-25T00:31:12.960859+0000","last_undegraded":"2024-04-25T00:31:12.960859+0000","last_fullsized":"2024-04-25T00:31:12.960859+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:05:16.471589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.130691+0000","last_change":"2024-04-25T00:30:48.537152+0000","last_active":"2024-04-25T00:30:54.130691+0000","last_peered":"2024-04-25T00:30:54.130691+0000","last_clean":"2024-04-25T00:30:54.130691+0000","last_became_active":"2024-04-25T00:30:48.536256+0000","last_became_peered":"2024-04-25T00:30:48.536256+0000","last_unstale":"2024-04-25T00:30:54.130691+0000","last_undegraded":"2024-04-25T00:30:54.130691+0000","last_fullsized":"2024-04-25T00:30:54.130691+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:57:53.870212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.630951+0000","last_change":"2024-04-25T00:30:52.781022+0000","last_active":"2024-04-25T00:30:54.630951+0000","last_peered":"2024-04-25T00:30:54.630951+0000","last_clean":"2024-04-25T00:30:54.630951+0000","last_became_active":"2024-04-25T00:30:52.780595+0000","last_became_peered":"2024-04-25T00:30:52.780595+0000","last_unstale":"2024-04-25T00:30:54.630951+0000","last_undegraded":"2024-04-25T00:30:54.630951+0000","last_fullsized":"2024-04-25T00:30:54.630951+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:27:02.580870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6,7],"acting":[1,5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.767244+0000","last_change":"2024-04-25T00:30:47.519118+0000","last_active":"2024-04-25T00:30:53.767244+0000","last_peered":"2024-04-25T00:30:53.767244+0000","last_clean":"2024-04-25T00:30:53.767244+0000","last_became_active":"2024-04-25T00:30:47.518639+0000","last_became_peered":"2024-04-25T00:30:47.518639+0000","last_unstale":"2024-04-25T00:30:53.767244+0000","last_undegraded":"2024-04-25T00:30:53.767244+0000","last_fullsized":"2024-04-25T00:30:53.767244+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:32:29.747903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.130732+0000","last_change":"2024-04-25T00:30:48.542073+0000","last_active":"2024-04-25T00:30:54.130732+0000","last_peered":"2024-04-25T00:30:54.130732+0000","last_clean":"2024-04-25T00:30:54.130732+0000","last_became_active":"2024-04-25T00:30:48.541722+0000","last_became_peered":"2024-04-25T00:30:48.541722+0000","last_unstale":"2024-04-25T00:30:54.130732+0000","last_undegraded":"2024-04-25T00:30:54.130732+0000","last_fullsized":"2024-04-25T00:30:54.130732+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:34:03.397539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.677591+0000","last_change":"2024-04-25T00:30:52.790234+0000","last_active":"2024-04-25T00:30:54.677591+0000","last_peered":"2024-04-25T00:30:54.677591+0000","last_clean":"2024-04-25T00:30:54.677591+0000","last_became_active":"2024-04-25T00:30:52.789788+0000","last_became_peered":"2024-04-25T00:30:52.789788+0000","last_unstale":"2024-04-25T00:30:54.677591+0000","last_undegraded":"2024-04-25T00:30:54.677591+0000","last_fullsized":"2024-04-25T00:30:54.677591+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:44:43.399771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,9,1,4],"acting":[0,9,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.d","version":"84'1","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631477+0000","last_change":"2024-04-25T00:30:47.523597+0000","last_active":"2024-04-25T00:30:54.631477+0000","last_peered":"2024-04-25T00:30:54.631477+0000","last_clean":"2024-04-25T00:30:54.631477+0000","last_became_active":"2024-04-25T00:30:47.523225+0000","last_became_peered":"2024-04-25T00:30:47.523225+0000","last_unstale":"2024-04-25T00:30:54.631477+0000","last_undegraded":"2024-04-25T00:30:54.631477+0000","last_fullsized":"2024-04-25T00:30:54.631477+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:51:12.341015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.130412+0000","last_change":"2024-04-25T00:30:48.541417+0000","last_active":"2024-04-25T00:30:54.130412+0000","last_peered":"2024-04-25T00:30:54.130412+0000","last_clean":"2024-04-25T00:30:54.130412+0000","last_became_active":"2024-04-25T00:30:48.541015+0000","last_became_peered":"2024-04-25T00:30:48.541015+0000","last_unstale":"2024-04-25T00:30:54.130412+0000","last_undegraded":"2024-04-25T00:30:54.130412+0000","last_fullsized":"2024-04-25T00:30:54.130412+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:21:35.299823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631288+0000","last_change":"2024-04-25T00:30:52.782230+0000","last_active":"2024-04-25T00:30:54.631288+0000","last_peered":"2024-04-25T00:30:54.631288+0000","last_clean":"2024-04-25T00:30:54.631288+0000","last_became_active":"2024-04-25T00:30:52.781708+0000","last_became_peered":"2024-04-25T00:30:52.781708+0000","last_unstale":"2024-04-25T00:30:54.631288+0000","last_undegraded":"2024-04-25T00:30:54.631288+0000","last_fullsized":"2024-04-25T00:30:54.631288+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:58:53.260985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7,1],"acting":[5,2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.e","version":"84'2","reported_seq":25,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631160+0000","last_change":"2024-04-25T00:30:47.522400+0000","last_active":"2024-04-25T00:30:54.631160+0000","last_peered":"2024-04-25T00:30:54.631160+0000","last_clean":"2024-04-25T00:30:54.631160+0000","last_became_active":"2024-04-25T00:30:47.522032+0000","last_became_peered":"2024-04-25T00:30:47.522032+0000","last_unstale":"2024-04-25T00:30:54.631160+0000","last_undegraded":"2024-04-25T00:30:54.631160+0000","last_fullsized":"2024-04-25T00:30:54.631160+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:24:17.749437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.130776+0000","last_change":"2024-04-25T00:30:48.532730+0000","last_active":"2024-04-25T00:30:54.130776+0000","last_peered":"2024-04-25T00:30:54.130776+0000","last_clean":"2024-04-25T00:30:54.130776+0000","last_became_active":"2024-04-25T00:30:48.532129+0000","last_became_peered":"2024-04-25T00:30:48.532129+0000","last_unstale":"2024-04-25T00:30:54.130776+0000","last_undegraded":"2024-04-25T00:30:54.130776+0000","last_fullsized":"2024-04-25T00:30:54.130776+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:36:30.115125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.232228+0000","last_change":"2024-04-25T00:30:48.535827+0000","last_active":"2024-04-25T00:30:54.232228+0000","last_peered":"2024-04-25T00:30:54.232228+0000","last_clean":"2024-04-25T00:30:54.232228+0000","last_became_active":"2024-04-25T00:30:48.535484+0000","last_became_peered":"2024-04-25T00:30:48.535484+0000","last_unstale":"2024-04-25T00:30:54.232228+0000","last_undegraded":"2024-04-25T00:30:54.232228+0000","last_fullsized":"2024-04-25T00:30:54.232228+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:11:44.596594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.766987+0000","last_change":"2024-04-25T00:30:48.540676+0000","last_active":"2024-04-25T00:30:53.766987+0000","last_peered":"2024-04-25T00:30:53.766987+0000","last_clean":"2024-04-25T00:30:53.766987+0000","last_became_active":"2024-04-25T00:30:48.540258+0000","last_became_peered":"2024-04-25T00:30:48.540258+0000","last_unstale":"2024-04-25T00:30:53.766987+0000","last_undegraded":"2024-04-25T00:30:53.766987+0000","last_fullsized":"2024-04-25T00:30:53.766987+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:10:29.930897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.766838+0000","last_change":"2024-04-25T00:30:48.542191+0000","last_active":"2024-04-25T00:30:53.766838+0000","last_peered":"2024-04-25T00:30:53.766838+0000","last_clean":"2024-04-25T00:30:53.766838+0000","last_became_active":"2024-04-25T00:30:48.540433+0000","last_became_peered":"2024-04-25T00:30:48.540433+0000","last_unstale":"2024-04-25T00:30:53.766838+0000","last_undegraded":"2024-04-25T00:30:53.766838+0000","last_fullsized":"2024-04-25T00:30:53.766838+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:21:55.694555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.677761+0000","last_change":"2024-04-25T00:30:48.539797+0000","last_active":"2024-04-25T00:30:54.677761+0000","last_peered":"2024-04-25T00:30:54.677761+0000","last_clean":"2024-04-25T00:30:54.677761+0000","last_became_active":"2024-04-25T00:30:48.539423+0000","last_became_peered":"2024-04-25T00:30:48.539423+0000","last_unstale":"2024-04-25T00:30:54.677761+0000","last_undegraded":"2024-04-25T00:30:54.677761+0000","last_fullsized":"2024-04-25T00:30:54.677761+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:12:54.427579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.632335+0000","last_change":"2024-04-25T00:30:48.527892+0000","last_active":"2024-04-25T00:30:54.632335+0000","last_peered":"2024-04-25T00:30:54.632335+0000","last_clean":"2024-04-25T00:30:54.632335+0000","last_became_active":"2024-04-25T00:30:48.527522+0000","last_became_peered":"2024-04-25T00:30:48.527522+0000","last_unstale":"2024-04-25T00:30:54.632335+0000","last_undegraded":"2024-04-25T00:30:54.632335+0000","last_fullsized":"2024-04-25T00:30:54.632335+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:21:22.779470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.232569+0000","last_change":"2024-04-25T00:30:48.535066+0000","last_active":"2024-04-25T00:30:54.232569+0000","last_peered":"2024-04-25T00:30:54.232569+0000","last_clean":"2024-04-25T00:30:54.232569+0000","last_became_active":"2024-04-25T00:30:48.534710+0000","last_became_peered":"2024-04-25T00:30:48.534710+0000","last_unstale":"2024-04-25T00:30:54.232569+0000","last_undegraded":"2024-04-25T00:30:54.232569+0000","last_fullsized":"2024-04-25T00:30:54.232569+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:57:18.582569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.767388+0000","last_change":"2024-04-25T00:30:48.541921+0000","last_active":"2024-04-25T00:30:53.767388+0000","last_peered":"2024-04-25T00:30:53.767388+0000","last_clean":"2024-04-25T00:30:53.767388+0000","last_became_active":"2024-04-25T00:30:48.541348+0000","last_became_peered":"2024-04-25T00:30:48.541348+0000","last_unstale":"2024-04-25T00:30:53.767388+0000","last_undegraded":"2024-04-25T00:30:53.767388+0000","last_fullsized":"2024-04-25T00:30:53.767388+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:07:00.001504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.632044+0000","last_change":"2024-04-25T00:30:48.536902+0000","last_active":"2024-04-25T00:30:54.632044+0000","last_peered":"2024-04-25T00:30:54.632044+0000","last_clean":"2024-04-25T00:30:54.632044+0000","last_became_active":"2024-04-25T00:30:48.536008+0000","last_became_peered":"2024-04-25T00:30:48.536008+0000","last_unstale":"2024-04-25T00:30:54.632044+0000","last_undegraded":"2024-04-25T00:30:54.632044+0000","last_fullsized":"2024-04-25T00:30:54.632044+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:19:15.050142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631104+0000","last_change":"2024-04-25T00:30:48.533104+0000","last_active":"2024-04-25T00:30:54.631104+0000","last_peered":"2024-04-25T00:30:54.631104+0000","last_clean":"2024-04-25T00:30:54.631104+0000","last_became_active":"2024-04-25T00:30:48.532124+0000","last_became_peered":"2024-04-25T00:30:48.532124+0000","last_unstale":"2024-04-25T00:30:54.631104+0000","last_undegraded":"2024-04-25T00:30:54.631104+0000","last_fullsized":"2024-04-25T00:30:54.631104+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:54:57.391433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631470+0000","last_change":"2024-04-25T00:30:48.542074+0000","last_active":"2024-04-25T00:30:54.631470+0000","last_peered":"2024-04-25T00:30:54.631470+0000","last_clean":"2024-04-25T00:30:54.631470+0000","last_became_active":"2024-04-25T00:30:48.538752+0000","last_became_peered":"2024-04-25T00:30:48.538752+0000","last_unstale":"2024-04-25T00:30:54.631470+0000","last_undegraded":"2024-04-25T00:30:54.631470+0000","last_fullsized":"2024-04-25T00:30:54.631470+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:55:30.613616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631263+0000","last_change":"2024-04-25T00:30:48.534158+0000","last_active":"2024-04-25T00:30:54.631263+0000","last_peered":"2024-04-25T00:30:54.631263+0000","last_clean":"2024-04-25T00:30:54.631263+0000","last_became_active":"2024-04-25T00:30:48.533821+0000","last_became_peered":"2024-04-25T00:30:48.533821+0000","last_unstale":"2024-04-25T00:30:54.631263+0000","last_undegraded":"2024-04-25T00:30:54.631263+0000","last_fullsized":"2024-04-25T00:30:54.631263+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:05:19.327608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631084+0000","last_change":"2024-04-25T00:30:48.531718+0000","last_active":"2024-04-25T00:30:54.631084+0000","last_peered":"2024-04-25T00:30:54.631084+0000","last_clean":"2024-04-25T00:30:54.631084+0000","last_became_active":"2024-04-25T00:30:48.531291+0000","last_became_peered":"2024-04-25T00:30:48.531291+0000","last_unstale":"2024-04-25T00:30:54.631084+0000","last_undegraded":"2024-04-25T00:30:54.631084+0000","last_fullsized":"2024-04-25T00:30:54.631084+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:56:49.032937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.232368+0000","last_change":"2024-04-25T00:30:48.537479+0000","last_active":"2024-04-25T00:30:54.232368+0000","last_peered":"2024-04-25T00:30:54.232368+0000","last_clean":"2024-04-25T00:30:54.232368+0000","last_became_active":"2024-04-25T00:30:48.537127+0000","last_became_peered":"2024-04-25T00:30:48.537127+0000","last_unstale":"2024-04-25T00:30:54.232368+0000","last_undegraded":"2024-04-25T00:30:54.232368+0000","last_fullsized":"2024-04-25T00:30:54.232368+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:26:37.289281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":8192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":12},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":4774,"num_objects":33,"num_object_clones":0,"num_object_copies":99,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":33,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":77,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":22,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":159744,"data_stored":16300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":67,"ondisk_log_size":67,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":80,"seq":343597383698,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32140,"kb_used_data":960,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713012,"statfs":{"total":95995035648,"available":95962124288,"internally_reserved":0,"allocated":983040,"data_stored":477215,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1596,"internal_metadata":31914436},"hb_peers":[0,1,2,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":10,"up_from":72,"seq":309237645335,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28580,"kb_used_data":1564,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716572,"statfs":{"total":95995035648,"available":95965769728,"internally_reserved":0,"allocated":1601536,"data_stored":1071619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1600,"internal_metadata":27654592},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":9,"up_from":67,"seq":287762808861,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36320,"kb_used_data":984,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708832,"statfs":{"total":95995035648,"available":95957843968,"internally_reserved":0,"allocated":1007616,"data_stored":482129,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1600,"internal_metadata":36174272},"hb_peers":[0,1,2,3,4,5,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":8,"up_from":59,"seq":253403070497,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32148,"kb_used_data":972,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713004,"statfs":{"total":95995035648,"available":95962116096,"internally_reserved":0,"allocated":995328,"data_stored":477841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":7,"up_from":51,"seq":219043332134,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32728,"kb_used_data":1544,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712424,"statfs":{"total":95995035648,"available":95961522176,"internally_reserved":0,"allocated":1581056,"data_stored":1068165,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495660,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36296,"kb_used_data":960,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708856,"statfs":{"total":95995035648,"available":95957868544,"internally_reserved":0,"allocated":983040,"data_stored":477215,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":36174278},"hb_peers":[0,1,2,3,4,5,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574915,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27988,"kb_used_data":972,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717164,"statfs":{"total":95995035648,"available":95966375936,"internally_reserved":0,"allocated":995328,"data_stored":481057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738442,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28004,"kb_used_data":988,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717148,"statfs":{"total":95995035648,"available":95966359552,"internally_reserved":0,"allocated":1011712,"data_stored":478693,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":18,"seq":77309411391,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27992,"kb_used_data":976,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717160,"statfs":{"total":95995035648,"available":95966371840,"internally_reserved":0,"allocated":999424,"data_stored":481091,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":25,"seq":107374182458,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36308,"kb_used_data":968,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708844,"statfs":{"total":95995035648,"available":95957856256,"internally_reserved":0,"allocated":991232,"data_stored":477319,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":36174279},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":33,"seq":141733920822,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27984,"kb_used_data":964,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717168,"statfs":{"total":95995035648,"available":95966380032,"internally_reserved":0,"allocated":987136,"data_stored":477237,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757297,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28576,"kb_used_data":1552,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716576,"statfs":{"total":95995035648,"available":95965773824,"internally_reserved":0,"allocated":1589248,"data_stored":1067765,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":3842,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":3876,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1204,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4036,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":8192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-25T00:31:17.782 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-04-25T00:31:17.837 INFO:journalctl@ceph.mds.b.smithi121.stdout:Apr 25 00:31:17 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-b[62582]: 2024-04-25T00:31:17.452+0000 7f6e5c388700 -1 mds.pinger is_rank_lagging: rank=2 was never sent ping request. 2024-04-25T00:31:17.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.919+0000 7fe1f71a9640 1 -- 172.21.15.32:0/3254106684 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe1f0102250 msgr2=0x7fe1f0108ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:17.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.919+0000 7fe1f71a9640 1 --2- 172.21.15.32:0/3254106684 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe1f0102250 0x7fe1f0108ae0 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7fe1d8009900 tx=0x7fe1d802a410 comp rx=0 tx=0).stop 2024-04-25T00:31:17.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.919+0000 7fe1f71a9640 1 -- 172.21.15.32:0/3254106684 shutdown_connections 2024-04-25T00:31:17.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.919+0000 7fe1f71a9640 1 --2- 172.21.15.32:0/3254106684 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe1f0102250 0x7fe1f0108ae0 unknown :-1 s=CLOSED pgs=284 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.919+0000 7fe1f71a9640 1 --2- 172.21.15.32:0/3254106684 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe1f0101890 0x7fe1f0101d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.919+0000 7fe1f71a9640 1 --2- 172.21.15.32:0/3254106684 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe1f01006a0 0x7fe1f0100ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.919+0000 7fe1f71a9640 1 -- 172.21.15.32:0/3254106684 >> 172.21.15.32:0/3254106684 conn(0x7fe1f00fbc30 msgr2=0x7fe1f00fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:17.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.919+0000 7fe1f71a9640 1 -- 172.21.15.32:0/3254106684 shutdown_connections 2024-04-25T00:31:17.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.919+0000 7fe1f71a9640 1 -- 172.21.15.32:0/3254106684 wait complete. 2024-04-25T00:31:17.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.919+0000 7fe1f71a9640 1 Processor -- start 2024-04-25T00:31:17.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.920+0000 7fe1f71a9640 1 -- start start 2024-04-25T00:31:17.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.920+0000 7fe1f71a9640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe1f01006a0 0x7fe1f0197ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:17.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.920+0000 7fe1f71a9640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe1f0101890 0x7fe1f0198020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:17.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.920+0000 7fe1f71a9640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe1f0102250 0x7fe1f019eff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:17.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.920+0000 7fe1f71a9640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fe1f010c300 con 0x7fe1f01006a0 2024-04-25T00:31:17.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.920+0000 7fe1f71a9640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fe1f010c180 con 0x7fe1f0101890 2024-04-25T00:31:17.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.920+0000 7fe1f71a9640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fe1f010c480 con 0x7fe1f0102250 2024-04-25T00:31:17.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.920+0000 7fe1f4f45640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe1f01006a0 0x7fe1f0197ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:17.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.920+0000 7fe1f4f45640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe1f01006a0 0x7fe1f0197ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:58032/0 (socket says 172.21.15.32:58032) 2024-04-25T00:31:17.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.920+0000 7fe1f4f45640 1 -- 172.21.15.32:0/186297096 learned_addr learned my addr 172.21.15.32:0/186297096 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:17.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.920+0000 7fe1effff640 1 --2- 172.21.15.32:0/186297096 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe1f0101890 0x7fe1f0198020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:17.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.920+0000 7fe1f5746640 1 --2- 172.21.15.32:0/186297096 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe1f0102250 0x7fe1f019eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:17.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.920+0000 7fe1f4f45640 1 -- 172.21.15.32:0/186297096 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe1f0102250 msgr2=0x7fe1f019eff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:17.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.920+0000 7fe1f4f45640 1 --2- 172.21.15.32:0/186297096 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe1f0102250 0x7fe1f019eff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.920+0000 7fe1f4f45640 1 -- 172.21.15.32:0/186297096 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe1f0101890 msgr2=0x7fe1f0198020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:17.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.920+0000 7fe1f4f45640 1 --2- 172.21.15.32:0/186297096 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe1f0101890 0x7fe1f0198020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:17.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.920+0000 7fe1f4f45640 1 -- 172.21.15.32:0/186297096 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe1f019f660 con 0x7fe1f01006a0 2024-04-25T00:31:17.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.920+0000 7fe1f4f45640 1 --2- 172.21.15.32:0/186297096 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe1f01006a0 0x7fe1f0197ae0 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7fe1e800d770 tx=0x7fe1e800dc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:17.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.920+0000 7fe1edffb640 1 -- 172.21.15.32:0/186297096 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe1e8017070 con 0x7fe1f01006a0 2024-04-25T00:31:17.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.920+0000 7fe1f71a9640 1 -- 172.21.15.32:0/186297096 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe1f019f950 con 0x7fe1f01006a0 2024-04-25T00:31:17.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.920+0000 7fe1edffb640 1 -- 172.21.15.32:0/186297096 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe1e8009c30 con 0x7fe1f01006a0 2024-04-25T00:31:17.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.920+0000 7fe1edffb640 1 -- 172.21.15.32:0/186297096 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe1e8003dd0 con 0x7fe1f01006a0 2024-04-25T00:31:17.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.921+0000 7fe1f71a9640 1 -- 172.21.15.32:0/186297096 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe1f019fef0 con 0x7fe1f01006a0 2024-04-25T00:31:17.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.922+0000 7fe1f71a9640 1 -- 172.21.15.32:0/186297096 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe1f00689b0 con 0x7fe1f01006a0 2024-04-25T00:31:17.928 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.923+0000 7fe1edffb640 1 -- 172.21.15.32:0/186297096 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7fe1e8003f70 con 0x7fe1f01006a0 2024-04-25T00:31:17.928 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.923+0000 7fe1edffb640 1 --2- 172.21.15.32:0/186297096 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe1dc0a2c20 0x7fe1dc0a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:17.928 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.923+0000 7fe1effff640 1 --2- 172.21.15.32:0/186297096 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe1dc0a2c20 0x7fe1dc0a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:17.928 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.923+0000 7fe1edffb640 1 -- 172.21.15.32:0/186297096 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7fe1e80cee20 con 0x7fe1f01006a0 2024-04-25T00:31:17.928 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.924+0000 7fe1effff640 1 --2- 172.21.15.32:0/186297096 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe1dc0a2c20 0x7fe1dc0a50e0 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7fe1f0198f50 tx=0x7fe1e0009500 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:17.928 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:17.928+0000 7fe1edffb640 1 -- 172.21.15.32:0/186297096 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fe1e809b9e0 con 0x7fe1f01006a0 2024-04-25T00:31:18.112 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:18.112+0000 7fe1f71a9640 1 -- 172.21.15.32:0/186297096 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7fe1f006a4b0 con 0x7fe1dc0a2c20 2024-04-25T00:31:18.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:18.121+0000 7fe1edffb640 1 -- 172.21.15.32:0/186297096 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+340269 (secure 0 0 0) 0x7fe1f006a4b0 con 0x7fe1dc0a2c20 2024-04-25T00:31:18.128 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:31:18.130 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-04-25T00:31:18.131 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:18.125+0000 7fe1f71a9640 1 -- 172.21.15.32:0/186297096 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe1dc0a2c20 msgr2=0x7fe1dc0a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:18.131 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:18.125+0000 7fe1f71a9640 1 --2- 172.21.15.32:0/186297096 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe1dc0a2c20 0x7fe1dc0a50e0 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7fe1f0198f50 tx=0x7fe1e0009500 comp rx=0 tx=0).stop 2024-04-25T00:31:18.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:18.125+0000 7fe1f71a9640 1 -- 172.21.15.32:0/186297096 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe1f01006a0 msgr2=0x7fe1f0197ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:18.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:18.125+0000 7fe1f71a9640 1 --2- 172.21.15.32:0/186297096 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe1f01006a0 0x7fe1f0197ae0 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7fe1e800d770 tx=0x7fe1e800dc30 comp rx=0 tx=0).stop 2024-04-25T00:31:18.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:18.125+0000 7fe1f71a9640 1 -- 172.21.15.32:0/186297096 shutdown_connections 2024-04-25T00:31:18.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:18.125+0000 7fe1f71a9640 1 --2- 172.21.15.32:0/186297096 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe1dc0a2c20 0x7fe1dc0a50e0 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:18.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:18.125+0000 7fe1f71a9640 1 --2- 172.21.15.32:0/186297096 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe1f0102250 0x7fe1f019eff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:18.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:18.125+0000 7fe1f71a9640 1 --2- 172.21.15.32:0/186297096 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe1f0101890 0x7fe1f0198020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:18.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:18.125+0000 7fe1f71a9640 1 --2- 172.21.15.32:0/186297096 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe1f01006a0 0x7fe1f0197ae0 unknown :-1 s=CLOSED pgs=285 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:18.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:18.125+0000 7fe1f71a9640 1 -- 172.21.15.32:0/186297096 >> 172.21.15.32:0/186297096 conn(0x7fe1f00fbc30 msgr2=0x7fe1f0069da0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:18.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:18.125+0000 7fe1f71a9640 1 -- 172.21.15.32:0/186297096 shutdown_connections 2024-04-25T00:31:18.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:18.125+0000 7fe1f71a9640 1 -- 172.21.15.32:0/186297096 wait complete. 2024-04-25T00:31:18.135 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:17 smithi032 ceph-mon[29357]: from='client.14943 -' entity='client.admin' cmd=[{"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:31:18.135 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:17 smithi032 ceph-mon[29357]: pgmap v306: 129 pgs: 129 active+clean; 581 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 85 B/s rd, 1023 B/s wr, 2 op/s 2024-04-25T00:31:18.135 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:17 smithi032 ceph-mon[29357]: mgrmap e17: x(active, since 7m), standbys: z, y 2024-04-25T00:31:18.136 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:17 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2555249369' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:31:18.136 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:17 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1358938102' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:31:18.136 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:17 smithi032 ceph-mon[29357]: from='client.14961 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:31:18.137 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":306,"stamp":"2024-04-25T00:31:16.866000+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595142,"num_objects":35,"num_object_clones":0,"num_object_copies":105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":35,"num_whiteouts":0,"num_read":137,"num_read_kb":120,"num_write":258,"num_write_kb":2967,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":22,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":204,"ondisk_log_size":204,"up":419,"acting":419,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":419,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":375064,"kb_used_data":13404,"kb_used_omap":18,"kb_used_meta":361517,"kb_avail":1124566760,"statfs":{"total":1151940427776,"available":1151556362240,"internally_reserved":0,"allocated":13725696,"data_stored":7517346,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19142,"internal_metadata":370193722},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":275,"num_objects":11,"num_object_clones":0,"num_object_copies":33,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":30,"num_write_kb":12,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":8,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003557"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.232735+0000","last_change":"2024-04-25T00:30:48.538719+0000","last_active":"2024-04-25T00:30:54.232735+0000","last_peered":"2024-04-25T00:30:54.232735+0000","last_clean":"2024-04-25T00:30:54.232735+0000","last_became_active":"2024-04-25T00:30:48.538356+0000","last_became_peered":"2024-04-25T00:30:48.538356+0000","last_unstale":"2024-04-25T00:30:54.232735+0000","last_undegraded":"2024-04-25T00:30:54.232735+0000","last_fullsized":"2024-04-25T00:30:54.232735+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:39:08.974838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.677681+0000","last_change":"2024-04-25T00:30:48.532262+0000","last_active":"2024-04-25T00:30:54.677681+0000","last_peered":"2024-04-25T00:30:54.677681+0000","last_clean":"2024-04-25T00:30:54.677681+0000","last_became_active":"2024-04-25T00:30:48.531926+0000","last_became_peered":"2024-04-25T00:30:48.531926+0000","last_unstale":"2024-04-25T00:30:54.677681+0000","last_undegraded":"2024-04-25T00:30:54.677681+0000","last_fullsized":"2024-04-25T00:30:54.677681+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:12:06.527649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.677572+0000","last_change":"2024-04-25T00:30:48.536383+0000","last_active":"2024-04-25T00:30:54.677572+0000","last_peered":"2024-04-25T00:30:54.677572+0000","last_clean":"2024-04-25T00:30:54.677572+0000","last_became_active":"2024-04-25T00:30:48.536048+0000","last_became_peered":"2024-04-25T00:30:48.536048+0000","last_unstale":"2024-04-25T00:30:54.677572+0000","last_undegraded":"2024-04-25T00:30:54.677572+0000","last_fullsized":"2024-04-25T00:30:54.677572+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:32:26.728441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.632025+0000","last_change":"2024-04-25T00:30:48.532195+0000","last_active":"2024-04-25T00:30:54.632025+0000","last_peered":"2024-04-25T00:30:54.632025+0000","last_clean":"2024-04-25T00:30:54.632025+0000","last_became_active":"2024-04-25T00:30:48.531851+0000","last_became_peered":"2024-04-25T00:30:48.531851+0000","last_unstale":"2024-04-25T00:30:54.632025+0000","last_undegraded":"2024-04-25T00:30:54.632025+0000","last_fullsized":"2024-04-25T00:30:54.632025+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:15:34.704958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.630613+0000","last_change":"2024-04-25T00:30:48.534024+0000","last_active":"2024-04-25T00:30:54.630613+0000","last_peered":"2024-04-25T00:30:54.630613+0000","last_clean":"2024-04-25T00:30:54.630613+0000","last_became_active":"2024-04-25T00:30:48.533681+0000","last_became_peered":"2024-04-25T00:30:48.533681+0000","last_unstale":"2024-04-25T00:30:54.630613+0000","last_undegraded":"2024-04-25T00:30:54.630613+0000","last_fullsized":"2024-04-25T00:30:54.630613+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:07:35.816816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.769396+0000","last_change":"2024-04-25T00:30:48.541263+0000","last_active":"2024-04-25T00:30:53.769396+0000","last_peered":"2024-04-25T00:30:53.769396+0000","last_clean":"2024-04-25T00:30:53.769396+0000","last_became_active":"2024-04-25T00:30:48.540889+0000","last_became_peered":"2024-04-25T00:30:48.540889+0000","last_unstale":"2024-04-25T00:30:53.769396+0000","last_undegraded":"2024-04-25T00:30:53.769396+0000","last_fullsized":"2024-04-25T00:30:53.769396+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:31:42.067004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,4],"acting":[10,9,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.130274+0000","last_change":"2024-04-25T00:30:48.533194+0000","last_active":"2024-04-25T00:30:54.130274+0000","last_peered":"2024-04-25T00:30:54.130274+0000","last_clean":"2024-04-25T00:30:54.130274+0000","last_became_active":"2024-04-25T00:30:48.532424+0000","last_became_peered":"2024-04-25T00:30:48.532424+0000","last_unstale":"2024-04-25T00:30:54.130274+0000","last_undegraded":"2024-04-25T00:30:54.130274+0000","last_fullsized":"2024-04-25T00:30:54.130274+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:06:50.187796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.630936+0000","last_change":"2024-04-25T00:30:48.542829+0000","last_active":"2024-04-25T00:30:54.630936+0000","last_peered":"2024-04-25T00:30:54.630936+0000","last_clean":"2024-04-25T00:30:54.630936+0000","last_became_active":"2024-04-25T00:30:48.542204+0000","last_became_peered":"2024-04-25T00:30:48.542204+0000","last_unstale":"2024-04-25T00:30:54.630936+0000","last_undegraded":"2024-04-25T00:30:54.630936+0000","last_fullsized":"2024-04-25T00:30:54.630936+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:56:47.869075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631732+0000","last_change":"2024-04-25T00:30:48.537924+0000","last_active":"2024-04-25T00:30:54.631732+0000","last_peered":"2024-04-25T00:30:54.631732+0000","last_clean":"2024-04-25T00:30:54.631732+0000","last_became_active":"2024-04-25T00:30:48.537547+0000","last_became_peered":"2024-04-25T00:30:48.537547+0000","last_unstale":"2024-04-25T00:30:54.631732+0000","last_undegraded":"2024-04-25T00:30:54.631732+0000","last_fullsized":"2024-04-25T00:30:54.631732+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:26:45.590923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.769787+0000","last_change":"2024-04-25T00:30:48.539717+0000","last_active":"2024-04-25T00:30:53.769787+0000","last_peered":"2024-04-25T00:30:53.769787+0000","last_clean":"2024-04-25T00:30:53.769787+0000","last_became_active":"2024-04-25T00:30:48.539242+0000","last_became_peered":"2024-04-25T00:30:48.539242+0000","last_unstale":"2024-04-25T00:30:53.769787+0000","last_undegraded":"2024-04-25T00:30:53.769787+0000","last_fullsized":"2024-04-25T00:30:53.769787+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:58:15.353959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.130730+0000","last_change":"2024-04-25T00:30:48.542767+0000","last_active":"2024-04-25T00:30:54.130730+0000","last_peered":"2024-04-25T00:30:54.130730+0000","last_clean":"2024-04-25T00:30:54.130730+0000","last_became_active":"2024-04-25T00:30:48.542252+0000","last_became_peered":"2024-04-25T00:30:48.542252+0000","last_unstale":"2024-04-25T00:30:54.130730+0000","last_undegraded":"2024-04-25T00:30:54.130730+0000","last_fullsized":"2024-04-25T00:30:54.130730+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:07:49.414660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631970+0000","last_change":"2024-04-25T00:30:48.532248+0000","last_active":"2024-04-25T00:30:54.631970+0000","last_peered":"2024-04-25T00:30:54.631970+0000","last_clean":"2024-04-25T00:30:54.631970+0000","last_became_active":"2024-04-25T00:30:48.531876+0000","last_became_peered":"2024-04-25T00:30:48.531876+0000","last_unstale":"2024-04-25T00:30:54.631970+0000","last_undegraded":"2024-04-25T00:30:54.631970+0000","last_fullsized":"2024-04-25T00:30:54.631970+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:00:35.711693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631287+0000","last_change":"2024-04-25T00:30:48.526871+0000","last_active":"2024-04-25T00:30:54.631287+0000","last_peered":"2024-04-25T00:30:54.631287+0000","last_clean":"2024-04-25T00:30:54.631287+0000","last_became_active":"2024-04-25T00:30:48.526477+0000","last_became_peered":"2024-04-25T00:30:48.526477+0000","last_unstale":"2024-04-25T00:30:54.631287+0000","last_undegraded":"2024-04-25T00:30:54.631287+0000","last_fullsized":"2024-04-25T00:30:54.631287+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:17:59.447932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.677352+0000","last_change":"2024-04-25T00:30:48.530297+0000","last_active":"2024-04-25T00:30:54.677352+0000","last_peered":"2024-04-25T00:30:54.677352+0000","last_clean":"2024-04-25T00:30:54.677352+0000","last_became_active":"2024-04-25T00:30:48.529091+0000","last_became_peered":"2024-04-25T00:30:48.529091+0000","last_unstale":"2024-04-25T00:30:54.677352+0000","last_undegraded":"2024-04-25T00:30:54.677352+0000","last_fullsized":"2024-04-25T00:30:54.677352+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:54:52.206881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.677314+0000","last_change":"2024-04-25T00:30:48.535629+0000","last_active":"2024-04-25T00:30:54.677314+0000","last_peered":"2024-04-25T00:30:54.677314+0000","last_clean":"2024-04-25T00:30:54.677314+0000","last_became_active":"2024-04-25T00:30:48.535190+0000","last_became_peered":"2024-04-25T00:30:48.535190+0000","last_unstale":"2024-04-25T00:30:54.677314+0000","last_undegraded":"2024-04-25T00:30:54.677314+0000","last_fullsized":"2024-04-25T00:30:54.677314+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:11:27.127964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.630736+0000","last_change":"2024-04-25T00:30:48.532988+0000","last_active":"2024-04-25T00:30:54.630736+0000","last_peered":"2024-04-25T00:30:54.630736+0000","last_clean":"2024-04-25T00:30:54.630736+0000","last_became_active":"2024-04-25T00:30:48.532670+0000","last_became_peered":"2024-04-25T00:30:48.532670+0000","last_unstale":"2024-04-25T00:30:54.630736+0000","last_undegraded":"2024-04-25T00:30:54.630736+0000","last_fullsized":"2024-04-25T00:30:54.630736+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:03:35.336175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.232626+0000","last_change":"2024-04-25T00:30:48.539499+0000","last_active":"2024-04-25T00:30:54.232626+0000","last_peered":"2024-04-25T00:30:54.232626+0000","last_clean":"2024-04-25T00:30:54.232626+0000","last_became_active":"2024-04-25T00:30:48.538516+0000","last_became_peered":"2024-04-25T00:30:48.538516+0000","last_unstale":"2024-04-25T00:30:54.232626+0000","last_undegraded":"2024-04-25T00:30:54.232626+0000","last_fullsized":"2024-04-25T00:30:54.232626+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:03:16.781091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.767084+0000","last_change":"2024-04-25T00:30:48.534204+0000","last_active":"2024-04-25T00:30:53.767084+0000","last_peered":"2024-04-25T00:30:53.767084+0000","last_clean":"2024-04-25T00:30:53.767084+0000","last_became_active":"2024-04-25T00:30:48.533818+0000","last_became_peered":"2024-04-25T00:30:48.533818+0000","last_unstale":"2024-04-25T00:30:53.767084+0000","last_undegraded":"2024-04-25T00:30:53.767084+0000","last_fullsized":"2024-04-25T00:30:53.767084+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:08:42.964056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.767341+0000","last_change":"2024-04-25T00:30:48.542756+0000","last_active":"2024-04-25T00:30:53.767341+0000","last_peered":"2024-04-25T00:30:53.767341+0000","last_clean":"2024-04-25T00:30:53.767341+0000","last_became_active":"2024-04-25T00:30:48.541824+0000","last_became_peered":"2024-04-25T00:30:48.541824+0000","last_unstale":"2024-04-25T00:30:53.767341+0000","last_undegraded":"2024-04-25T00:30:53.767341+0000","last_fullsized":"2024-04-25T00:30:53.767341+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:31:40.996474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.766876+0000","last_change":"2024-04-25T00:30:52.788477+0000","last_active":"2024-04-25T00:30:53.766876+0000","last_peered":"2024-04-25T00:30:53.766876+0000","last_clean":"2024-04-25T00:30:53.766876+0000","last_became_active":"2024-04-25T00:30:52.788073+0000","last_became_peered":"2024-04-25T00:30:52.788073+0000","last_unstale":"2024-04-25T00:30:53.766876+0000","last_undegraded":"2024-04-25T00:30:53.766876+0000","last_fullsized":"2024-04-25T00:30:53.766876+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:29:34.781119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,9,0],"acting":[2,3,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"89'9","reported_seq":33,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:31:13.975508+0000","last_change":"2024-04-25T00:30:47.524580+0000","last_active":"2024-04-25T00:31:13.975508+0000","last_peered":"2024-04-25T00:31:13.975508+0000","last_clean":"2024-04-25T00:31:13.975508+0000","last_became_active":"2024-04-25T00:30:47.524170+0000","last_became_peered":"2024-04-25T00:30:47.524170+0000","last_unstale":"2024-04-25T00:31:13.975508+0000","last_undegraded":"2024-04-25T00:31:13.975508+0000","last_fullsized":"2024-04-25T00:31:13.975508+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:53:08.734842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.632256+0000","last_change":"2024-04-25T00:30:48.542077+0000","last_active":"2024-04-25T00:30:54.632256+0000","last_peered":"2024-04-25T00:30:54.632256+0000","last_clean":"2024-04-25T00:30:54.632256+0000","last_became_active":"2024-04-25T00:30:48.539043+0000","last_became_peered":"2024-04-25T00:30:48.539043+0000","last_unstale":"2024-04-25T00:30:54.632256+0000","last_undegraded":"2024-04-25T00:30:54.632256+0000","last_fullsized":"2024-04-25T00:30:54.632256+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:48:52.254800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.630870+0000","last_change":"2024-04-25T00:30:52.786091+0000","last_active":"2024-04-25T00:30:54.630870+0000","last_peered":"2024-04-25T00:30:54.630870+0000","last_clean":"2024-04-25T00:30:54.630870+0000","last_became_active":"2024-04-25T00:30:52.785511+0000","last_became_peered":"2024-04-25T00:30:52.785511+0000","last_unstale":"2024-04-25T00:30:54.630870+0000","last_undegraded":"2024-04-25T00:30:54.630870+0000","last_fullsized":"2024-04-25T00:30:54.630870+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:19:17.992033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,9,3],"acting":[4,6,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1e","version":"84'1","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.632356+0000","last_change":"2024-04-25T00:30:47.529293+0000","last_active":"2024-04-25T00:30:54.632356+0000","last_peered":"2024-04-25T00:30:54.632356+0000","last_clean":"2024-04-25T00:30:54.632356+0000","last_became_active":"2024-04-25T00:30:47.527975+0000","last_became_peered":"2024-04-25T00:30:47.527975+0000","last_unstale":"2024-04-25T00:30:54.632356+0000","last_undegraded":"2024-04-25T00:30:54.632356+0000","last_fullsized":"2024-04-25T00:30:54.632356+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:17:40.057962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.677687+0000","last_change":"2024-04-25T00:30:48.530993+0000","last_active":"2024-04-25T00:30:54.677687+0000","last_peered":"2024-04-25T00:30:54.677687+0000","last_clean":"2024-04-25T00:30:54.677687+0000","last_became_active":"2024-04-25T00:30:48.530550+0000","last_became_peered":"2024-04-25T00:30:48.530550+0000","last_unstale":"2024-04-25T00:30:54.677687+0000","last_undegraded":"2024-04-25T00:30:54.677687+0000","last_fullsized":"2024-04-25T00:30:54.677687+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:11:35.308456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631715+0000","last_change":"2024-04-25T00:30:52.787774+0000","last_active":"2024-04-25T00:30:54.631715+0000","last_peered":"2024-04-25T00:30:54.631715+0000","last_clean":"2024-04-25T00:30:54.631715+0000","last_became_active":"2024-04-25T00:30:52.786988+0000","last_became_peered":"2024-04-25T00:30:52.786988+0000","last_unstale":"2024-04-25T00:30:54.631715+0000","last_undegraded":"2024-04-25T00:30:54.631715+0000","last_fullsized":"2024-04-25T00:30:54.631715+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:58:28.081767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,9,5],"acting":[4,3,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1d","version":"89'4","reported_seq":27,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:31:13.974259+0000","last_change":"2024-04-25T00:30:47.523477+0000","last_active":"2024-04-25T00:31:13.974259+0000","last_peered":"2024-04-25T00:31:13.974259+0000","last_clean":"2024-04-25T00:31:13.974259+0000","last_became_active":"2024-04-25T00:30:47.522961+0000","last_became_peered":"2024-04-25T00:30:47.522961+0000","last_unstale":"2024-04-25T00:31:13.974259+0000","last_undegraded":"2024-04-25T00:31:13.974259+0000","last_fullsized":"2024-04-25T00:31:13.974259+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:39:39.793477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631367+0000","last_change":"2024-04-25T00:30:48.527746+0000","last_active":"2024-04-25T00:30:54.631367+0000","last_peered":"2024-04-25T00:30:54.631367+0000","last_clean":"2024-04-25T00:30:54.631367+0000","last_became_active":"2024-04-25T00:30:48.527374+0000","last_became_peered":"2024-04-25T00:30:48.527374+0000","last_unstale":"2024-04-25T00:30:54.631367+0000","last_undegraded":"2024-04-25T00:30:54.631367+0000","last_fullsized":"2024-04-25T00:30:54.631367+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:28:36.822220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631458+0000","last_change":"2024-04-25T00:30:52.782368+0000","last_active":"2024-04-25T00:30:54.631458+0000","last_peered":"2024-04-25T00:30:54.631458+0000","last_clean":"2024-04-25T00:30:54.631458+0000","last_became_active":"2024-04-25T00:30:52.781875+0000","last_became_peered":"2024-04-25T00:30:52.781875+0000","last_unstale":"2024-04-25T00:30:54.631458+0000","last_undegraded":"2024-04-25T00:30:54.631458+0000","last_fullsized":"2024-04-25T00:30:54.631458+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:07:55.509131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5,3],"acting":[4,6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1c","version":"84'2","reported_seq":25,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631104+0000","last_change":"2024-04-25T00:30:47.523707+0000","last_active":"2024-04-25T00:30:54.631104+0000","last_peered":"2024-04-25T00:30:54.631104+0000","last_clean":"2024-04-25T00:30:54.631104+0000","last_became_active":"2024-04-25T00:30:47.523216+0000","last_became_peered":"2024-04-25T00:30:47.523216+0000","last_unstale":"2024-04-25T00:30:54.631104+0000","last_undegraded":"2024-04-25T00:30:54.631104+0000","last_fullsized":"2024-04-25T00:30:54.631104+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:21:21.006368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631375+0000","last_change":"2024-04-25T00:30:48.529235+0000","last_active":"2024-04-25T00:30:54.631375+0000","last_peered":"2024-04-25T00:30:54.631375+0000","last_clean":"2024-04-25T00:30:54.631375+0000","last_became_active":"2024-04-25T00:30:48.528887+0000","last_became_peered":"2024-04-25T00:30:48.528887+0000","last_unstale":"2024-04-25T00:30:54.631375+0000","last_undegraded":"2024-04-25T00:30:54.631375+0000","last_fullsized":"2024-04-25T00:30:54.631375+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:18:05.290578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.767234+0000","last_change":"2024-04-25T00:30:52.784576+0000","last_active":"2024-04-25T00:30:53.767234+0000","last_peered":"2024-04-25T00:30:53.767234+0000","last_clean":"2024-04-25T00:30:53.767234+0000","last_became_active":"2024-04-25T00:30:52.784027+0000","last_became_peered":"2024-04-25T00:30:52.784027+0000","last_unstale":"2024-04-25T00:30:53.767234+0000","last_undegraded":"2024-04-25T00:30:53.767234+0000","last_fullsized":"2024-04-25T00:30:53.767234+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:31:18.798072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7,11],"acting":[2,5,7,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1b","version":"84'1","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631112+0000","last_change":"2024-04-25T00:30:47.518472+0000","last_active":"2024-04-25T00:30:54.631112+0000","last_peered":"2024-04-25T00:30:54.631112+0000","last_clean":"2024-04-25T00:30:54.631112+0000","last_became_active":"2024-04-25T00:30:47.518140+0000","last_became_peered":"2024-04-25T00:30:47.518140+0000","last_unstale":"2024-04-25T00:30:54.631112+0000","last_undegraded":"2024-04-25T00:30:54.631112+0000","last_fullsized":"2024-04-25T00:30:54.631112+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:19:07.780949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.630861+0000","last_change":"2024-04-25T00:30:48.533757+0000","last_active":"2024-04-25T00:30:54.630861+0000","last_peered":"2024-04-25T00:30:54.630861+0000","last_clean":"2024-04-25T00:30:54.630861+0000","last_became_active":"2024-04-25T00:30:48.533421+0000","last_became_peered":"2024-04-25T00:30:48.533421+0000","last_unstale":"2024-04-25T00:30:54.630861+0000","last_undegraded":"2024-04-25T00:30:54.630861+0000","last_fullsized":"2024-04-25T00:30:54.630861+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:11:38.190783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.767144+0000","last_change":"2024-04-25T00:30:52.785179+0000","last_active":"2024-04-25T00:30:53.767144+0000","last_peered":"2024-04-25T00:30:53.767144+0000","last_clean":"2024-04-25T00:30:53.767144+0000","last_became_active":"2024-04-25T00:30:52.784761+0000","last_became_peered":"2024-04-25T00:30:52.784761+0000","last_unstale":"2024-04-25T00:30:53.767144+0000","last_undegraded":"2024-04-25T00:30:53.767144+0000","last_fullsized":"2024-04-25T00:30:53.767144+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:57:37.449915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,9,5],"acting":[2,6,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1a","version":"84'1","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.769354+0000","last_change":"2024-04-25T00:30:47.527306+0000","last_active":"2024-04-25T00:30:53.769354+0000","last_peered":"2024-04-25T00:30:53.769354+0000","last_clean":"2024-04-25T00:30:53.769354+0000","last_became_active":"2024-04-25T00:30:47.526935+0000","last_became_peered":"2024-04-25T00:30:47.526935+0000","last_unstale":"2024-04-25T00:30:53.769354+0000","last_undegraded":"2024-04-25T00:30:53.769354+0000","last_fullsized":"2024-04-25T00:30:53.769354+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:41:37.951143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631197+0000","last_change":"2024-04-25T00:30:48.536040+0000","last_active":"2024-04-25T00:30:54.631197+0000","last_peered":"2024-04-25T00:30:54.631197+0000","last_clean":"2024-04-25T00:30:54.631197+0000","last_became_active":"2024-04-25T00:30:48.535399+0000","last_became_peered":"2024-04-25T00:30:48.535399+0000","last_unstale":"2024-04-25T00:30:54.631197+0000","last_undegraded":"2024-04-25T00:30:54.631197+0000","last_fullsized":"2024-04-25T00:30:54.631197+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:44:42.906907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.232625+0000","last_change":"2024-04-25T00:30:52.784986+0000","last_active":"2024-04-25T00:30:54.232625+0000","last_peered":"2024-04-25T00:30:54.232625+0000","last_clean":"2024-04-25T00:30:54.232625+0000","last_became_active":"2024-04-25T00:30:52.784593+0000","last_became_peered":"2024-04-25T00:30:52.784593+0000","last_unstale":"2024-04-25T00:30:54.232625+0000","last_undegraded":"2024-04-25T00:30:54.232625+0000","last_fullsized":"2024-04-25T00:30:54.232625+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:04:05.833730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,9,0],"acting":[11,4,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631512+0000","last_change":"2024-04-25T00:30:47.523663+0000","last_active":"2024-04-25T00:30:54.631512+0000","last_peered":"2024-04-25T00:30:54.631512+0000","last_clean":"2024-04-25T00:30:54.631512+0000","last_became_active":"2024-04-25T00:30:47.523324+0000","last_became_peered":"2024-04-25T00:30:47.523324+0000","last_unstale":"2024-04-25T00:30:54.631512+0000","last_undegraded":"2024-04-25T00:30:54.631512+0000","last_fullsized":"2024-04-25T00:30:54.631512+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:51:23.593579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.630759+0000","last_change":"2024-04-25T00:30:48.529738+0000","last_active":"2024-04-25T00:30:54.630759+0000","last_peered":"2024-04-25T00:30:54.630759+0000","last_clean":"2024-04-25T00:30:54.630759+0000","last_became_active":"2024-04-25T00:30:48.529350+0000","last_became_peered":"2024-04-25T00:30:48.529350+0000","last_unstale":"2024-04-25T00:30:54.630759+0000","last_undegraded":"2024-04-25T00:30:54.630759+0000","last_fullsized":"2024-04-25T00:30:54.630759+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:51:29.238293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.677788+0000","last_change":"2024-04-25T00:30:52.788480+0000","last_active":"2024-04-25T00:30:54.677788+0000","last_peered":"2024-04-25T00:30:54.677788+0000","last_clean":"2024-04-25T00:30:54.677788+0000","last_became_active":"2024-04-25T00:30:52.788065+0000","last_became_peered":"2024-04-25T00:30:52.788065+0000","last_unstale":"2024-04-25T00:30:54.677788+0000","last_undegraded":"2024-04-25T00:30:54.677788+0000","last_fullsized":"2024-04-25T00:30:54.677788+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:16:50.179177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6,1],"acting":[0,7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.630894+0000","last_change":"2024-04-25T00:30:47.524706+0000","last_active":"2024-04-25T00:30:54.630894+0000","last_peered":"2024-04-25T00:30:54.630894+0000","last_clean":"2024-04-25T00:30:54.630894+0000","last_became_active":"2024-04-25T00:30:47.524355+0000","last_became_peered":"2024-04-25T00:30:47.524355+0000","last_unstale":"2024-04-25T00:30:54.630894+0000","last_undegraded":"2024-04-25T00:30:54.630894+0000","last_fullsized":"2024-04-25T00:30:54.630894+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:22:38.093362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.766423+0000","last_change":"2024-04-25T00:30:48.533266+0000","last_active":"2024-04-25T00:30:53.766423+0000","last_peered":"2024-04-25T00:30:53.766423+0000","last_clean":"2024-04-25T00:30:53.766423+0000","last_became_active":"2024-04-25T00:30:48.532932+0000","last_became_peered":"2024-04-25T00:30:48.532932+0000","last_unstale":"2024-04-25T00:30:53.766423+0000","last_undegraded":"2024-04-25T00:30:53.766423+0000","last_fullsized":"2024-04-25T00:30:53.766423+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:00:54.147725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631874+0000","last_change":"2024-04-25T00:30:52.780317+0000","last_active":"2024-04-25T00:30:54.631874+0000","last_peered":"2024-04-25T00:30:54.631874+0000","last_clean":"2024-04-25T00:30:54.631874+0000","last_became_active":"2024-04-25T00:30:52.778539+0000","last_became_peered":"2024-04-25T00:30:52.778539+0000","last_unstale":"2024-04-25T00:30:54.631874+0000","last_undegraded":"2024-04-25T00:30:54.631874+0000","last_fullsized":"2024-04-25T00:30:54.631874+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:34:49.527307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1,5],"acting":[3,7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.769733+0000","last_change":"2024-04-25T00:30:47.525647+0000","last_active":"2024-04-25T00:30:53.769733+0000","last_peered":"2024-04-25T00:30:53.769733+0000","last_clean":"2024-04-25T00:30:53.769733+0000","last_became_active":"2024-04-25T00:30:47.525280+0000","last_became_peered":"2024-04-25T00:30:47.525280+0000","last_unstale":"2024-04-25T00:30:53.769733+0000","last_undegraded":"2024-04-25T00:30:53.769733+0000","last_fullsized":"2024-04-25T00:30:53.769733+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:56:15.761173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631401+0000","last_change":"2024-04-25T00:30:48.527913+0000","last_active":"2024-04-25T00:30:54.631401+0000","last_peered":"2024-04-25T00:30:54.631401+0000","last_clean":"2024-04-25T00:30:54.631401+0000","last_became_active":"2024-04-25T00:30:48.527538+0000","last_became_peered":"2024-04-25T00:30:48.527538+0000","last_unstale":"2024-04-25T00:30:54.631401+0000","last_undegraded":"2024-04-25T00:30:54.631401+0000","last_fullsized":"2024-04-25T00:30:54.631401+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:48:38.646867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.766316+0000","last_change":"2024-04-25T00:30:52.788267+0000","last_active":"2024-04-25T00:30:53.766316+0000","last_peered":"2024-04-25T00:30:53.766316+0000","last_clean":"2024-04-25T00:30:53.766316+0000","last_became_active":"2024-04-25T00:30:52.785188+0000","last_became_peered":"2024-04-25T00:30:52.785188+0000","last_unstale":"2024-04-25T00:30:53.766316+0000","last_undegraded":"2024-04-25T00:30:53.766316+0000","last_fullsized":"2024-04-25T00:30:53.766316+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:38:25.194673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,0,2,4],"acting":[9,0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.16","version":"89'2","reported_seq":25,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:31:12.963192+0000","last_change":"2024-04-25T00:30:47.522213+0000","last_active":"2024-04-25T00:31:12.963192+0000","last_peered":"2024-04-25T00:31:12.963192+0000","last_clean":"2024-04-25T00:31:12.963192+0000","last_became_active":"2024-04-25T00:30:47.521519+0000","last_became_peered":"2024-04-25T00:30:47.521519+0000","last_unstale":"2024-04-25T00:31:12.963192+0000","last_undegraded":"2024-04-25T00:31:12.963192+0000","last_fullsized":"2024-04-25T00:31:12.963192+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:16:10.001222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.677800+0000","last_change":"2024-04-25T00:30:48.540784+0000","last_active":"2024-04-25T00:30:54.677800+0000","last_peered":"2024-04-25T00:30:54.677800+0000","last_clean":"2024-04-25T00:30:54.677800+0000","last_became_active":"2024-04-25T00:30:48.539782+0000","last_became_peered":"2024-04-25T00:30:48.539782+0000","last_unstale":"2024-04-25T00:30:54.677800+0000","last_undegraded":"2024-04-25T00:30:54.677800+0000","last_fullsized":"2024-04-25T00:30:54.677800+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:45:10.701113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631980+0000","last_change":"2024-04-25T00:30:52.786818+0000","last_active":"2024-04-25T00:30:54.631980+0000","last_peered":"2024-04-25T00:30:54.631980+0000","last_clean":"2024-04-25T00:30:54.631980+0000","last_became_active":"2024-04-25T00:30:52.786293+0000","last_became_peered":"2024-04-25T00:30:52.786293+0000","last_unstale":"2024-04-25T00:30:54.631980+0000","last_undegraded":"2024-04-25T00:30:54.631980+0000","last_fullsized":"2024-04-25T00:30:54.631980+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:27:02.751296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2,9],"acting":[4,1,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.15","version":"84'3","reported_seq":26,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631242+0000","last_change":"2024-04-25T00:30:47.520882+0000","last_active":"2024-04-25T00:30:54.631242+0000","last_peered":"2024-04-25T00:30:54.631242+0000","last_clean":"2024-04-25T00:30:54.631242+0000","last_became_active":"2024-04-25T00:30:47.520545+0000","last_became_peered":"2024-04-25T00:30:47.520545+0000","last_unstale":"2024-04-25T00:30:54.631242+0000","last_undegraded":"2024-04-25T00:30:54.631242+0000","last_fullsized":"2024-04-25T00:30:54.631242+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:46:04.715580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631561+0000","last_change":"2024-04-25T00:30:48.536814+0000","last_active":"2024-04-25T00:30:54.631561+0000","last_peered":"2024-04-25T00:30:54.631561+0000","last_clean":"2024-04-25T00:30:54.631561+0000","last_became_active":"2024-04-25T00:30:48.536486+0000","last_became_peered":"2024-04-25T00:30:48.536486+0000","last_unstale":"2024-04-25T00:30:54.631561+0000","last_undegraded":"2024-04-25T00:30:54.631561+0000","last_fullsized":"2024-04-25T00:30:54.631561+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:58:23.861959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.766639+0000","last_change":"2024-04-25T00:30:52.786070+0000","last_active":"2024-04-25T00:30:53.766639+0000","last_peered":"2024-04-25T00:30:53.766639+0000","last_clean":"2024-04-25T00:30:53.766639+0000","last_became_active":"2024-04-25T00:30:52.785582+0000","last_became_peered":"2024-04-25T00:30:52.785582+0000","last_unstale":"2024-04-25T00:30:53.766639+0000","last_undegraded":"2024-04-25T00:30:53.766639+0000","last_fullsized":"2024-04-25T00:30:53.766639+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:41:59.584597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,8,11,5],"acting":[9,8,11,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.14","version":"89'10","reported_seq":33,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:31:04.470078+0000","last_change":"2024-04-25T00:30:47.526899+0000","last_active":"2024-04-25T00:31:04.470078+0000","last_peered":"2024-04-25T00:31:04.470078+0000","last_clean":"2024-04-25T00:31:04.470078+0000","last_became_active":"2024-04-25T00:30:47.526528+0000","last_became_peered":"2024-04-25T00:30:47.526528+0000","last_unstale":"2024-04-25T00:31:04.470078+0000","last_undegraded":"2024-04-25T00:31:04.470078+0000","last_fullsized":"2024-04-25T00:31:04.470078+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:13:34.004584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3842,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631381+0000","last_change":"2024-04-25T00:30:48.538225+0000","last_active":"2024-04-25T00:30:54.631381+0000","last_peered":"2024-04-25T00:30:54.631381+0000","last_clean":"2024-04-25T00:30:54.631381+0000","last_became_active":"2024-04-25T00:30:48.537807+0000","last_became_peered":"2024-04-25T00:30:48.537807+0000","last_unstale":"2024-04-25T00:30:54.631381+0000","last_undegraded":"2024-04-25T00:30:54.631381+0000","last_fullsized":"2024-04-25T00:30:54.631381+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:00:46.275584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631725+0000","last_change":"2024-04-25T00:30:52.782266+0000","last_active":"2024-04-25T00:30:54.631725+0000","last_peered":"2024-04-25T00:30:54.631725+0000","last_clean":"2024-04-25T00:30:54.631725+0000","last_became_active":"2024-04-25T00:30:52.781133+0000","last_became_peered":"2024-04-25T00:30:52.781133+0000","last_unstale":"2024-04-25T00:30:54.631725+0000","last_undegraded":"2024-04-25T00:30:54.631725+0000","last_fullsized":"2024-04-25T00:30:54.631725+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:19:03.818560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,8,3,6],"acting":[5,8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.13","version":"84'2","reported_seq":25,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.767022+0000","last_change":"2024-04-25T00:30:47.520909+0000","last_active":"2024-04-25T00:30:53.767022+0000","last_peered":"2024-04-25T00:30:53.767022+0000","last_clean":"2024-04-25T00:30:53.767022+0000","last_became_active":"2024-04-25T00:30:47.520287+0000","last_became_peered":"2024-04-25T00:30:47.520287+0000","last_unstale":"2024-04-25T00:30:53.767022+0000","last_undegraded":"2024-04-25T00:30:53.767022+0000","last_fullsized":"2024-04-25T00:30:53.767022+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:49:43.795115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631201+0000","last_change":"2024-04-25T00:30:48.532270+0000","last_active":"2024-04-25T00:30:54.631201+0000","last_peered":"2024-04-25T00:30:54.631201+0000","last_clean":"2024-04-25T00:30:54.631201+0000","last_became_active":"2024-04-25T00:30:48.531882+0000","last_became_peered":"2024-04-25T00:30:48.531882+0000","last_unstale":"2024-04-25T00:30:54.631201+0000","last_undegraded":"2024-04-25T00:30:54.631201+0000","last_fullsized":"2024-04-25T00:30:54.631201+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:58:17.865635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.232374+0000","last_change":"2024-04-25T00:30:52.777708+0000","last_active":"2024-04-25T00:30:54.232374+0000","last_peered":"2024-04-25T00:30:54.232374+0000","last_clean":"2024-04-25T00:30:54.232374+0000","last_became_active":"2024-04-25T00:30:52.777254+0000","last_became_peered":"2024-04-25T00:30:52.777254+0000","last_unstale":"2024-04-25T00:30:54.232374+0000","last_undegraded":"2024-04-25T00:30:54.232374+0000","last_fullsized":"2024-04-25T00:30:54.232374+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:29:47.427190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,9,7,6],"acting":[11,9,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631506+0000","last_change":"2024-04-25T00:30:47.524667+0000","last_active":"2024-04-25T00:30:54.631506+0000","last_peered":"2024-04-25T00:30:54.631506+0000","last_clean":"2024-04-25T00:30:54.631506+0000","last_became_active":"2024-04-25T00:30:47.524361+0000","last_became_peered":"2024-04-25T00:30:47.524361+0000","last_unstale":"2024-04-25T00:30:54.631506+0000","last_undegraded":"2024-04-25T00:30:54.631506+0000","last_fullsized":"2024-04-25T00:30:54.631506+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:16:08.805720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.130366+0000","last_change":"2024-04-25T00:30:48.538658+0000","last_active":"2024-04-25T00:30:54.130366+0000","last_peered":"2024-04-25T00:30:54.130366+0000","last_clean":"2024-04-25T00:30:54.130366+0000","last_became_active":"2024-04-25T00:30:48.538316+0000","last_became_peered":"2024-04-25T00:30:48.538316+0000","last_unstale":"2024-04-25T00:30:54.130366+0000","last_undegraded":"2024-04-25T00:30:54.130366+0000","last_fullsized":"2024-04-25T00:30:54.130366+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:24:47.303559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.767144+0000","last_change":"2024-04-25T00:30:52.789512+0000","last_active":"2024-04-25T00:30:53.767144+0000","last_peered":"2024-04-25T00:30:53.767144+0000","last_clean":"2024-04-25T00:30:53.767144+0000","last_became_active":"2024-04-25T00:30:52.788929+0000","last_became_peered":"2024-04-25T00:30:52.788929+0000","last_unstale":"2024-04-25T00:30:53.767144+0000","last_undegraded":"2024-04-25T00:30:53.767144+0000","last_fullsized":"2024-04-25T00:30:53.767144+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:21:05.195296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,0,3],"acting":[9,7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631352+0000","last_change":"2024-04-25T00:30:47.513627+0000","last_active":"2024-04-25T00:30:54.631352+0000","last_peered":"2024-04-25T00:30:54.631352+0000","last_clean":"2024-04-25T00:30:54.631352+0000","last_became_active":"2024-04-25T00:30:47.513215+0000","last_became_peered":"2024-04-25T00:30:47.513215+0000","last_unstale":"2024-04-25T00:30:54.631352+0000","last_undegraded":"2024-04-25T00:30:54.631352+0000","last_fullsized":"2024-04-25T00:30:54.631352+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:32:16.111590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.130421+0000","last_change":"2024-04-25T00:30:48.533056+0000","last_active":"2024-04-25T00:30:54.130421+0000","last_peered":"2024-04-25T00:30:54.130421+0000","last_clean":"2024-04-25T00:30:54.130421+0000","last_became_active":"2024-04-25T00:30:48.532698+0000","last_became_peered":"2024-04-25T00:30:48.532698+0000","last_unstale":"2024-04-25T00:30:54.130421+0000","last_undegraded":"2024-04-25T00:30:54.130421+0000","last_fullsized":"2024-04-25T00:30:54.130421+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:36:24.987784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631231+0000","last_change":"2024-04-25T00:30:52.780669+0000","last_active":"2024-04-25T00:30:54.631231+0000","last_peered":"2024-04-25T00:30:54.631231+0000","last_clean":"2024-04-25T00:30:54.631231+0000","last_became_active":"2024-04-25T00:30:52.780222+0000","last_became_peered":"2024-04-25T00:30:52.780222+0000","last_unstale":"2024-04-25T00:30:54.631231+0000","last_undegraded":"2024-04-25T00:30:54.631231+0000","last_fullsized":"2024-04-25T00:30:54.631231+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:50:27.415077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,9,11],"acting":[6,3,9,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"89'15","reported_seq":38,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:31:13.976458+0000","last_change":"2024-04-25T00:30:47.522547+0000","last_active":"2024-04-25T00:31:13.976458+0000","last_peered":"2024-04-25T00:31:13.976458+0000","last_clean":"2024-04-25T00:31:13.976458+0000","last_became_active":"2024-04-25T00:30:47.522193+0000","last_became_peered":"2024-04-25T00:30:47.522193+0000","last_unstale":"2024-04-25T00:31:13.976458+0000","last_undegraded":"2024-04-25T00:31:13.976458+0000","last_fullsized":"2024-04-25T00:31:13.976458+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:10:10.417552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,2],"acting":[8,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631325+0000","last_change":"2024-04-25T00:30:48.534363+0000","last_active":"2024-04-25T00:30:54.631325+0000","last_peered":"2024-04-25T00:30:54.631325+0000","last_clean":"2024-04-25T00:30:54.631325+0000","last_became_active":"2024-04-25T00:30:48.533855+0000","last_became_peered":"2024-04-25T00:30:48.533855+0000","last_unstale":"2024-04-25T00:30:54.631325+0000","last_undegraded":"2024-04-25T00:30:54.631325+0000","last_fullsized":"2024-04-25T00:30:54.631325+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:28:29.374501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631290+0000","last_change":"2024-04-25T00:30:52.790323+0000","last_active":"2024-04-25T00:30:54.631290+0000","last_peered":"2024-04-25T00:30:54.631290+0000","last_clean":"2024-04-25T00:30:54.631290+0000","last_became_active":"2024-04-25T00:30:52.789905+0000","last_became_peered":"2024-04-25T00:30:52.789905+0000","last_unstale":"2024-04-25T00:30:54.631290+0000","last_undegraded":"2024-04-25T00:30:54.631290+0000","last_fullsized":"2024-04-25T00:30:54.631290+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:19:03.687318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3,1],"acting":[5,0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"84'1","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631456+0000","last_change":"2024-04-25T00:30:47.516468+0000","last_active":"2024-04-25T00:30:54.631456+0000","last_peered":"2024-04-25T00:30:54.631456+0000","last_clean":"2024-04-25T00:30:54.631456+0000","last_became_active":"2024-04-25T00:30:47.515672+0000","last_became_peered":"2024-04-25T00:30:47.515672+0000","last_unstale":"2024-04-25T00:30:54.631456+0000","last_undegraded":"2024-04-25T00:30:54.631456+0000","last_fullsized":"2024-04-25T00:30:54.631456+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:43:38.525018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.232539+0000","last_change":"2024-04-25T00:30:48.539679+0000","last_active":"2024-04-25T00:30:54.232539+0000","last_peered":"2024-04-25T00:30:54.232539+0000","last_clean":"2024-04-25T00:30:54.232539+0000","last_became_active":"2024-04-25T00:30:48.538065+0000","last_became_peered":"2024-04-25T00:30:48.538065+0000","last_unstale":"2024-04-25T00:30:54.232539+0000","last_undegraded":"2024-04-25T00:30:54.232539+0000","last_fullsized":"2024-04-25T00:30:54.232539+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:05:16.435010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.630856+0000","last_change":"2024-04-25T00:30:52.804824+0000","last_active":"2024-04-25T00:30:54.630856+0000","last_peered":"2024-04-25T00:30:54.630856+0000","last_clean":"2024-04-25T00:30:54.630856+0000","last_became_active":"2024-04-25T00:30:52.804322+0000","last_became_peered":"2024-04-25T00:30:52.804322+0000","last_unstale":"2024-04-25T00:30:54.630856+0000","last_undegraded":"2024-04-25T00:30:54.630856+0000","last_fullsized":"2024-04-25T00:30:54.630856+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:38:08.385170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,10,11,9],"acting":[8,10,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.766709+0000","last_change":"2024-04-25T00:30:47.516916+0000","last_active":"2024-04-25T00:30:53.766709+0000","last_peered":"2024-04-25T00:30:53.766709+0000","last_clean":"2024-04-25T00:30:53.766709+0000","last_became_active":"2024-04-25T00:30:47.516520+0000","last_became_peered":"2024-04-25T00:30:47.516520+0000","last_unstale":"2024-04-25T00:30:53.766709+0000","last_undegraded":"2024-04-25T00:30:53.766709+0000","last_fullsized":"2024-04-25T00:30:53.766709+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:29:19.099764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.769568+0000","last_change":"2024-04-25T00:30:48.537584+0000","last_active":"2024-04-25T00:30:53.769568+0000","last_peered":"2024-04-25T00:30:53.769568+0000","last_clean":"2024-04-25T00:30:53.769568+0000","last_became_active":"2024-04-25T00:30:48.537193+0000","last_became_peered":"2024-04-25T00:30:48.537193+0000","last_unstale":"2024-04-25T00:30:53.769568+0000","last_undegraded":"2024-04-25T00:30:53.769568+0000","last_fullsized":"2024-04-25T00:30:53.769568+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:15:22.075323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.632203+0000","last_change":"2024-04-25T00:30:52.784802+0000","last_active":"2024-04-25T00:30:54.632203+0000","last_peered":"2024-04-25T00:30:54.632203+0000","last_clean":"2024-04-25T00:30:54.632203+0000","last_became_active":"2024-04-25T00:30:52.784359+0000","last_became_peered":"2024-04-25T00:30:52.784359+0000","last_unstale":"2024-04-25T00:30:54.632203+0000","last_undegraded":"2024-04-25T00:30:54.632203+0000","last_fullsized":"2024-04-25T00:30:54.632203+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:19:57.723542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1,8],"acting":[6,0,1,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.130525+0000","last_change":"2024-04-25T00:30:47.525494+0000","last_active":"2024-04-25T00:30:54.130525+0000","last_peered":"2024-04-25T00:30:54.130525+0000","last_clean":"2024-04-25T00:30:54.130525+0000","last_became_active":"2024-04-25T00:30:47.525158+0000","last_became_peered":"2024-04-25T00:30:47.525158+0000","last_unstale":"2024-04-25T00:30:54.130525+0000","last_undegraded":"2024-04-25T00:30:54.130525+0000","last_fullsized":"2024-04-25T00:30:54.130525+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:51:18.543352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.769260+0000","last_change":"2024-04-25T00:30:48.540926+0000","last_active":"2024-04-25T00:30:53.769260+0000","last_peered":"2024-04-25T00:30:53.769260+0000","last_clean":"2024-04-25T00:30:53.769260+0000","last_became_active":"2024-04-25T00:30:48.540204+0000","last_became_peered":"2024-04-25T00:30:48.540204+0000","last_unstale":"2024-04-25T00:30:53.769260+0000","last_undegraded":"2024-04-25T00:30:53.769260+0000","last_fullsized":"2024-04-25T00:30:53.769260+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:02:45.530266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631372+0000","last_change":"2024-04-25T00:30:52.805780+0000","last_active":"2024-04-25T00:30:54.631372+0000","last_peered":"2024-04-25T00:30:54.631372+0000","last_clean":"2024-04-25T00:30:54.631372+0000","last_became_active":"2024-04-25T00:30:52.805222+0000","last_became_peered":"2024-04-25T00:30:52.805222+0000","last_unstale":"2024-04-25T00:30:54.631372+0000","last_undegraded":"2024-04-25T00:30:54.631372+0000","last_fullsized":"2024-04-25T00:30:54.631372+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:32:17.034630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,10,0],"acting":[6,5,10,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631237+0000","last_change":"2024-04-25T00:30:48.534335+0000","last_active":"2024-04-25T00:30:54.631237+0000","last_peered":"2024-04-25T00:30:54.631237+0000","last_clean":"2024-04-25T00:30:54.631237+0000","last_became_active":"2024-04-25T00:30:48.532670+0000","last_became_peered":"2024-04-25T00:30:48.532670+0000","last_unstale":"2024-04-25T00:30:54.631237+0000","last_undegraded":"2024-04-25T00:30:54.631237+0000","last_fullsized":"2024-04-25T00:30:54.631237+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:26:23.145918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"89'1","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631506+0000","last_change":"2024-04-25T00:30:47.527695+0000","last_active":"2024-04-25T00:30:54.631506+0000","last_peered":"2024-04-25T00:30:54.631506+0000","last_clean":"2024-04-25T00:30:54.631506+0000","last_became_active":"2024-04-25T00:30:47.527239+0000","last_became_peered":"2024-04-25T00:30:47.527239+0000","last_unstale":"2024-04-25T00:30:54.631506+0000","last_undegraded":"2024-04-25T00:30:54.631506+0000","last_fullsized":"2024-04-25T00:30:54.631506+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:54:30.911284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.130333+0000","last_change":"2024-04-25T00:30:52.781091+0000","last_active":"2024-04-25T00:30:54.130333+0000","last_peered":"2024-04-25T00:30:54.130333+0000","last_clean":"2024-04-25T00:30:54.130333+0000","last_became_active":"2024-04-25T00:30:52.780538+0000","last_became_peered":"2024-04-25T00:30:52.780538+0000","last_unstale":"2024-04-25T00:30:54.130333+0000","last_undegraded":"2024-04-25T00:30:54.130333+0000","last_fullsized":"2024-04-25T00:30:54.130333+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:49:18.141802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,11,9],"acting":[7,5,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631440+0000","last_change":"2024-04-25T00:30:48.538730+0000","last_active":"2024-04-25T00:30:54.631440+0000","last_peered":"2024-04-25T00:30:54.631440+0000","last_clean":"2024-04-25T00:30:54.631440+0000","last_became_active":"2024-04-25T00:30:48.538336+0000","last_became_peered":"2024-04-25T00:30:48.538336+0000","last_unstale":"2024-04-25T00:30:54.631440+0000","last_undegraded":"2024-04-25T00:30:54.631440+0000","last_fullsized":"2024-04-25T00:30:54.631440+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:44:22.370448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631739+0000","last_change":"2024-04-25T00:30:47.520450+0000","last_active":"2024-04-25T00:30:54.631739+0000","last_peered":"2024-04-25T00:30:54.631739+0000","last_clean":"2024-04-25T00:30:54.631739+0000","last_became_active":"2024-04-25T00:30:47.520073+0000","last_became_peered":"2024-04-25T00:30:47.520073+0000","last_unstale":"2024-04-25T00:30:54.631739+0000","last_undegraded":"2024-04-25T00:30:54.631739+0000","last_fullsized":"2024-04-25T00:30:54.631739+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:49:53.570421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"22'137","reported_seq":356,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.130117+0000","last_change":"2024-04-25T00:29:34.720552+0000","last_active":"2024-04-25T00:30:54.130117+0000","last_peered":"2024-04-25T00:30:54.130117+0000","last_clean":"2024-04-25T00:30:54.130117+0000","last_became_active":"2024-04-25T00:29:34.720041+0000","last_became_peered":"2024-04-25T00:29:34.720041+0000","last_unstale":"2024-04-25T00:30:54.130117+0000","last_undegraded":"2024-04-25T00:30:54.130117+0000","last_fullsized":"2024-04-25T00:30:54.130117+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:26:07.691582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:26:07.691582+0000","last_clean_scrub_stamp":"2024-04-25T00:26:07.691582+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:15:27.995162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631112+0000","last_change":"2024-04-25T00:30:52.805489+0000","last_active":"2024-04-25T00:30:54.631112+0000","last_peered":"2024-04-25T00:30:54.631112+0000","last_clean":"2024-04-25T00:30:54.631112+0000","last_became_active":"2024-04-25T00:30:52.804758+0000","last_became_peered":"2024-04-25T00:30:52.804758+0000","last_unstale":"2024-04-25T00:30:54.631112+0000","last_undegraded":"2024-04-25T00:30:54.631112+0000","last_fullsized":"2024-04-25T00:30:54.631112+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:17:31.325548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,6,10],"acting":[5,9,6,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631348+0000","last_change":"2024-04-25T00:30:48.537699+0000","last_active":"2024-04-25T00:30:54.631348+0000","last_peered":"2024-04-25T00:30:54.631348+0000","last_clean":"2024-04-25T00:30:54.631348+0000","last_became_active":"2024-04-25T00:30:48.537291+0000","last_became_peered":"2024-04-25T00:30:48.537291+0000","last_unstale":"2024-04-25T00:30:54.631348+0000","last_undegraded":"2024-04-25T00:30:54.631348+0000","last_fullsized":"2024-04-25T00:30:54.631348+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:25:36.816499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"89'1","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.130635+0000","last_change":"2024-04-25T00:30:47.524694+0000","last_active":"2024-04-25T00:30:54.130635+0000","last_peered":"2024-04-25T00:30:54.130635+0000","last_clean":"2024-04-25T00:30:54.130635+0000","last_became_active":"2024-04-25T00:30:47.524359+0000","last_became_peered":"2024-04-25T00:30:47.524359+0000","last_unstale":"2024-04-25T00:30:54.130635+0000","last_undegraded":"2024-04-25T00:30:54.130635+0000","last_fullsized":"2024-04-25T00:30:54.130635+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:55:16.677234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.232415+0000","last_change":"2024-04-25T00:30:52.780364+0000","last_active":"2024-04-25T00:30:54.232415+0000","last_peered":"2024-04-25T00:30:54.232415+0000","last_clean":"2024-04-25T00:30:54.232415+0000","last_became_active":"2024-04-25T00:30:52.779904+0000","last_became_peered":"2024-04-25T00:30:52.779904+0000","last_unstale":"2024-04-25T00:30:54.232415+0000","last_undegraded":"2024-04-25T00:30:54.232415+0000","last_fullsized":"2024-04-25T00:30:54.232415+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:11:26.738526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,7,6],"acting":[11,5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.5","version":"89'1","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631522+0000","last_change":"2024-04-25T00:30:47.525205+0000","last_active":"2024-04-25T00:30:54.631522+0000","last_peered":"2024-04-25T00:30:54.631522+0000","last_clean":"2024-04-25T00:30:54.631522+0000","last_became_active":"2024-04-25T00:30:47.517729+0000","last_became_peered":"2024-04-25T00:30:47.517729+0000","last_unstale":"2024-04-25T00:30:54.631522+0000","last_undegraded":"2024-04-25T00:30:54.631522+0000","last_fullsized":"2024-04-25T00:30:54.631522+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:21:14.810119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.632230+0000","last_change":"2024-04-25T00:30:48.533822+0000","last_active":"2024-04-25T00:30:54.632230+0000","last_peered":"2024-04-25T00:30:54.632230+0000","last_clean":"2024-04-25T00:30:54.632230+0000","last_became_active":"2024-04-25T00:30:48.533479+0000","last_became_peered":"2024-04-25T00:30:48.533479+0000","last_unstale":"2024-04-25T00:30:54.632230+0000","last_undegraded":"2024-04-25T00:30:54.632230+0000","last_fullsized":"2024-04-25T00:30:54.632230+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:49:01.028124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.630925+0000","last_change":"2024-04-25T00:30:52.804770+0000","last_active":"2024-04-25T00:30:54.630925+0000","last_peered":"2024-04-25T00:30:54.630925+0000","last_clean":"2024-04-25T00:30:54.630925+0000","last_became_active":"2024-04-25T00:30:52.804242+0000","last_became_peered":"2024-04-25T00:30:52.804242+0000","last_unstale":"2024-04-25T00:30:54.630925+0000","last_undegraded":"2024-04-25T00:30:54.630925+0000","last_fullsized":"2024-04-25T00:30:54.630925+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:34:11.761478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,10,0,7],"acting":[3,10,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"89'4","reported_seq":27,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:31:13.975840+0000","last_change":"2024-04-25T00:30:47.521364+0000","last_active":"2024-04-25T00:31:13.975840+0000","last_peered":"2024-04-25T00:31:13.975840+0000","last_clean":"2024-04-25T00:31:13.975840+0000","last_became_active":"2024-04-25T00:30:47.521032+0000","last_became_peered":"2024-04-25T00:30:47.521032+0000","last_unstale":"2024-04-25T00:31:13.975840+0000","last_undegraded":"2024-04-25T00:31:13.975840+0000","last_fullsized":"2024-04-25T00:31:13.975840+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:33:26.156568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,8,3],"acting":[9,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631867+0000","last_change":"2024-04-25T00:30:48.532942+0000","last_active":"2024-04-25T00:30:54.631867+0000","last_peered":"2024-04-25T00:30:54.631867+0000","last_clean":"2024-04-25T00:30:54.631867+0000","last_became_active":"2024-04-25T00:30:48.532580+0000","last_became_peered":"2024-04-25T00:30:48.532580+0000","last_unstale":"2024-04-25T00:30:54.631867+0000","last_undegraded":"2024-04-25T00:30:54.631867+0000","last_fullsized":"2024-04-25T00:30:54.631867+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:42:22.887920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.766806+0000","last_change":"2024-04-25T00:30:52.779757+0000","last_active":"2024-04-25T00:30:53.766806+0000","last_peered":"2024-04-25T00:30:53.766806+0000","last_clean":"2024-04-25T00:30:53.766806+0000","last_became_active":"2024-04-25T00:30:52.779324+0000","last_became_peered":"2024-04-25T00:30:52.779324+0000","last_unstale":"2024-04-25T00:30:53.766806+0000","last_undegraded":"2024-04-25T00:30:53.766806+0000","last_fullsized":"2024-04-25T00:30:53.766806+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:22:39.062000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4,7],"acting":[2,1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"84'2","reported_seq":25,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.632132+0000","last_change":"2024-04-25T00:30:47.518212+0000","last_active":"2024-04-25T00:30:54.632132+0000","last_peered":"2024-04-25T00:30:54.632132+0000","last_clean":"2024-04-25T00:30:54.632132+0000","last_became_active":"2024-04-25T00:30:47.517610+0000","last_became_peered":"2024-04-25T00:30:47.517610+0000","last_unstale":"2024-04-25T00:30:54.632132+0000","last_undegraded":"2024-04-25T00:30:54.632132+0000","last_fullsized":"2024-04-25T00:30:54.632132+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:13:58.904020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.677796+0000","last_change":"2024-04-25T00:30:48.539808+0000","last_active":"2024-04-25T00:30:54.677796+0000","last_peered":"2024-04-25T00:30:54.677796+0000","last_clean":"2024-04-25T00:30:54.677796+0000","last_became_active":"2024-04-25T00:30:48.539168+0000","last_became_peered":"2024-04-25T00:30:48.539168+0000","last_unstale":"2024-04-25T00:30:54.677796+0000","last_undegraded":"2024-04-25T00:30:54.677796+0000","last_fullsized":"2024-04-25T00:30:54.677796+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:33:07.445896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.632161+0000","last_change":"2024-04-25T00:30:52.804915+0000","last_active":"2024-04-25T00:30:54.632161+0000","last_peered":"2024-04-25T00:30:54.632161+0000","last_clean":"2024-04-25T00:30:54.632161+0000","last_became_active":"2024-04-25T00:30:52.804491+0000","last_became_peered":"2024-04-25T00:30:52.804491+0000","last_unstale":"2024-04-25T00:30:54.632161+0000","last_undegraded":"2024-04-25T00:30:54.632161+0000","last_fullsized":"2024-04-25T00:30:54.632161+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:51:50.219556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,10,9],"acting":[4,5,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.630747+0000","last_change":"2024-04-25T00:30:47.521909+0000","last_active":"2024-04-25T00:30:54.630747+0000","last_peered":"2024-04-25T00:30:54.630747+0000","last_clean":"2024-04-25T00:30:54.630747+0000","last_became_active":"2024-04-25T00:30:47.521551+0000","last_became_peered":"2024-04-25T00:30:47.521551+0000","last_unstale":"2024-04-25T00:30:54.630747+0000","last_undegraded":"2024-04-25T00:30:54.630747+0000","last_fullsized":"2024-04-25T00:30:54.630747+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:51:16.368609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.630574+0000","last_change":"2024-04-25T00:30:48.530514+0000","last_active":"2024-04-25T00:30:54.630574+0000","last_peered":"2024-04-25T00:30:54.630574+0000","last_clean":"2024-04-25T00:30:54.630574+0000","last_became_active":"2024-04-25T00:30:48.530129+0000","last_became_peered":"2024-04-25T00:30:48.530129+0000","last_unstale":"2024-04-25T00:30:54.630574+0000","last_undegraded":"2024-04-25T00:30:54.630574+0000","last_fullsized":"2024-04-25T00:30:54.630574+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:51:25.623186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.767244+0000","last_change":"2024-04-25T00:30:52.785767+0000","last_active":"2024-04-25T00:30:53.767244+0000","last_peered":"2024-04-25T00:30:53.767244+0000","last_clean":"2024-04-25T00:30:53.767244+0000","last_became_active":"2024-04-25T00:30:52.785301+0000","last_became_peered":"2024-04-25T00:30:52.785301+0000","last_unstale":"2024-04-25T00:30:53.767244+0000","last_undegraded":"2024-04-25T00:30:53.767244+0000","last_fullsized":"2024-04-25T00:30:53.767244+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:10:14.071733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,2,5],"acting":[9,3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631562+0000","last_change":"2024-04-25T00:30:47.526364+0000","last_active":"2024-04-25T00:30:54.631562+0000","last_peered":"2024-04-25T00:30:54.631562+0000","last_clean":"2024-04-25T00:30:54.631562+0000","last_became_active":"2024-04-25T00:30:47.526009+0000","last_became_peered":"2024-04-25T00:30:47.526009+0000","last_unstale":"2024-04-25T00:30:54.631562+0000","last_undegraded":"2024-04-25T00:30:54.631562+0000","last_fullsized":"2024-04-25T00:30:54.631562+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:17:38.471878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.232186+0000","last_change":"2024-04-25T00:30:48.532283+0000","last_active":"2024-04-25T00:30:54.232186+0000","last_peered":"2024-04-25T00:30:54.232186+0000","last_clean":"2024-04-25T00:30:54.232186+0000","last_became_active":"2024-04-25T00:30:48.531935+0000","last_became_peered":"2024-04-25T00:30:48.531935+0000","last_unstale":"2024-04-25T00:30:54.232186+0000","last_undegraded":"2024-04-25T00:30:54.232186+0000","last_fullsized":"2024-04-25T00:30:54.232186+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:01:15.965611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631657+0000","last_change":"2024-04-25T00:30:52.786529+0000","last_active":"2024-04-25T00:30:54.631657+0000","last_peered":"2024-04-25T00:30:54.631657+0000","last_clean":"2024-04-25T00:30:54.631657+0000","last_became_active":"2024-04-25T00:30:52.785993+0000","last_became_peered":"2024-04-25T00:30:52.785993+0000","last_unstale":"2024-04-25T00:30:54.631657+0000","last_undegraded":"2024-04-25T00:30:54.631657+0000","last_fullsized":"2024-04-25T00:30:54.631657+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:59:24.360756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,9,7,8],"acting":[4,9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.766939+0000","last_change":"2024-04-25T00:30:47.520608+0000","last_active":"2024-04-25T00:30:53.766939+0000","last_peered":"2024-04-25T00:30:53.766939+0000","last_clean":"2024-04-25T00:30:53.766939+0000","last_became_active":"2024-04-25T00:30:47.520063+0000","last_became_peered":"2024-04-25T00:30:47.520063+0000","last_unstale":"2024-04-25T00:30:53.766939+0000","last_undegraded":"2024-04-25T00:30:53.766939+0000","last_fullsized":"2024-04-25T00:30:53.766939+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:58:47.823712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.632035+0000","last_change":"2024-04-25T00:30:48.535073+0000","last_active":"2024-04-25T00:30:54.632035+0000","last_peered":"2024-04-25T00:30:54.632035+0000","last_clean":"2024-04-25T00:30:54.632035+0000","last_became_active":"2024-04-25T00:30:48.534247+0000","last_became_peered":"2024-04-25T00:30:48.534247+0000","last_unstale":"2024-04-25T00:30:54.632035+0000","last_undegraded":"2024-04-25T00:30:54.632035+0000","last_fullsized":"2024-04-25T00:30:54.632035+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:46:18.613705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631695+0000","last_change":"2024-04-25T00:30:52.804519+0000","last_active":"2024-04-25T00:30:54.631695+0000","last_peered":"2024-04-25T00:30:54.631695+0000","last_clean":"2024-04-25T00:30:54.631695+0000","last_became_active":"2024-04-25T00:30:52.804074+0000","last_became_peered":"2024-04-25T00:30:52.804074+0000","last_unstale":"2024-04-25T00:30:54.631695+0000","last_undegraded":"2024-04-25T00:30:54.631695+0000","last_fullsized":"2024-04-25T00:30:54.631695+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:48:45.058873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,10,3],"acting":[4,2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.b","version":"89'4","reported_seq":27,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:31:12.960859+0000","last_change":"2024-04-25T00:30:47.521054+0000","last_active":"2024-04-25T00:31:12.960859+0000","last_peered":"2024-04-25T00:31:12.960859+0000","last_clean":"2024-04-25T00:31:12.960859+0000","last_became_active":"2024-04-25T00:30:47.520716+0000","last_became_peered":"2024-04-25T00:30:47.520716+0000","last_unstale":"2024-04-25T00:31:12.960859+0000","last_undegraded":"2024-04-25T00:31:12.960859+0000","last_fullsized":"2024-04-25T00:31:12.960859+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:05:16.471589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.130691+0000","last_change":"2024-04-25T00:30:48.537152+0000","last_active":"2024-04-25T00:30:54.130691+0000","last_peered":"2024-04-25T00:30:54.130691+0000","last_clean":"2024-04-25T00:30:54.130691+0000","last_became_active":"2024-04-25T00:30:48.536256+0000","last_became_peered":"2024-04-25T00:30:48.536256+0000","last_unstale":"2024-04-25T00:30:54.130691+0000","last_undegraded":"2024-04-25T00:30:54.130691+0000","last_fullsized":"2024-04-25T00:30:54.130691+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:57:53.870212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.630951+0000","last_change":"2024-04-25T00:30:52.781022+0000","last_active":"2024-04-25T00:30:54.630951+0000","last_peered":"2024-04-25T00:30:54.630951+0000","last_clean":"2024-04-25T00:30:54.630951+0000","last_became_active":"2024-04-25T00:30:52.780595+0000","last_became_peered":"2024-04-25T00:30:52.780595+0000","last_unstale":"2024-04-25T00:30:54.630951+0000","last_undegraded":"2024-04-25T00:30:54.630951+0000","last_fullsized":"2024-04-25T00:30:54.630951+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:27:02.580870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6,7],"acting":[1,5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.767244+0000","last_change":"2024-04-25T00:30:47.519118+0000","last_active":"2024-04-25T00:30:53.767244+0000","last_peered":"2024-04-25T00:30:53.767244+0000","last_clean":"2024-04-25T00:30:53.767244+0000","last_became_active":"2024-04-25T00:30:47.518639+0000","last_became_peered":"2024-04-25T00:30:47.518639+0000","last_unstale":"2024-04-25T00:30:53.767244+0000","last_undegraded":"2024-04-25T00:30:53.767244+0000","last_fullsized":"2024-04-25T00:30:53.767244+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:32:29.747903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.130732+0000","last_change":"2024-04-25T00:30:48.542073+0000","last_active":"2024-04-25T00:30:54.130732+0000","last_peered":"2024-04-25T00:30:54.130732+0000","last_clean":"2024-04-25T00:30:54.130732+0000","last_became_active":"2024-04-25T00:30:48.541722+0000","last_became_peered":"2024-04-25T00:30:48.541722+0000","last_unstale":"2024-04-25T00:30:54.130732+0000","last_undegraded":"2024-04-25T00:30:54.130732+0000","last_fullsized":"2024-04-25T00:30:54.130732+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:34:03.397539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.677591+0000","last_change":"2024-04-25T00:30:52.790234+0000","last_active":"2024-04-25T00:30:54.677591+0000","last_peered":"2024-04-25T00:30:54.677591+0000","last_clean":"2024-04-25T00:30:54.677591+0000","last_became_active":"2024-04-25T00:30:52.789788+0000","last_became_peered":"2024-04-25T00:30:52.789788+0000","last_unstale":"2024-04-25T00:30:54.677591+0000","last_undegraded":"2024-04-25T00:30:54.677591+0000","last_fullsized":"2024-04-25T00:30:54.677591+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:44:43.399771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,9,1,4],"acting":[0,9,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.d","version":"84'1","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631477+0000","last_change":"2024-04-25T00:30:47.523597+0000","last_active":"2024-04-25T00:30:54.631477+0000","last_peered":"2024-04-25T00:30:54.631477+0000","last_clean":"2024-04-25T00:30:54.631477+0000","last_became_active":"2024-04-25T00:30:47.523225+0000","last_became_peered":"2024-04-25T00:30:47.523225+0000","last_unstale":"2024-04-25T00:30:54.631477+0000","last_undegraded":"2024-04-25T00:30:54.631477+0000","last_fullsized":"2024-04-25T00:30:54.631477+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:51:12.341015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.130412+0000","last_change":"2024-04-25T00:30:48.541417+0000","last_active":"2024-04-25T00:30:54.130412+0000","last_peered":"2024-04-25T00:30:54.130412+0000","last_clean":"2024-04-25T00:30:54.130412+0000","last_became_active":"2024-04-25T00:30:48.541015+0000","last_became_peered":"2024-04-25T00:30:48.541015+0000","last_unstale":"2024-04-25T00:30:54.130412+0000","last_undegraded":"2024-04-25T00:30:54.130412+0000","last_fullsized":"2024-04-25T00:30:54.130412+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:21:35.299823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631288+0000","last_change":"2024-04-25T00:30:52.782230+0000","last_active":"2024-04-25T00:30:54.631288+0000","last_peered":"2024-04-25T00:30:54.631288+0000","last_clean":"2024-04-25T00:30:54.631288+0000","last_became_active":"2024-04-25T00:30:52.781708+0000","last_became_peered":"2024-04-25T00:30:52.781708+0000","last_unstale":"2024-04-25T00:30:54.631288+0000","last_undegraded":"2024-04-25T00:30:54.631288+0000","last_fullsized":"2024-04-25T00:30:54.631288+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:58:53.260985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7,1],"acting":[5,2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.e","version":"84'2","reported_seq":25,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631160+0000","last_change":"2024-04-25T00:30:47.522400+0000","last_active":"2024-04-25T00:30:54.631160+0000","last_peered":"2024-04-25T00:30:54.631160+0000","last_clean":"2024-04-25T00:30:54.631160+0000","last_became_active":"2024-04-25T00:30:47.522032+0000","last_became_peered":"2024-04-25T00:30:47.522032+0000","last_unstale":"2024-04-25T00:30:54.631160+0000","last_undegraded":"2024-04-25T00:30:54.631160+0000","last_fullsized":"2024-04-25T00:30:54.631160+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:24:17.749437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.130776+0000","last_change":"2024-04-25T00:30:48.532730+0000","last_active":"2024-04-25T00:30:54.130776+0000","last_peered":"2024-04-25T00:30:54.130776+0000","last_clean":"2024-04-25T00:30:54.130776+0000","last_became_active":"2024-04-25T00:30:48.532129+0000","last_became_peered":"2024-04-25T00:30:48.532129+0000","last_unstale":"2024-04-25T00:30:54.130776+0000","last_undegraded":"2024-04-25T00:30:54.130776+0000","last_fullsized":"2024-04-25T00:30:54.130776+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:36:30.115125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.232228+0000","last_change":"2024-04-25T00:30:48.535827+0000","last_active":"2024-04-25T00:30:54.232228+0000","last_peered":"2024-04-25T00:30:54.232228+0000","last_clean":"2024-04-25T00:30:54.232228+0000","last_became_active":"2024-04-25T00:30:48.535484+0000","last_became_peered":"2024-04-25T00:30:48.535484+0000","last_unstale":"2024-04-25T00:30:54.232228+0000","last_undegraded":"2024-04-25T00:30:54.232228+0000","last_fullsized":"2024-04-25T00:30:54.232228+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:11:44.596594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.766987+0000","last_change":"2024-04-25T00:30:48.540676+0000","last_active":"2024-04-25T00:30:53.766987+0000","last_peered":"2024-04-25T00:30:53.766987+0000","last_clean":"2024-04-25T00:30:53.766987+0000","last_became_active":"2024-04-25T00:30:48.540258+0000","last_became_peered":"2024-04-25T00:30:48.540258+0000","last_unstale":"2024-04-25T00:30:53.766987+0000","last_undegraded":"2024-04-25T00:30:53.766987+0000","last_fullsized":"2024-04-25T00:30:53.766987+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:10:29.930897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.766838+0000","last_change":"2024-04-25T00:30:48.542191+0000","last_active":"2024-04-25T00:30:53.766838+0000","last_peered":"2024-04-25T00:30:53.766838+0000","last_clean":"2024-04-25T00:30:53.766838+0000","last_became_active":"2024-04-25T00:30:48.540433+0000","last_became_peered":"2024-04-25T00:30:48.540433+0000","last_unstale":"2024-04-25T00:30:53.766838+0000","last_undegraded":"2024-04-25T00:30:53.766838+0000","last_fullsized":"2024-04-25T00:30:53.766838+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:21:55.694555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.677761+0000","last_change":"2024-04-25T00:30:48.539797+0000","last_active":"2024-04-25T00:30:54.677761+0000","last_peered":"2024-04-25T00:30:54.677761+0000","last_clean":"2024-04-25T00:30:54.677761+0000","last_became_active":"2024-04-25T00:30:48.539423+0000","last_became_peered":"2024-04-25T00:30:48.539423+0000","last_unstale":"2024-04-25T00:30:54.677761+0000","last_undegraded":"2024-04-25T00:30:54.677761+0000","last_fullsized":"2024-04-25T00:30:54.677761+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:12:54.427579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.632335+0000","last_change":"2024-04-25T00:30:48.527892+0000","last_active":"2024-04-25T00:30:54.632335+0000","last_peered":"2024-04-25T00:30:54.632335+0000","last_clean":"2024-04-25T00:30:54.632335+0000","last_became_active":"2024-04-25T00:30:48.527522+0000","last_became_peered":"2024-04-25T00:30:48.527522+0000","last_unstale":"2024-04-25T00:30:54.632335+0000","last_undegraded":"2024-04-25T00:30:54.632335+0000","last_fullsized":"2024-04-25T00:30:54.632335+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:21:22.779470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.232569+0000","last_change":"2024-04-25T00:30:48.535066+0000","last_active":"2024-04-25T00:30:54.232569+0000","last_peered":"2024-04-25T00:30:54.232569+0000","last_clean":"2024-04-25T00:30:54.232569+0000","last_became_active":"2024-04-25T00:30:48.534710+0000","last_became_peered":"2024-04-25T00:30:48.534710+0000","last_unstale":"2024-04-25T00:30:54.232569+0000","last_undegraded":"2024-04-25T00:30:54.232569+0000","last_fullsized":"2024-04-25T00:30:54.232569+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:57:18.582569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:53.767388+0000","last_change":"2024-04-25T00:30:48.541921+0000","last_active":"2024-04-25T00:30:53.767388+0000","last_peered":"2024-04-25T00:30:53.767388+0000","last_clean":"2024-04-25T00:30:53.767388+0000","last_became_active":"2024-04-25T00:30:48.541348+0000","last_became_peered":"2024-04-25T00:30:48.541348+0000","last_unstale":"2024-04-25T00:30:53.767388+0000","last_undegraded":"2024-04-25T00:30:53.767388+0000","last_fullsized":"2024-04-25T00:30:53.767388+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:07:00.001504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.632044+0000","last_change":"2024-04-25T00:30:48.536902+0000","last_active":"2024-04-25T00:30:54.632044+0000","last_peered":"2024-04-25T00:30:54.632044+0000","last_clean":"2024-04-25T00:30:54.632044+0000","last_became_active":"2024-04-25T00:30:48.536008+0000","last_became_peered":"2024-04-25T00:30:48.536008+0000","last_unstale":"2024-04-25T00:30:54.632044+0000","last_undegraded":"2024-04-25T00:30:54.632044+0000","last_fullsized":"2024-04-25T00:30:54.632044+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:19:15.050142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631104+0000","last_change":"2024-04-25T00:30:48.533104+0000","last_active":"2024-04-25T00:30:54.631104+0000","last_peered":"2024-04-25T00:30:54.631104+0000","last_clean":"2024-04-25T00:30:54.631104+0000","last_became_active":"2024-04-25T00:30:48.532124+0000","last_became_peered":"2024-04-25T00:30:48.532124+0000","last_unstale":"2024-04-25T00:30:54.631104+0000","last_undegraded":"2024-04-25T00:30:54.631104+0000","last_fullsized":"2024-04-25T00:30:54.631104+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:54:57.391433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631470+0000","last_change":"2024-04-25T00:30:48.542074+0000","last_active":"2024-04-25T00:30:54.631470+0000","last_peered":"2024-04-25T00:30:54.631470+0000","last_clean":"2024-04-25T00:30:54.631470+0000","last_became_active":"2024-04-25T00:30:48.538752+0000","last_became_peered":"2024-04-25T00:30:48.538752+0000","last_unstale":"2024-04-25T00:30:54.631470+0000","last_undegraded":"2024-04-25T00:30:54.631470+0000","last_fullsized":"2024-04-25T00:30:54.631470+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:55:30.613616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631263+0000","last_change":"2024-04-25T00:30:48.534158+0000","last_active":"2024-04-25T00:30:54.631263+0000","last_peered":"2024-04-25T00:30:54.631263+0000","last_clean":"2024-04-25T00:30:54.631263+0000","last_became_active":"2024-04-25T00:30:48.533821+0000","last_became_peered":"2024-04-25T00:30:48.533821+0000","last_unstale":"2024-04-25T00:30:54.631263+0000","last_undegraded":"2024-04-25T00:30:54.631263+0000","last_fullsized":"2024-04-25T00:30:54.631263+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:05:19.327608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.631084+0000","last_change":"2024-04-25T00:30:48.531718+0000","last_active":"2024-04-25T00:30:54.631084+0000","last_peered":"2024-04-25T00:30:54.631084+0000","last_clean":"2024-04-25T00:30:54.631084+0000","last_became_active":"2024-04-25T00:30:48.531291+0000","last_became_peered":"2024-04-25T00:30:48.531291+0000","last_unstale":"2024-04-25T00:30:54.631084+0000","last_undegraded":"2024-04-25T00:30:54.631084+0000","last_fullsized":"2024-04-25T00:30:54.631084+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:56:49.032937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-04-25T00:30:54.232368+0000","last_change":"2024-04-25T00:30:48.537479+0000","last_active":"2024-04-25T00:30:54.232368+0000","last_peered":"2024-04-25T00:30:54.232368+0000","last_clean":"2024-04-25T00:30:54.232368+0000","last_became_active":"2024-04-25T00:30:48.537127+0000","last_became_peered":"2024-04-25T00:30:48.537127+0000","last_unstale":"2024-04-25T00:30:54.232368+0000","last_undegraded":"2024-04-25T00:30:54.232368+0000","last_fullsized":"2024-04-25T00:30:54.232368+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:26:37.289281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":8192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":12},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":4774,"num_objects":33,"num_object_clones":0,"num_object_copies":99,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":33,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":77,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":22,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":159744,"data_stored":16300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":67,"ondisk_log_size":67,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":80,"seq":343597383698,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32140,"kb_used_data":960,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713012,"statfs":{"total":95995035648,"available":95962124288,"internally_reserved":0,"allocated":983040,"data_stored":477215,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1596,"internal_metadata":31914436},"hb_peers":[0,1,2,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":10,"up_from":72,"seq":309237645335,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28580,"kb_used_data":1564,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716572,"statfs":{"total":95995035648,"available":95965769728,"internally_reserved":0,"allocated":1601536,"data_stored":1071619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1600,"internal_metadata":27654592},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":9,"up_from":67,"seq":287762808861,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36320,"kb_used_data":984,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708832,"statfs":{"total":95995035648,"available":95957843968,"internally_reserved":0,"allocated":1007616,"data_stored":482129,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1600,"internal_metadata":36174272},"hb_peers":[0,1,2,3,4,5,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":8,"up_from":59,"seq":253403070497,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32148,"kb_used_data":972,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713004,"statfs":{"total":95995035648,"available":95962116096,"internally_reserved":0,"allocated":995328,"data_stored":477841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":7,"up_from":51,"seq":219043332134,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32728,"kb_used_data":1544,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712424,"statfs":{"total":95995035648,"available":95961522176,"internally_reserved":0,"allocated":1581056,"data_stored":1068165,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495660,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36296,"kb_used_data":960,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708856,"statfs":{"total":95995035648,"available":95957868544,"internally_reserved":0,"allocated":983040,"data_stored":477215,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":36174278},"hb_peers":[0,1,2,3,4,5,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574915,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27988,"kb_used_data":972,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717164,"statfs":{"total":95995035648,"available":95966375936,"internally_reserved":0,"allocated":995328,"data_stored":481057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738442,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28004,"kb_used_data":988,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717148,"statfs":{"total":95995035648,"available":95966359552,"internally_reserved":0,"allocated":1011712,"data_stored":478693,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":18,"seq":77309411391,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27992,"kb_used_data":976,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717160,"statfs":{"total":95995035648,"available":95966371840,"internally_reserved":0,"allocated":999424,"data_stored":481091,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":25,"seq":107374182458,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36308,"kb_used_data":968,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708844,"statfs":{"total":95995035648,"available":95957856256,"internally_reserved":0,"allocated":991232,"data_stored":477319,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":36174279},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":33,"seq":141733920822,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27984,"kb_used_data":964,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717168,"statfs":{"total":95995035648,"available":95966380032,"internally_reserved":0,"allocated":987136,"data_stored":477237,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757297,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28576,"kb_used_data":1552,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716576,"statfs":{"total":95995035648,"available":95965773824,"internally_reserved":0,"allocated":1589248,"data_stored":1067765,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":3842,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":3876,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1204,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4036,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":8192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-25T00:31:18.142 INFO:tasks.cephfs.filesystem.ceph_manager:clean! 2024-04-25T00:31:18.142 INFO:tasks.cephadm:Setting up client nodes... 2024-04-25T00:31:18.142 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-25T00:31:18.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:17 smithi121 ceph-mon[32231]: from='client.14943 -' entity='client.admin' cmd=[{"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:31:18.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:17 smithi121 ceph-mon[32231]: pgmap v306: 129 pgs: 129 active+clean; 581 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 85 B/s rd, 1023 B/s wr, 2 op/s 2024-04-25T00:31:18.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:17 smithi121 ceph-mon[32231]: mgrmap e17: x(active, since 7m), standbys: z, y 2024-04-25T00:31:18.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:17 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2555249369' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:31:18.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:17 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1358938102' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:31:18.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:17 smithi121 ceph-mon[32231]: from='client.14961 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:31:18.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:17 smithi130 ceph-mon[32525]: from='client.14943 -' entity='client.admin' cmd=[{"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:31:18.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:17 smithi130 ceph-mon[32525]: pgmap v306: 129 pgs: 129 active+clean; 581 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 85 B/s rd, 1023 B/s wr, 2 op/s 2024-04-25T00:31:18.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:17 smithi130 ceph-mon[32525]: mgrmap e17: x(active, since 7m), standbys: z, y 2024-04-25T00:31:18.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:17 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2555249369' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:31:18.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:17 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1358938102' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:31:18.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:17 smithi130 ceph-mon[32525]: from='client.14961 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:31:18.425 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:31:19.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:18 smithi032 ceph-mon[29357]: from='client.14967 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:31:19.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:18 smithi121 ceph-mon[32231]: from='client.14967 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:31:19.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:18 smithi130 ceph-mon[32525]: from='client.14967 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:31:20.238 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:19 smithi032 ceph-mon[29357]: pgmap v307: 129 pgs: 129 active+clean; 620 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 5.8 KiB/s wr, 11 op/s 2024-04-25T00:31:20.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:19 smithi121 ceph-mon[32231]: pgmap v307: 129 pgs: 129 active+clean; 620 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 5.8 KiB/s wr, 11 op/s 2024-04-25T00:31:20.362 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.354+0000 7fa8053a0700 1 -- 172.21.15.32:0/1304392252 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa800108080 msgr2=0x7fa80010a470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:20.362 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.354+0000 7fa8053a0700 1 --2- 172.21.15.32:0/1304392252 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa800108080 0x7fa80010a470 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7fa7f800b0a0 tx=0x7fa7f800b3b0 comp rx=0 tx=0).stop 2024-04-25T00:31:20.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.354+0000 7fa8053a0700 1 -- 172.21.15.32:0/1304392252 shutdown_connections 2024-04-25T00:31:20.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.354+0000 7fa8053a0700 1 --2- 172.21.15.32:0/1304392252 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa800108080 0x7fa80010a470 unknown :-1 s=CLOSED pgs=286 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:20.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.354+0000 7fa8053a0700 1 --2- 172.21.15.32:0/1304392252 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa800102e20 0x7fa800105210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:20.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.354+0000 7fa8053a0700 1 --2- 172.21.15.32:0/1304392252 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa800105750 0x7fa800107b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:20.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.354+0000 7fa8053a0700 1 -- 172.21.15.32:0/1304392252 >> 172.21.15.32:0/1304392252 conn(0x7fa8000fc790 msgr2=0x7fa8000febb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:20.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.354+0000 7fa8053a0700 1 -- 172.21.15.32:0/1304392252 shutdown_connections 2024-04-25T00:31:20.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.354+0000 7fa8053a0700 1 -- 172.21.15.32:0/1304392252 wait complete. 2024-04-25T00:31:20.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.355+0000 7fa8053a0700 1 Processor -- start 2024-04-25T00:31:20.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.355+0000 7fa8053a0700 1 -- start start 2024-04-25T00:31:20.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.355+0000 7fa8053a0700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa800102e20 0x7fa8001982b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:20.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.355+0000 7fa8053a0700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa800105750 0x7fa8001987f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:20.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.355+0000 7fa8053a0700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa800108080 0x7fa80019f820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:20.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.355+0000 7fa8053a0700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fa80010ded0 con 0x7fa800108080 2024-04-25T00:31:20.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.355+0000 7fa8053a0700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fa80010dd50 con 0x7fa800102e20 2024-04-25T00:31:20.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.355+0000 7fa8053a0700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fa80010e050 con 0x7fa800105750 2024-04-25T00:31:20.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.355+0000 7fa7ff7fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa800108080 0x7fa80019f820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:20.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.355+0000 7fa7ff7fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa800108080 0x7fa80019f820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:58048/0 (socket says 172.21.15.32:58048) 2024-04-25T00:31:20.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.355+0000 7fa7ff7fe700 1 -- 172.21.15.32:0/183737963 learned_addr learned my addr 172.21.15.32:0/183737963 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:20.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.356+0000 7fa7fe7fc700 1 --2- 172.21.15.32:0/183737963 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa800105750 0x7fa8001987f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:20.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.356+0000 7fa7feffd700 1 --2- 172.21.15.32:0/183737963 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa800102e20 0x7fa8001982b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:20.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.356+0000 7fa7ff7fe700 1 -- 172.21.15.32:0/183737963 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa800105750 msgr2=0x7fa8001987f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:20.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.356+0000 7fa7ff7fe700 1 --2- 172.21.15.32:0/183737963 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa800105750 0x7fa8001987f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:20.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.356+0000 7fa7ff7fe700 1 -- 172.21.15.32:0/183737963 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa800102e20 msgr2=0x7fa8001982b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:20.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.356+0000 7fa7ff7fe700 1 --2- 172.21.15.32:0/183737963 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa800102e20 0x7fa8001982b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:20.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.356+0000 7fa7ff7fe700 1 -- 172.21.15.32:0/183737963 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa80019fec0 con 0x7fa800108080 2024-04-25T00:31:20.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.356+0000 7fa7fe7fc700 1 --2- 172.21.15.32:0/183737963 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa800105750 0x7fa8001987f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:31:20.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.356+0000 7fa7ff7fe700 1 --2- 172.21.15.32:0/183737963 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa800108080 0x7fa80019f820 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7fa7f800d010 tx=0x7fa7f8004e80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:20.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.356+0000 7fa7e7fff700 1 -- 172.21.15.32:0/183737963 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa7f8021070 con 0x7fa800108080 2024-04-25T00:31:20.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.356+0000 7fa8053a0700 1 -- 172.21.15.32:0/183737963 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa8001a0150 con 0x7fa800108080 2024-04-25T00:31:20.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.356+0000 7fa7e7fff700 1 -- 172.21.15.32:0/183737963 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa7f8004020 con 0x7fa800108080 2024-04-25T00:31:20.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.356+0000 7fa7e7fff700 1 -- 172.21.15.32:0/183737963 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa7f801d6b0 con 0x7fa800108080 2024-04-25T00:31:20.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.356+0000 7fa8053a0700 1 -- 172.21.15.32:0/183737963 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa8001a0650 con 0x7fa800108080 2024-04-25T00:31:20.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.357+0000 7fa8053a0700 1 -- 172.21.15.32:0/183737963 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa8000507a0 con 0x7fa800108080 2024-04-25T00:31:20.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.358+0000 7fa7e7fff700 1 -- 172.21.15.32:0/183737963 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7fa7f80041d0 con 0x7fa800108080 2024-04-25T00:31:20.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.359+0000 7fa7e7fff700 1 --2- 172.21.15.32:0/183737963 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa7e80a2d60 0x7fa7e80a5220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:20.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.359+0000 7fa7feffd700 1 --2- 172.21.15.32:0/183737963 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa7e80a2d60 0x7fa7e80a5220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:20.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.359+0000 7fa7e7fff700 1 -- 172.21.15.32:0/183737963 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7fa7f800f020 con 0x7fa800108080 2024-04-25T00:31:20.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.359+0000 7fa7feffd700 1 --2- 172.21.15.32:0/183737963 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa7e80a2d60 0x7fa7e80a5220 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7fa7f0005950 tx=0x7fa7f00058e0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:20.366 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.363+0000 7fa7e7fff700 1 -- 172.21.15.32:0/183737963 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fa7f802a020 con 0x7fa800108080 2024-04-25T00:31:20.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:19 smithi130 ceph-mon[32525]: pgmap v307: 129 pgs: 129 active+clean; 620 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 5.8 KiB/s wr, 11 op/s 2024-04-25T00:31:20.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.658+0000 7fa8053a0700 1 -- 172.21.15.32:0/183737963 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]} v 0) -- 0x7fa80002df70 con 0x7fa800108080 2024-04-25T00:31:20.661 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.661+0000 7fa7e7fff700 1 -- 172.21.15.32:0/183737963 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]=0 v32) ==== 170+0+59 (secure 0 0 0) 0x7fa80002df70 con 0x7fa800108080 2024-04-25T00:31:20.661 INFO:teuthology.orchestra.run.smithi032.stdout:[client.0] 2024-04-25T00:31:20.661 INFO:teuthology.orchestra.run.smithi032.stdout: key = AQBYpClmOpZYJxAA2D7QQnuu1rUMfZ5UlIbn9Q== 2024-04-25T00:31:20.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.663+0000 7fa8053a0700 1 -- 172.21.15.32:0/183737963 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa7e80a2d60 msgr2=0x7fa7e80a5220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:20.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.663+0000 7fa8053a0700 1 --2- 172.21.15.32:0/183737963 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa7e80a2d60 0x7fa7e80a5220 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7fa7f0005950 tx=0x7fa7f00058e0 comp rx=0 tx=0).stop 2024-04-25T00:31:20.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.663+0000 7fa8053a0700 1 -- 172.21.15.32:0/183737963 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa800108080 msgr2=0x7fa80019f820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:20.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.663+0000 7fa8053a0700 1 --2- 172.21.15.32:0/183737963 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa800108080 0x7fa80019f820 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7fa7f800d010 tx=0x7fa7f8004e80 comp rx=0 tx=0).stop 2024-04-25T00:31:20.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.663+0000 7fa8053a0700 1 -- 172.21.15.32:0/183737963 shutdown_connections 2024-04-25T00:31:20.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.663+0000 7fa8053a0700 1 --2- 172.21.15.32:0/183737963 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fa7e80a2d60 0x7fa7e80a5220 unknown :-1 s=CLOSED pgs=191 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:20.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.663+0000 7fa8053a0700 1 --2- 172.21.15.32:0/183737963 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa800108080 0x7fa80019f820 unknown :-1 s=CLOSED pgs=287 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:20.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.663+0000 7fa8053a0700 1 --2- 172.21.15.32:0/183737963 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa800102e20 0x7fa8001982b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:20.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.663+0000 7fa8053a0700 1 --2- 172.21.15.32:0/183737963 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa800105750 0x7fa8001987f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:20.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.663+0000 7fa8053a0700 1 -- 172.21.15.32:0/183737963 >> 172.21.15.32:0/183737963 conn(0x7fa8000fc790 msgr2=0x7fa800103760 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:20.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.663+0000 7fa8053a0700 1 -- 172.21.15.32:0/183737963 shutdown_connections 2024-04-25T00:31:20.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:20.663+0000 7fa8053a0700 1 -- 172.21.15.32:0/183737963 wait complete. 2024-04-25T00:31:21.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:20 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/183737963' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-25T00:31:21.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:20 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/183737963' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-25T00:31:21.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:20 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/183737963' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-25T00:31:21.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:20 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/183737963' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-25T00:31:21.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:20 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/183737963' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-25T00:31:21.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:20 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/183737963' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-25T00:31:21.571 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-04-25T00:31:21.571 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-25T00:31:21.571 DEBUG:teuthology.orchestra.run.smithi032:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-25T00:31:21.609 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-25T00:31:21.609 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-25T00:31:21.609 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-25T00:31:21.900 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:31:22.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:22 smithi032 ceph-mon[29357]: pgmap v308: 129 pgs: 129 active+clean; 620 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 5.7 KiB/s wr, 11 op/s 2024-04-25T00:31:23.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:22 smithi121 ceph-mon[32231]: pgmap v308: 129 pgs: 129 active+clean; 620 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 5.7 KiB/s wr, 11 op/s 2024-04-25T00:31:23.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:22 smithi130 ceph-mon[32525]: pgmap v308: 129 pgs: 129 active+clean; 620 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 5.7 KiB/s wr, 11 op/s 2024-04-25T00:31:23.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.887+0000 7fbb2fa5f700 1 -- 172.21.15.32:0/3147583084 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbb28107570 msgr2=0x7fbb2810f140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:23.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.887+0000 7fbb2fa5f700 1 --2- 172.21.15.32:0/3147583084 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbb28107570 0x7fbb2810f140 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7fbb2000b0a0 tx=0x7fbb2000b3b0 comp rx=0 tx=0).stop 2024-04-25T00:31:23.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.887+0000 7fbb2fa5f700 1 -- 172.21.15.32:0/3147583084 shutdown_connections 2024-04-25T00:31:23.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.887+0000 7fbb2fa5f700 1 --2- 172.21.15.32:0/3147583084 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbb28107570 0x7fbb2810f140 unknown :-1 s=CLOSED pgs=288 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:23.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.887+0000 7fbb2fa5f700 1 --2- 172.21.15.32:0/3147583084 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbb281024e0 0x7fbb281028c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:23.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.887+0000 7fbb2fa5f700 1 --2- 172.21.15.32:0/3147583084 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbb28102e90 0x7fbb28106e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:23.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.887+0000 7fbb2fa5f700 1 -- 172.21.15.32:0/3147583084 >> 172.21.15.32:0/3147583084 conn(0x7fbb28079390 msgr2=0x7fbb280797a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:23.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.887+0000 7fbb2fa5f700 1 -- 172.21.15.32:0/3147583084 shutdown_connections 2024-04-25T00:31:23.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.887+0000 7fbb2fa5f700 1 -- 172.21.15.32:0/3147583084 wait complete. 2024-04-25T00:31:23.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.888+0000 7fbb2fa5f700 1 Processor -- start 2024-04-25T00:31:23.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.888+0000 7fbb2fa5f700 1 -- start start 2024-04-25T00:31:23.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.888+0000 7fbb2fa5f700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbb281024e0 0x7fbb281a0860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:23.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.888+0000 7fbb2fa5f700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbb28102e90 0x7fbb281a0da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:23.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.888+0000 7fbb2fa5f700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbb28107570 0x7fbb2819aa20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:23.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.888+0000 7fbb2fa5f700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fbb281125b0 con 0x7fbb281024e0 2024-04-25T00:31:23.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.888+0000 7fbb2fa5f700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fbb28112430 con 0x7fbb28107570 2024-04-25T00:31:23.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.888+0000 7fbb2fa5f700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fbb28112730 con 0x7fbb28102e90 2024-04-25T00:31:23.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.888+0000 7fbb2d7fb700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbb281024e0 0x7fbb281a0860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:23.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.888+0000 7fbb2d7fb700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbb281024e0 0x7fbb281a0860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:34088/0 (socket says 172.21.15.32:34088) 2024-04-25T00:31:23.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.888+0000 7fbb2d7fb700 1 -- 172.21.15.32:0/3887473021 learned_addr learned my addr 172.21.15.32:0/3887473021 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:23.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.888+0000 7fbb2cffa700 1 --2- 172.21.15.32:0/3887473021 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbb28102e90 0x7fbb281a0da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:23.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.888+0000 7fbb2dffc700 1 --2- 172.21.15.32:0/3887473021 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbb28107570 0x7fbb2819aa20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:23.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.889+0000 7fbb2d7fb700 1 -- 172.21.15.32:0/3887473021 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbb28102e90 msgr2=0x7fbb281a0da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:23.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.889+0000 7fbb2d7fb700 1 --2- 172.21.15.32:0/3887473021 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbb28102e90 0x7fbb281a0da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:23.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.889+0000 7fbb2d7fb700 1 -- 172.21.15.32:0/3887473021 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbb28107570 msgr2=0x7fbb2819aa20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:23.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.889+0000 7fbb2d7fb700 1 --2- 172.21.15.32:0/3887473021 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbb28107570 0x7fbb2819aa20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:23.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.889+0000 7fbb2d7fb700 1 -- 172.21.15.32:0/3887473021 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbb2819b180 con 0x7fbb281024e0 2024-04-25T00:31:23.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.889+0000 7fbb2cffa700 1 --2- 172.21.15.32:0/3887473021 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbb28102e90 0x7fbb281a0da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:31:23.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.889+0000 7fbb2d7fb700 1 --2- 172.21.15.32:0/3887473021 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbb281024e0 0x7fbb281a0860 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7fbb2400b530 tx=0x7fbb2400b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:23.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.889+0000 7fbb1e7fc700 1 -- 172.21.15.32:0/3887473021 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbb24015020 con 0x7fbb281024e0 2024-04-25T00:31:23.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.889+0000 7fbb1e7fc700 1 -- 172.21.15.32:0/3887473021 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbb24011d10 con 0x7fbb281024e0 2024-04-25T00:31:23.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.889+0000 7fbb2fa5f700 1 -- 172.21.15.32:0/3887473021 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbb2819b470 con 0x7fbb281024e0 2024-04-25T00:31:23.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.889+0000 7fbb1e7fc700 1 -- 172.21.15.32:0/3887473021 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbb2400f2e0 con 0x7fbb281024e0 2024-04-25T00:31:23.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.889+0000 7fbb2fa5f700 1 -- 172.21.15.32:0/3887473021 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fbb281a7730 con 0x7fbb281024e0 2024-04-25T00:31:23.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.890+0000 7fbb2fa5f700 1 -- 172.21.15.32:0/3887473021 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fbb28068f20 con 0x7fbb281024e0 2024-04-25T00:31:23.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.891+0000 7fbb1e7fc700 1 -- 172.21.15.32:0/3887473021 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7fbb24018020 con 0x7fbb281024e0 2024-04-25T00:31:23.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.892+0000 7fbb1e7fc700 1 --2- 172.21.15.32:0/3887473021 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fbb140a2d30 0x7fbb140a51f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:23.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.892+0000 7fbb2cffa700 1 --2- 172.21.15.32:0/3887473021 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fbb140a2d30 0x7fbb140a51f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:23.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.892+0000 7fbb1e7fc700 1 -- 172.21.15.32:0/3887473021 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7fbb240cebc0 con 0x7fbb281024e0 2024-04-25T00:31:23.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.892+0000 7fbb2cffa700 1 --2- 172.21.15.32:0/3887473021 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fbb140a2d30 0x7fbb140a51f0 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7fbb18005fd0 tx=0x7fbb18005e30 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:23.897 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:23.896+0000 7fbb1e7fc700 1 -- 172.21.15.32:0/3887473021 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fbb2401f020 con 0x7fbb281024e0 2024-04-25T00:31:24.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:24.161+0000 7fbb2fa5f700 1 -- 172.21.15.32:0/3887473021 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "mgr dump", "format": "json"} v 0) -- 0x7fbb281a79c0 con 0x7fbb281024e0 2024-04-25T00:31:24.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:24.166+0000 7fbb1e7fc700 1 -- 172.21.15.32:0/3887473021 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json"}]=0 v17) ==== 74+0+263198 (secure 0 0 0) 0x7fbb2409b890 con 0x7fbb281024e0 2024-04-25T00:31:24.170 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:31:24.172 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:24.170+0000 7fbb2fa5f700 1 -- 172.21.15.32:0/3887473021 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fbb140a2d30 msgr2=0x7fbb140a51f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:24.172 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:24.170+0000 7fbb2fa5f700 1 --2- 172.21.15.32:0/3887473021 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fbb140a2d30 0x7fbb140a51f0 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7fbb18005fd0 tx=0x7fbb18005e30 comp rx=0 tx=0).stop 2024-04-25T00:31:24.172 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:24.171+0000 7fbb2fa5f700 1 -- 172.21.15.32:0/3887473021 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbb281024e0 msgr2=0x7fbb281a0860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:24.172 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:24.171+0000 7fbb2fa5f700 1 --2- 172.21.15.32:0/3887473021 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbb281024e0 0x7fbb281a0860 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7fbb2400b530 tx=0x7fbb2400b840 comp rx=0 tx=0).stop 2024-04-25T00:31:24.172 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:24.171+0000 7fbb2fa5f700 1 -- 172.21.15.32:0/3887473021 shutdown_connections 2024-04-25T00:31:24.172 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:24.171+0000 7fbb2fa5f700 1 --2- 172.21.15.32:0/3887473021 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fbb140a2d30 0x7fbb140a51f0 unknown :-1 s=CLOSED pgs=192 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:24.172 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:24.171+0000 7fbb2fa5f700 1 --2- 172.21.15.32:0/3887473021 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbb28107570 0x7fbb2819aa20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:24.172 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:24.171+0000 7fbb2fa5f700 1 --2- 172.21.15.32:0/3887473021 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbb281024e0 0x7fbb281a0860 unknown :-1 s=CLOSED pgs=289 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:24.172 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:24.171+0000 7fbb2fa5f700 1 --2- 172.21.15.32:0/3887473021 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbb28102e90 0x7fbb281a0da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:24.172 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:24.171+0000 7fbb2fa5f700 1 -- 172.21.15.32:0/3887473021 >> 172.21.15.32:0/3887473021 conn(0x7fbb28079390 msgr2=0x7fbb28100430 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:24.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:24.171+0000 7fbb2fa5f700 1 -- 172.21.15.32:0/3887473021 shutdown_connections 2024-04-25T00:31:24.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:24.171+0000 7fbb2fa5f700 1 -- 172.21.15.32:0/3887473021 wait complete. 2024-04-25T00:31:24.945 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:24 smithi032 ceph-mon[29357]: pgmap v309: 129 pgs: 129 active+clean; 625 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 6.3 KiB/s wr, 14 op/s 2024-04-25T00:31:24.945 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:24 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3887473021' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-25T00:31:24.976 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":17,"flags":0,"active_gid":14148,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6800","nonce":585810931},{"type":"v1","addr":"172.21.15.32:6801","nonce":585810931}]},"active_addr":"172.21.15.32:6801/585810931","active_change":"2024-04-25T00:23:42.776554+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":24109,"name":"z","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.1","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]},{"gid":24113,"name":"y","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.1","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.1","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.32:8443/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":3,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.32:0","nonce":1096527590}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.32:0","nonce":3915353181}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.32:0","nonce":1767429535}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.32:0","nonce":1694990206}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.32:0","nonce":1089850969}]}]} 2024-04-25T00:31:24.981 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-25T00:31:24.981 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-25T00:31:24.981 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-25T00:31:25.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:24 smithi121 ceph-mon[32231]: pgmap v309: 129 pgs: 129 active+clean; 625 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 6.3 KiB/s wr, 14 op/s 2024-04-25T00:31:25.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:24 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3887473021' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-25T00:31:25.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:24 smithi130 ceph-mon[32525]: pgmap v309: 129 pgs: 129 active+clean; 625 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 6.3 KiB/s wr, 14 op/s 2024-04-25T00:31:25.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:24 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3887473021' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-25T00:31:25.234 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:31:26.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:26 smithi032 ceph-mon[29357]: pgmap v310: 129 pgs: 129 active+clean; 625 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 7.6 KiB/s rd, 6.3 KiB/s wr, 14 op/s 2024-04-25T00:31:27.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.029+0000 7f05dc099700 1 -- 172.21.15.32:0/1173002983 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f05d4074460 msgr2=0x7f05d4074840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:27.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.029+0000 7f05dc099700 1 --2- 172.21.15.32:0/1173002983 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f05d4074460 0x7f05d4074840 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7f05d0009990 tx=0x7f05d0009ca0 comp rx=0 tx=0).stop 2024-04-25T00:31:27.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.029+0000 7f05dc099700 1 -- 172.21.15.32:0/1173002983 shutdown_connections 2024-04-25T00:31:27.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.029+0000 7f05dc099700 1 --2- 172.21.15.32:0/1173002983 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f05d4113400 0x7f05d41157f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:27.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.029+0000 7f05dc099700 1 --2- 172.21.15.32:0/1173002983 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f05d4074460 0x7f05d4074840 unknown :-1 s=CLOSED pgs=290 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:27.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.029+0000 7f05dc099700 1 --2- 172.21.15.32:0/1173002983 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f05d4074d80 0x7f05d4112ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:27.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.029+0000 7f05dc099700 1 -- 172.21.15.32:0/1173002983 >> 172.21.15.32:0/1173002983 conn(0x7f05d4079390 msgr2=0x7f05d40797a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:27.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.029+0000 7f05dc099700 1 -- 172.21.15.32:0/1173002983 shutdown_connections 2024-04-25T00:31:27.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.029+0000 7f05dc099700 1 -- 172.21.15.32:0/1173002983 wait complete. 2024-04-25T00:31:27.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.029+0000 7f05dc099700 1 Processor -- start 2024-04-25T00:31:27.031 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.029+0000 7f05dc099700 1 -- start start 2024-04-25T00:31:27.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.030+0000 7f05dc099700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f05d4074460 0x7f05d419efc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:27.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.030+0000 7f05dc099700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f05d4074d80 0x7f05d419f500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:27.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.030+0000 7f05dc099700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f05d4113400 0x7f05d41a3840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:27.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.030+0000 7f05dc099700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f05d4118b50 con 0x7f05d4113400 2024-04-25T00:31:27.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.030+0000 7f05dc099700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f05d41189d0 con 0x7f05d4074d80 2024-04-25T00:31:27.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.030+0000 7f05dc099700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f05d4118cd0 con 0x7f05d4074460 2024-04-25T00:31:27.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.030+0000 7f05da636700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f05d4113400 0x7f05d41a3840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:27.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.030+0000 7f05d9e35700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f05d4074460 0x7f05d419efc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:27.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.030+0000 7f05d9e35700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f05d4074460 0x7f05d419efc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:59894/0 (socket says 172.21.15.32:59894) 2024-04-25T00:31:27.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.030+0000 7f05da636700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f05d4113400 0x7f05d41a3840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:34104/0 (socket says 172.21.15.32:34104) 2024-04-25T00:31:27.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.030+0000 7f05d9e35700 1 -- 172.21.15.32:0/3614722292 learned_addr learned my addr 172.21.15.32:0/3614722292 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:27.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.030+0000 7f05d9634700 1 --2- 172.21.15.32:0/3614722292 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f05d4074d80 0x7f05d419f500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:27.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.030+0000 7f05da636700 1 -- 172.21.15.32:0/3614722292 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f05d4074460 msgr2=0x7f05d419efc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:27.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.030+0000 7f05da636700 1 --2- 172.21.15.32:0/3614722292 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f05d4074460 0x7f05d419efc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:27.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.030+0000 7f05da636700 1 -- 172.21.15.32:0/3614722292 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f05d4074d80 msgr2=0x7f05d419f500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:27.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.030+0000 7f05da636700 1 --2- 172.21.15.32:0/3614722292 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f05d4074d80 0x7f05d419f500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:27.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.030+0000 7f05da636700 1 -- 172.21.15.32:0/3614722292 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f05d41a3f80 con 0x7f05d4113400 2024-04-25T00:31:27.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.030+0000 7f05da636700 1 --2- 172.21.15.32:0/3614722292 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f05d4113400 0x7f05d41a3840 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7f05cc007ee0 tx=0x7f05cc00d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:27.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.031+0000 7f05caffd700 1 -- 172.21.15.32:0/3614722292 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f05cc004db0 con 0x7f05d4113400 2024-04-25T00:31:27.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.031+0000 7f05caffd700 1 -- 172.21.15.32:0/3614722292 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f05cc009e20 con 0x7f05d4113400 2024-04-25T00:31:27.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.031+0000 7f05dc099700 1 -- 172.21.15.32:0/3614722292 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f05d41a4270 con 0x7f05d4113400 2024-04-25T00:31:27.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.031+0000 7f05caffd700 1 -- 172.21.15.32:0/3614722292 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f05cc015400 con 0x7f05d4113400 2024-04-25T00:31:27.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.031+0000 7f05dc099700 1 -- 172.21.15.32:0/3614722292 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f05d41abaf0 con 0x7f05d4113400 2024-04-25T00:31:27.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.032+0000 7f05dc099700 1 -- 172.21.15.32:0/3614722292 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f05d4068f20 con 0x7f05d4113400 2024-04-25T00:31:27.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.033+0000 7f05caffd700 1 -- 172.21.15.32:0/3614722292 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f05cc00db20 con 0x7f05d4113400 2024-04-25T00:31:27.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.033+0000 7f05caffd700 1 --2- 172.21.15.32:0/3614722292 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f05c00a2cb0 0x7f05c00a5170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:27.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.034+0000 7f05d9e35700 1 --2- 172.21.15.32:0/3614722292 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f05c00a2cb0 0x7f05c00a5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:27.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.034+0000 7f05caffd700 1 -- 172.21.15.32:0/3614722292 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f05cc0cecc0 con 0x7f05d4113400 2024-04-25T00:31:27.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.034+0000 7f05d9e35700 1 --2- 172.21.15.32:0/3614722292 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f05c00a2cb0 0x7f05c00a5170 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f05d000b5c0 tx=0x7f05d0005be0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:27.038 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.038+0000 7f05caffd700 1 -- 172.21.15.32:0/3614722292 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f05cc010070 con 0x7f05d4113400 2024-04-25T00:31:27.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:26 smithi121 ceph-mon[32231]: pgmap v310: 129 pgs: 129 active+clean; 625 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 7.6 KiB/s rd, 6.3 KiB/s wr, 14 op/s 2024-04-25T00:31:27.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:26 smithi130 ceph-mon[32525]: pgmap v310: 129 pgs: 129 active+clean; 625 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 7.6 KiB/s rd, 6.3 KiB/s wr, 14 op/s 2024-04-25T00:31:27.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.246+0000 7f05dc099700 1 -- 172.21.15.32:0/3614722292 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f05d41abcc0 con 0x7f05d4113400 2024-04-25T00:31:27.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.247+0000 7f05caffd700 1 -- 172.21.15.32:0/3614722292 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v89) ==== 74+0+24128 (secure 0 0 0) 0x7f05cc09b880 con 0x7f05d4113400 2024-04-25T00:31:27.248 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:31:27.248 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":89,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:30:53.134605+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-25T00:30:46.239556+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-25T00:30:47.257182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2024-04-25T00:30:51.193975+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":83,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-num-failure-domains":"0","crush-osds-per-failure-domain":"0","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:31:27.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.249+0000 7f05dc099700 1 -- 172.21.15.32:0/3614722292 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f05c00a2cb0 msgr2=0x7f05c00a5170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:27.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.249+0000 7f05dc099700 1 --2- 172.21.15.32:0/3614722292 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f05c00a2cb0 0x7f05c00a5170 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f05d000b5c0 tx=0x7f05d0005be0 comp rx=0 tx=0).stop 2024-04-25T00:31:27.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.249+0000 7f05dc099700 1 -- 172.21.15.32:0/3614722292 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f05d4113400 msgr2=0x7f05d41a3840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:27.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.249+0000 7f05dc099700 1 --2- 172.21.15.32:0/3614722292 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f05d4113400 0x7f05d41a3840 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7f05cc007ee0 tx=0x7f05cc00d3b0 comp rx=0 tx=0).stop 2024-04-25T00:31:27.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.249+0000 7f05dc099700 1 -- 172.21.15.32:0/3614722292 shutdown_connections 2024-04-25T00:31:27.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.249+0000 7f05dc099700 1 --2- 172.21.15.32:0/3614722292 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f05c00a2cb0 0x7f05c00a5170 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:27.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.250+0000 7f05dc099700 1 --2- 172.21.15.32:0/3614722292 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f05d4113400 0x7f05d41a3840 unknown :-1 s=CLOSED pgs=291 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:27.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.250+0000 7f05dc099700 1 --2- 172.21.15.32:0/3614722292 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f05d4074460 0x7f05d419efc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:27.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.250+0000 7f05dc099700 1 --2- 172.21.15.32:0/3614722292 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f05d4074d80 0x7f05d419f500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:27.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.250+0000 7f05dc099700 1 -- 172.21.15.32:0/3614722292 >> 172.21.15.32:0/3614722292 conn(0x7f05d4079390 msgr2=0x7f05d41121c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:27.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.250+0000 7f05dc099700 1 -- 172.21.15.32:0/3614722292 shutdown_connections 2024-04-25T00:31:27.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:27.250+0000 7f05dc099700 1 -- 172.21.15.32:0/3614722292 wait complete. 2024-04-25T00:31:27.887 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:27 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3614722292' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:31:27.918 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-25T00:31:27.919 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-25T00:31:28.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:27 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3614722292' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:31:28.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:27 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3614722292' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:31:28.170 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:31:28.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:28 smithi032 ceph-mon[29357]: pgmap v311: 129 pgs: 129 active+clean; 625 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 7.6 KiB/s rd, 5.6 KiB/s wr, 13 op/s 2024-04-25T00:31:29.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:28 smithi121 ceph-mon[32231]: pgmap v311: 129 pgs: 129 active+clean; 625 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 7.6 KiB/s rd, 5.6 KiB/s wr, 13 op/s 2024-04-25T00:31:29.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:28 smithi130 ceph-mon[32525]: pgmap v311: 129 pgs: 129 active+clean; 625 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 7.6 KiB/s rd, 5.6 KiB/s wr, 13 op/s 2024-04-25T00:31:30.020 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.013+0000 7f9243190700 1 -- 172.21.15.32:0/3218739739 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f923c06ac40 msgr2=0x7f923c06b020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:30.020 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.013+0000 7f9243190700 1 --2- 172.21.15.32:0/3218739739 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f923c06ac40 0x7f923c06b020 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7f922c0099c0 tx=0x7f922c009cd0 comp rx=0 tx=0).stop 2024-04-25T00:31:30.020 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.013+0000 7f9243190700 1 -- 172.21.15.32:0/3218739739 shutdown_connections 2024-04-25T00:31:30.020 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.013+0000 7f9243190700 1 --2- 172.21.15.32:0/3218739739 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f923c107590 0x7f923c10f160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:30.020 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.013+0000 7f9243190700 1 --2- 172.21.15.32:0/3218739739 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f923c06ac40 0x7f923c06b020 unknown :-1 s=CLOSED pgs=292 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:30.020 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.013+0000 7f9243190700 1 --2- 172.21.15.32:0/3218739739 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f923c06b5f0 0x7f923c106eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:30.020 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.013+0000 7f9243190700 1 -- 172.21.15.32:0/3218739739 >> 172.21.15.32:0/3218739739 conn(0x7f923c0781d0 msgr2=0x7f923c0785e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:30.020 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.013+0000 7f9243190700 1 -- 172.21.15.32:0/3218739739 shutdown_connections 2024-04-25T00:31:30.020 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.013+0000 7f9243190700 1 -- 172.21.15.32:0/3218739739 wait complete. 2024-04-25T00:31:30.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.014+0000 7f9243190700 1 Processor -- start 2024-04-25T00:31:30.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.014+0000 7f9243190700 1 -- start start 2024-04-25T00:31:30.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.014+0000 7f9243190700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f923c06ac40 0x7f923c1011a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:30.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.014+0000 7f9243190700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f923c06b5f0 0x7f923c1016e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:30.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.014+0000 7f9243190700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f923c107590 0x7f923c101c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:30.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.014+0000 7f9243190700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f923c112670 con 0x7f923c06b5f0 2024-04-25T00:31:30.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.014+0000 7f9243190700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f923c1124f0 con 0x7f923c06ac40 2024-04-25T00:31:30.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.014+0000 7f9243190700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f923c1127f0 con 0x7f923c107590 2024-04-25T00:31:30.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.014+0000 7f923bfff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f923c06b5f0 0x7f923c1016e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:30.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.014+0000 7f9240f2c700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f923c06ac40 0x7f923c1011a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:30.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.014+0000 7f923bfff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f923c06b5f0 0x7f923c1016e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:34108/0 (socket says 172.21.15.32:34108) 2024-04-25T00:31:30.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.014+0000 7f923bfff700 1 -- 172.21.15.32:0/606097616 learned_addr learned my addr 172.21.15.32:0/606097616 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:30.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.014+0000 7f924172d700 1 --2- 172.21.15.32:0/606097616 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f923c107590 0x7f923c101c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:30.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.014+0000 7f9240f2c700 1 -- 172.21.15.32:0/606097616 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f923c107590 msgr2=0x7f923c101c20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:30.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.014+0000 7f9240f2c700 1 --2- 172.21.15.32:0/606097616 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f923c107590 0x7f923c101c20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:30.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.014+0000 7f9240f2c700 1 -- 172.21.15.32:0/606097616 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f923c06b5f0 msgr2=0x7f923c1016e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:30.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.014+0000 7f9240f2c700 1 --2- 172.21.15.32:0/606097616 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f923c06b5f0 0x7f923c1016e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:30.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.014+0000 7f9240f2c700 1 -- 172.21.15.32:0/606097616 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f923c1a7090 con 0x7f923c06ac40 2024-04-25T00:31:30.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.015+0000 7f9240f2c700 1 --2- 172.21.15.32:0/606097616 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f923c06ac40 0x7f923c1011a0 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f922c009fd0 tx=0x7f922c005f10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:30.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.015+0000 7f9239ffb700 1 -- 172.21.15.32:0/606097616 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f922c004830 con 0x7f923c06ac40 2024-04-25T00:31:30.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.015+0000 7f9243190700 1 -- 172.21.15.32:0/606097616 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f923c1a7320 con 0x7f923c06ac40 2024-04-25T00:31:30.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.015+0000 7f9239ffb700 1 -- 172.21.15.32:0/606097616 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f922c005020 con 0x7f923c06ac40 2024-04-25T00:31:30.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.015+0000 7f9239ffb700 1 -- 172.21.15.32:0/606097616 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f922c003f90 con 0x7f923c06ac40 2024-04-25T00:31:30.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.015+0000 7f9243190700 1 -- 172.21.15.32:0/606097616 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f923c1a7770 con 0x7f923c06ac40 2024-04-25T00:31:30.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.016+0000 7f9243190700 1 -- 172.21.15.32:0/606097616 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f923c068f20 con 0x7f923c06ac40 2024-04-25T00:31:30.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.018+0000 7f9239ffb700 1 -- 172.21.15.32:0/606097616 <== mon.2 v2:172.21.15.121:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f922c029060 con 0x7f923c06ac40 2024-04-25T00:31:30.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.018+0000 7f9239ffb700 1 --2- 172.21.15.32:0/606097616 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f92240a2cb0 0x7f92240a5170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:30.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.018+0000 7f923bfff700 1 --2- 172.21.15.32:0/606097616 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f92240a2cb0 0x7f92240a5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:30.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.018+0000 7f9239ffb700 1 -- 172.21.15.32:0/606097616 <== mon.2 v2:172.21.15.121:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f922c0cff80 con 0x7f923c06ac40 2024-04-25T00:31:30.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.019+0000 7f923bfff700 1 --2- 172.21.15.32:0/606097616 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f92240a2cb0 0x7f92240a5170 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f923c100d00 tx=0x7f9230008040 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:30.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.022+0000 7f9239ffb700 1 -- 172.21.15.32:0/606097616 <== mon.2 v2:172.21.15.121:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f922c013070 con 0x7f923c06ac40 2024-04-25T00:31:30.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.229+0000 7f9243190700 1 -- 172.21.15.32:0/606097616 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f923c02dcf0 con 0x7f923c06ac40 2024-04-25T00:31:30.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.231+0000 7f9239ffb700 1 -- 172.21.15.32:0/606097616 <== mon.2 v2:172.21.15.121:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v89) ==== 74+0+24128 (secure 0 0 0) 0x7f922c09cb70 con 0x7f923c06ac40 2024-04-25T00:31:30.232 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:31:30.232 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":89,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:30:53.134605+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-25T00:30:46.239556+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-25T00:30:47.257182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2024-04-25T00:30:51.193975+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":83,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-num-failure-domains":"0","crush-osds-per-failure-domain":"0","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:31:30.233 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.233+0000 7f9243190700 1 -- 172.21.15.32:0/606097616 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f92240a2cb0 msgr2=0x7f92240a5170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:30.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.233+0000 7f9243190700 1 --2- 172.21.15.32:0/606097616 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f92240a2cb0 0x7f92240a5170 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f923c100d00 tx=0x7f9230008040 comp rx=0 tx=0).stop 2024-04-25T00:31:30.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.233+0000 7f9243190700 1 -- 172.21.15.32:0/606097616 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f923c06ac40 msgr2=0x7f923c1011a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:30.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.233+0000 7f9243190700 1 --2- 172.21.15.32:0/606097616 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f923c06ac40 0x7f923c1011a0 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f922c009fd0 tx=0x7f922c005f10 comp rx=0 tx=0).stop 2024-04-25T00:31:30.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.233+0000 7f9243190700 1 -- 172.21.15.32:0/606097616 shutdown_connections 2024-04-25T00:31:30.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.233+0000 7f9243190700 1 --2- 172.21.15.32:0/606097616 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f92240a2cb0 0x7f92240a5170 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:30.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.233+0000 7f9243190700 1 --2- 172.21.15.32:0/606097616 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f923c107590 0x7f923c101c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:30.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.233+0000 7f9243190700 1 --2- 172.21.15.32:0/606097616 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f923c06ac40 0x7f923c1011a0 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:30.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.233+0000 7f9243190700 1 --2- 172.21.15.32:0/606097616 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f923c06b5f0 0x7f923c1016e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:30.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.233+0000 7f9243190700 1 -- 172.21.15.32:0/606097616 >> 172.21.15.32:0/606097616 conn(0x7f923c0781d0 msgr2=0x7f923c106720 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:30.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.233+0000 7f9243190700 1 -- 172.21.15.32:0/606097616 shutdown_connections 2024-04-25T00:31:30.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:30.233+0000 7f9243190700 1 -- 172.21.15.32:0/606097616 wait complete. 2024-04-25T00:31:30.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:30 smithi032 ceph-mon[29357]: pgmap v312: 129 pgs: 129 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 10 KiB/s rd, 7.0 KiB/s wr, 15 op/s 2024-04-25T00:31:30.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:30 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/606097616' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:31:31.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:30 smithi121 ceph-mon[32231]: pgmap v312: 129 pgs: 129 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 10 KiB/s rd, 7.0 KiB/s wr, 15 op/s 2024-04-25T00:31:31.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:30 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/606097616' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:31:31.090 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-25T00:31:31.091 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-25T00:31:31.091 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-25T00:31:31.092 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-25T00:31:31.092 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-25T00:31:31.093 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-25T00:31:31.094 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-25T00:31:31.094 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-25T00:31:31.095 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph tell osd.8 flush_pg_stats 2024-04-25T00:31:31.096 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph tell osd.9 flush_pg_stats 2024-04-25T00:31:31.096 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph tell osd.10 flush_pg_stats 2024-04-25T00:31:31.097 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph tell osd.11 flush_pg_stats 2024-04-25T00:31:31.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:30 smithi130 ceph-mon[32525]: pgmap v312: 129 pgs: 129 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 10 KiB/s rd, 7.0 KiB/s wr, 15 op/s 2024-04-25T00:31:31.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:30 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/606097616' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:31:31.822 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:31:31.830 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:31:32.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:32 smithi032 ceph-mon[29357]: pgmap v313: 129 pgs: 129 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-04-25T00:31:33.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:32 smithi121 ceph-mon[32231]: pgmap v313: 129 pgs: 129 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-04-25T00:31:33.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:32 smithi130 ceph-mon[32525]: pgmap v313: 129 pgs: 129 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-04-25T00:31:33.392 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:31:34.327 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:31:34.330 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:31:34.330 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:31:34.338 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:31:34.342 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:31:34.342 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:31:34.343 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:31:34.347 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:31:34.377 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:31:34.905 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:34 smithi032 ceph-mon[29357]: pgmap v314: 129 pgs: 129 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 2.2 KiB/s wr, 8 op/s 2024-04-25T00:31:35.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:34 smithi121 ceph-mon[32231]: pgmap v314: 129 pgs: 129 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 2.2 KiB/s wr, 8 op/s 2024-04-25T00:31:35.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:34 smithi130 ceph-mon[32525]: pgmap v314: 129 pgs: 129 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 2.2 KiB/s wr, 8 op/s 2024-04-25T00:31:35.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.770+0000 7f8d861a9700 1 -- 172.21.15.32:0/1876062749 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8d80102f00 msgr2=0x7f8d80106e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:35.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.770+0000 7f8d861a9700 1 --2- 172.21.15.32:0/1876062749 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8d80102f00 0x7f8d80106e60 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7f8d680099c0 tx=0x7f8d68009cd0 comp rx=0 tx=0).stop 2024-04-25T00:31:35.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.770+0000 7f8d861a9700 1 -- 172.21.15.32:0/1876062749 shutdown_connections 2024-04-25T00:31:35.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.770+0000 7f8d861a9700 1 --2- 172.21.15.32:0/1876062749 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8d80107540 0x7f8d8010f110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:35.908 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.770+0000 7f8d861a9700 1 --2- 172.21.15.32:0/1876062749 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8d80102550 0x7f8d80102930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:35.908 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.770+0000 7f8d861a9700 1 --2- 172.21.15.32:0/1876062749 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8d80102f00 0x7f8d80106e60 unknown :-1 s=CLOSED pgs=295 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:35.908 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.770+0000 7f8d861a9700 1 -- 172.21.15.32:0/1876062749 >> 172.21.15.32:0/1876062749 conn(0x7f8d800793f0 msgr2=0x7f8d80079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:35.908 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.770+0000 7f8d861a9700 1 -- 172.21.15.32:0/1876062749 shutdown_connections 2024-04-25T00:31:35.908 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.770+0000 7f8d861a9700 1 -- 172.21.15.32:0/1876062749 wait complete. 2024-04-25T00:31:35.908 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.771+0000 7f8d861a9700 1 Processor -- start 2024-04-25T00:31:35.908 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.771+0000 7f8d861a9700 1 -- start start 2024-04-25T00:31:35.908 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.771+0000 7f8d861a9700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8d80102550 0x7f8d8019e6c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:35.908 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.771+0000 7f8d861a9700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8d80102f00 0x7f8d8019ec00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:35.908 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.771+0000 7f8d861a9700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8d80107540 0x7f8d80198880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:35.908 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.771+0000 7f8d861a9700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f8d80112580 con 0x7f8d80102f00 2024-04-25T00:31:35.908 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.771+0000 7f8d861a9700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f8d80112400 con 0x7f8d80102550 2024-04-25T00:31:35.908 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.771+0000 7f8d861a9700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f8d80112700 con 0x7f8d80107540 2024-04-25T00:31:35.908 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.771+0000 7f8d7effd700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8d80102f00 0x7f8d8019ec00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:35.909 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.771+0000 7f8d7effd700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8d80102f00 0x7f8d8019ec00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:41694/0 (socket says 172.21.15.32:41694) 2024-04-25T00:31:35.909 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.771+0000 7f8d7effd700 1 -- 172.21.15.32:0/3584015907 learned_addr learned my addr 172.21.15.32:0/3584015907 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:35.909 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.771+0000 7f8d7f7fe700 1 --2- 172.21.15.32:0/3584015907 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8d80102550 0x7f8d8019e6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:35.909 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.772+0000 7f8d7ffff700 1 --2- 172.21.15.32:0/3584015907 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8d80107540 0x7f8d80198880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:35.909 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.772+0000 7f8d7effd700 1 -- 172.21.15.32:0/3584015907 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8d80107540 msgr2=0x7f8d80198880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:35.909 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.772+0000 7f8d7effd700 1 --2- 172.21.15.32:0/3584015907 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8d80107540 0x7f8d80198880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:35.909 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.772+0000 7f8d7effd700 1 -- 172.21.15.32:0/3584015907 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8d80102550 msgr2=0x7f8d8019e6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:35.909 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.772+0000 7f8d7effd700 1 --2- 172.21.15.32:0/3584015907 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8d80102550 0x7f8d8019e6c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:35.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.772+0000 7f8d7effd700 1 -- 172.21.15.32:0/3584015907 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8d80198fe0 con 0x7f8d80102f00 2024-04-25T00:31:35.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.772+0000 7f8d7f7fe700 1 --2- 172.21.15.32:0/3584015907 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8d80102550 0x7f8d8019e6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:31:35.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.772+0000 7f8d7effd700 1 --2- 172.21.15.32:0/3584015907 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8d80102f00 0x7f8d8019ec00 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7f8d68009fd0 tx=0x7f8d68005f10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:35.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.772+0000 7f8d7cff9700 1 -- 172.21.15.32:0/3584015907 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8d68004830 con 0x7f8d80102f00 2024-04-25T00:31:35.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.772+0000 7f8d861a9700 1 -- 172.21.15.32:0/3584015907 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8d80199270 con 0x7f8d80102f00 2024-04-25T00:31:35.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.772+0000 7f8d7cff9700 1 -- 172.21.15.32:0/3584015907 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8d68005020 con 0x7f8d80102f00 2024-04-25T00:31:35.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.772+0000 7f8d7cff9700 1 -- 172.21.15.32:0/3584015907 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8d68003f90 con 0x7f8d80102f00 2024-04-25T00:31:35.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.772+0000 7f8d861a9700 1 -- 172.21.15.32:0/3584015907 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8d801a5590 con 0x7f8d80102f00 2024-04-25T00:31:35.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.773+0000 7f8d861a9700 1 -- 172.21.15.32:0/3584015907 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f8d800629c0 con 0x7f8d80102f00 2024-04-25T00:31:35.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.775+0000 7f8d7cff9700 1 -- 172.21.15.32:0/3584015907 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f8d68005540 con 0x7f8d80102f00 2024-04-25T00:31:35.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.775+0000 7f8d7cff9700 1 --2- 172.21.15.32:0/3584015907 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8d6c0a2cb0 0x7f8d6c0a5170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:35.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.776+0000 7f8d7f7fe700 1 --2- 172.21.15.32:0/3584015907 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8d6c0a2cb0 0x7f8d6c0a5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:35.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.776+0000 7f8d7cff9700 1 -- 172.21.15.32:0/3584015907 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f8d680d0c90 con 0x7f8d80102f00 2024-04-25T00:31:35.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.776+0000 7f8d7cff9700 1 --2- 172.21.15.32:0/3584015907 >> [v2:172.21.15.130:6824/2386788822,v1:172.21.15.130:6825/2386788822] conn(0x7f8d6c0ad970 0x7f8d6c0afdb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:35.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.776+0000 7f8d7cff9700 1 -- 172.21.15.32:0/3584015907 --> [v2:172.21.15.130:6824/2386788822,v1:172.21.15.130:6825/2386788822] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f8d6801de50 con 0x7f8d6c0ad970 2024-04-25T00:31:35.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.776+0000 7f8d7cff9700 1 -- 172.21.15.32:0/3584015907 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_get_version_reply(handle=1 version=89) ==== 24+0+0 (secure 0 0 0) 0x7f8d680d1040 con 0x7f8d80102f00 2024-04-25T00:31:35.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.776+0000 7f8d7f7fe700 1 --2- 172.21.15.32:0/3584015907 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8d6c0a2cb0 0x7f8d6c0a5170 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f8d78005950 tx=0x7f8d7800a300 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:35.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.776+0000 7f8d7ffff700 1 --2- 172.21.15.32:0/3584015907 >> [v2:172.21.15.130:6824/2386788822,v1:172.21.15.130:6825/2386788822] conn(0x7f8d6c0ad970 0x7f8d6c0afdb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:35.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.777+0000 7f8d7ffff700 1 --2- 172.21.15.32:0/3584015907 >> [v2:172.21.15.130:6824/2386788822,v1:172.21.15.130:6825/2386788822] conn(0x7f8d6c0ad970 0x7f8d6c0afdb0 crc :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.11 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:35.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.777+0000 7f8d7cff9700 1 -- 172.21.15.32:0/3584015907 <== osd.11 v2:172.21.15.130:6824/2386788822 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f8d6801de50 con 0x7f8d6c0ad970 2024-04-25T00:31:35.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.802+0000 7f8d861a9700 1 -- 172.21.15.32:0/3584015907 --> [v2:172.21.15.130:6824/2386788822,v1:172.21.15.130:6825/2386788822] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f8d80062dc0 con 0x7f8d6c0ad970 2024-04-25T00:31:35.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.802+0000 7f8d7cff9700 1 -- 172.21.15.32:0/3584015907 <== osd.11 v2:172.21.15.130:6824/2386788822 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7f8d80062dc0 con 0x7f8d6c0ad970 2024-04-25T00:31:35.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.803+0000 7f8d861a9700 1 -- 172.21.15.32:0/3584015907 >> [v2:172.21.15.130:6824/2386788822,v1:172.21.15.130:6825/2386788822] conn(0x7f8d6c0ad970 msgr2=0x7f8d6c0afdb0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:35.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.803+0000 7f8d861a9700 1 --2- 172.21.15.32:0/3584015907 >> [v2:172.21.15.130:6824/2386788822,v1:172.21.15.130:6825/2386788822] conn(0x7f8d6c0ad970 0x7f8d6c0afdb0 crc :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:35.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.803+0000 7f8d861a9700 1 -- 172.21.15.32:0/3584015907 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8d6c0a2cb0 msgr2=0x7f8d6c0a5170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:35.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.803+0000 7f8d861a9700 1 --2- 172.21.15.32:0/3584015907 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8d6c0a2cb0 0x7f8d6c0a5170 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f8d78005950 tx=0x7f8d7800a300 comp rx=0 tx=0).stop 2024-04-25T00:31:35.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.803+0000 7f8d861a9700 1 -- 172.21.15.32:0/3584015907 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8d80102f00 msgr2=0x7f8d8019ec00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:35.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.803+0000 7f8d861a9700 1 --2- 172.21.15.32:0/3584015907 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8d80102f00 0x7f8d8019ec00 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7f8d68009fd0 tx=0x7f8d68005f10 comp rx=0 tx=0).stop 2024-04-25T00:31:35.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.803+0000 7f8d7ffff700 1 -- 172.21.15.32:0/3584015907 reap_dead start 2024-04-25T00:31:35.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.803+0000 7f8d861a9700 1 -- 172.21.15.32:0/3584015907 shutdown_connections 2024-04-25T00:31:35.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.803+0000 7f8d861a9700 1 -- 172.21.15.32:0/3584015907 >> 172.21.15.32:0/3584015907 conn(0x7f8d800793f0 msgr2=0x7f8d80100340 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:35.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.803+0000 7f8d861a9700 1 -- 172.21.15.32:0/3584015907 shutdown_connections 2024-04-25T00:31:35.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.803+0000 7f8d861a9700 1 -- 172.21.15.32:0/3584015907 wait complete. 2024-04-25T00:31:36.850 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:36 smithi032 ceph-mon[29357]: pgmap v315: 129 pgs: 129 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 1.5 KiB/s wr, 6 op/s 2024-04-25T00:31:36.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.619+0000 7f04d4172700 1 -- 172.21.15.32:0/4128679637 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f04cc102ec0 msgr2=0x7f04cc106eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:36.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.619+0000 7f04d4172700 1 --2- 172.21.15.32:0/4128679637 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f04cc102ec0 0x7f04cc106eb0 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7f04bc009990 tx=0x7f04bc009ca0 comp rx=0 tx=0).stop 2024-04-25T00:31:36.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.620+0000 7f04d4172700 1 -- 172.21.15.32:0/4128679637 shutdown_connections 2024-04-25T00:31:36.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.620+0000 7f04d4172700 1 --2- 172.21.15.32:0/4128679637 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f04cc107590 0x7f04cc10f160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:36.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.620+0000 7f04d4172700 1 --2- 172.21.15.32:0/4128679637 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f04cc102510 0x7f04cc1028f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:36.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.620+0000 7f04d4172700 1 --2- 172.21.15.32:0/4128679637 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f04cc102ec0 0x7f04cc106eb0 unknown :-1 s=CLOSED pgs=293 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:36.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.620+0000 7f04d4172700 1 -- 172.21.15.32:0/4128679637 >> 172.21.15.32:0/4128679637 conn(0x7f04cc0793a0 msgr2=0x7f04cc0797b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:36.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.620+0000 7f04d4172700 1 -- 172.21.15.32:0/4128679637 shutdown_connections 2024-04-25T00:31:36.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.620+0000 7f04d4172700 1 -- 172.21.15.32:0/4128679637 wait complete. 2024-04-25T00:31:36.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.620+0000 7f04d4172700 1 Processor -- start 2024-04-25T00:31:36.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.620+0000 7f04d4172700 1 -- start start 2024-04-25T00:31:36.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.621+0000 7f04d4172700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f04cc102510 0x7f04cc19e6a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:36.942 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.621+0000 7f04d4172700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f04cc102ec0 0x7f04cc19ebe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:36.942 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.621+0000 7f04d4172700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f04cc107590 0x7f04cc198770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:36.942 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.621+0000 7f04d4172700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f04cc112530 con 0x7f04cc102ec0 2024-04-25T00:31:36.942 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.621+0000 7f04d4172700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f04cc1123b0 con 0x7f04cc102510 2024-04-25T00:31:36.942 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.621+0000 7f04d4172700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f04cc1126b0 con 0x7f04cc107590 2024-04-25T00:31:36.942 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.621+0000 7f04d170d700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f04cc102ec0 0x7f04cc19ebe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:36.942 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.621+0000 7f04d170d700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f04cc102ec0 0x7f04cc19ebe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:41682/0 (socket says 172.21.15.32:41682) 2024-04-25T00:31:36.942 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.621+0000 7f04d170d700 1 -- 172.21.15.32:0/715190527 learned_addr learned my addr 172.21.15.32:0/715190527 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:36.942 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.621+0000 7f04d270f700 1 --2- 172.21.15.32:0/715190527 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f04cc107590 0x7f04cc198770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:36.942 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.621+0000 7f04d1f0e700 1 --2- 172.21.15.32:0/715190527 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f04cc102510 0x7f04cc19e6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:36.942 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.621+0000 7f04d170d700 1 -- 172.21.15.32:0/715190527 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f04cc107590 msgr2=0x7f04cc198770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:36.943 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.621+0000 7f04d170d700 1 --2- 172.21.15.32:0/715190527 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f04cc107590 0x7f04cc198770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:36.943 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.621+0000 7f04d170d700 1 -- 172.21.15.32:0/715190527 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f04cc102510 msgr2=0x7f04cc19e6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:36.943 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.621+0000 7f04d170d700 1 --2- 172.21.15.32:0/715190527 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f04cc102510 0x7f04cc19e6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:36.943 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.621+0000 7f04d170d700 1 -- 172.21.15.32:0/715190527 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f04cc199050 con 0x7f04cc102ec0 2024-04-25T00:31:36.943 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.621+0000 7f04d170d700 1 --2- 172.21.15.32:0/715190527 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f04cc102ec0 0x7f04cc19ebe0 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7f04bc000c00 tx=0x7f04bc003680 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:36.943 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.621+0000 7f04c2ffd700 1 -- 172.21.15.32:0/715190527 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f04bc027070 con 0x7f04cc102ec0 2024-04-25T00:31:36.943 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.622+0000 7f04c2ffd700 1 -- 172.21.15.32:0/715190527 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f04bc004110 con 0x7f04cc102ec0 2024-04-25T00:31:36.944 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.622+0000 7f04d4172700 1 -- 172.21.15.32:0/715190527 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f04cc1992e0 con 0x7f04cc102ec0 2024-04-25T00:31:36.944 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.622+0000 7f04c2ffd700 1 -- 172.21.15.32:0/715190527 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f04bc00f460 con 0x7f04cc102ec0 2024-04-25T00:31:36.944 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.622+0000 7f04d4172700 1 -- 172.21.15.32:0/715190527 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f04cc1a5450 con 0x7f04cc102ec0 2024-04-25T00:31:36.944 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.622+0000 7f04d4172700 1 -- 172.21.15.32:0/715190527 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f04cc062980 con 0x7f04cc102ec0 2024-04-25T00:31:36.944 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.624+0000 7f04c2ffd700 1 -- 172.21.15.32:0/715190527 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f04bc02a070 con 0x7f04cc102ec0 2024-04-25T00:31:36.944 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.624+0000 7f04c2ffd700 1 --2- 172.21.15.32:0/715190527 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f04b80a2cb0 0x7f04b80a5170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:36.944 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.624+0000 7f04d1f0e700 1 --2- 172.21.15.32:0/715190527 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f04b80a2cb0 0x7f04b80a5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:36.944 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.624+0000 7f04c2ffd700 1 -- 172.21.15.32:0/715190527 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8802+0+0 (secure 0 0 0) 0x7f04bc0d0a80 con 0x7f04cc102ec0 2024-04-25T00:31:36.944 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.624+0000 7f04c2ffd700 1 --2- 172.21.15.32:0/715190527 >> [v2:172.21.15.121:6800/431442351,v1:172.21.15.121:6801/431442351] conn(0x7f04b80ad970 0x7f04b80afdb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:36.944 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.624+0000 7f04c2ffd700 1 -- 172.21.15.32:0/715190527 --> [v2:172.21.15.121:6800/431442351,v1:172.21.15.121:6801/431442351] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f04bc001d30 con 0x7f04b80ad970 2024-04-25T00:31:36.944 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.624+0000 7f04c2ffd700 1 -- 172.21.15.32:0/715190527 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_get_version_reply(handle=1 version=89) ==== 24+0+0 (secure 0 0 0) 0x7f04bc0d0ed0 con 0x7f04cc102ec0 2024-04-25T00:31:36.944 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.625+0000 7f04d1f0e700 1 --2- 172.21.15.32:0/715190527 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f04b80a2cb0 0x7f04b80a5170 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f04c8005fd0 tx=0x7f04c800a560 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:36.944 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.625+0000 7f04d270f700 1 --2- 172.21.15.32:0/715190527 >> [v2:172.21.15.121:6800/431442351,v1:172.21.15.121:6801/431442351] conn(0x7f04b80ad970 0x7f04b80afdb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:36.944 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.625+0000 7f04d270f700 1 --2- 172.21.15.32:0/715190527 >> [v2:172.21.15.121:6800/431442351,v1:172.21.15.121:6801/431442351] conn(0x7f04b80ad970 0x7f04b80afdb0 crc :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:36.944 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.626+0000 7f04c2ffd700 1 -- 172.21.15.32:0/715190527 <== osd.1 v2:172.21.15.121:6800/431442351 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f04bc001d30 con 0x7f04b80ad970 2024-04-25T00:31:36.945 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.642+0000 7f04d4172700 1 -- 172.21.15.32:0/715190527 --> [v2:172.21.15.121:6800/431442351,v1:172.21.15.121:6801/431442351] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f04cc062e00 con 0x7f04b80ad970 2024-04-25T00:31:36.945 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.643+0000 7f04c2ffd700 1 -- 172.21.15.32:0/715190527 <== osd.1 v2:172.21.15.121:6800/431442351 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (crc 0 0 0) 0x7f04cc062e00 con 0x7f04b80ad970 2024-04-25T00:31:36.945 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.643+0000 7f04d4172700 1 -- 172.21.15.32:0/715190527 >> [v2:172.21.15.121:6800/431442351,v1:172.21.15.121:6801/431442351] conn(0x7f04b80ad970 msgr2=0x7f04b80afdb0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:36.945 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.643+0000 7f04d4172700 1 --2- 172.21.15.32:0/715190527 >> [v2:172.21.15.121:6800/431442351,v1:172.21.15.121:6801/431442351] conn(0x7f04b80ad970 0x7f04b80afdb0 crc :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:36.945 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.643+0000 7f04d4172700 1 -- 172.21.15.32:0/715190527 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f04b80a2cb0 msgr2=0x7f04b80a5170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:36.945 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.643+0000 7f04d4172700 1 --2- 172.21.15.32:0/715190527 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f04b80a2cb0 0x7f04b80a5170 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f04c8005fd0 tx=0x7f04c800a560 comp rx=0 tx=0).stop 2024-04-25T00:31:36.945 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.643+0000 7f04d4172700 1 -- 172.21.15.32:0/715190527 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f04cc102ec0 msgr2=0x7f04cc19ebe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:36.945 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.643+0000 7f04d4172700 1 --2- 172.21.15.32:0/715190527 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f04cc102ec0 0x7f04cc19ebe0 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7f04bc000c00 tx=0x7f04bc003680 comp rx=0 tx=0).stop 2024-04-25T00:31:36.945 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.643+0000 7f04d270f700 1 -- 172.21.15.32:0/715190527 reap_dead start 2024-04-25T00:31:36.945 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.643+0000 7f04d4172700 1 -- 172.21.15.32:0/715190527 shutdown_connections 2024-04-25T00:31:36.945 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.643+0000 7f04d4172700 1 -- 172.21.15.32:0/715190527 >> 172.21.15.32:0/715190527 conn(0x7f04cc0793a0 msgr2=0x7f04cc1002c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:36.945 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.644+0000 7f04d4172700 1 -- 172.21.15.32:0/715190527 shutdown_connections 2024-04-25T00:31:36.945 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:35.644+0000 7f04d4172700 1 -- 172.21.15.32:0/715190527 wait complete. 2024-04-25T00:31:37.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:36 smithi121 ceph-mon[32231]: pgmap v315: 129 pgs: 129 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 1.5 KiB/s wr, 6 op/s 2024-04-25T00:31:37.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:36 smithi130 ceph-mon[32525]: pgmap v315: 129 pgs: 129 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 1.5 KiB/s wr, 6 op/s 2024-04-25T00:31:38.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:38 smithi032 ceph-mon[29357]: pgmap v316: 129 pgs: 129 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 1.5 KiB/s wr, 5 op/s 2024-04-25T00:31:39.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:38 smithi121 ceph-mon[32231]: pgmap v316: 129 pgs: 129 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 1.5 KiB/s wr, 5 op/s 2024-04-25T00:31:39.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:38 smithi130 ceph-mon[32525]: pgmap v316: 129 pgs: 129 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 1.5 KiB/s wr, 5 op/s 2024-04-25T00:31:40.273 INFO:teuthology.orchestra.run.smithi032.stdout:343597383703 2024-04-25T00:31:40.273 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.11 2024-04-25T00:31:40.940 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:40 smithi032 ceph-mon[29357]: pgmap v317: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 1.7 KiB/s wr, 8 op/s 2024-04-25T00:31:41.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:40 smithi121 ceph-mon[32231]: pgmap v317: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 1.7 KiB/s wr, 8 op/s 2024-04-25T00:31:41.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:40 smithi130 ceph-mon[32525]: pgmap v317: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 1.7 KiB/s wr, 8 op/s 2024-04-25T00:31:42.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:42 smithi032 ceph-mon[29357]: pgmap v318: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 170 B/s wr, 5 op/s 2024-04-25T00:31:43.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:42 smithi121 ceph-mon[32231]: pgmap v318: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 170 B/s wr, 5 op/s 2024-04-25T00:31:43.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:42 smithi130 ceph-mon[32525]: pgmap v318: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 170 B/s wr, 5 op/s 2024-04-25T00:31:43.723 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [3, 6]}]: dispatch 2024-04-25T00:31:43.723 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [4, 7]}]: dispatch 2024-04-25T00:31:43.723 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [5, 9]}]: dispatch 2024-04-25T00:31:43.724 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.20", "id": [5, 9]}]: dispatch 2024-04-25T00:31:43.724 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.23", "id": [4, 8]}]: dispatch 2024-04-25T00:31:43.724 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2c", "id": [4, 9]}]: dispatch 2024-04-25T00:31:43.724 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [9, 10]}]: dispatch 2024-04-25T00:31:43.724 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [9, 10]}]: dispatch 2024-04-25T00:31:43.724 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [9, 8]}]: dispatch 2024-04-25T00:31:43.724 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:43 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [5, 8]}]: dispatch 2024-04-25T00:31:43.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.927+0000 7f3e4fcf3700 1 -- 172.21.15.32:0/3341487514 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3e48101dd0 msgr2=0x7f3e4810dfc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:43.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.927+0000 7f3e4fcf3700 1 --2- 172.21.15.32:0/3341487514 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3e48101dd0 0x7f3e4810dfc0 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7f3e4000b0a0 tx=0x7f3e4000b3b0 comp rx=0 tx=0).stop 2024-04-25T00:31:43.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.927+0000 7f3e4fcf3700 1 -- 172.21.15.32:0/3341487514 shutdown_connections 2024-04-25T00:31:43.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.927+0000 7f3e4fcf3700 1 --2- 172.21.15.32:0/3341487514 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3e48101dd0 0x7f3e4810dfc0 unknown :-1 s=CLOSED pgs=297 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:43.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.927+0000 7f3e4fcf3700 1 --2- 172.21.15.32:0/3341487514 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3e48107280 0x7f3e48107660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:43.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.927+0000 7f3e4fcf3700 1 --2- 172.21.15.32:0/3341487514 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3e48101410 0x7f3e48101890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:43.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.927+0000 7f3e4fcf3700 1 -- 172.21.15.32:0/3341487514 >> 172.21.15.32:0/3341487514 conn(0x7f3e48078020 msgr2=0x7f3e48078430 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:43.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.927+0000 7f3e4fcf3700 1 -- 172.21.15.32:0/3341487514 shutdown_connections 2024-04-25T00:31:43.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.927+0000 7f3e4fcf3700 1 -- 172.21.15.32:0/3341487514 wait complete. 2024-04-25T00:31:43.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.928+0000 7f3e4fcf3700 1 Processor -- start 2024-04-25T00:31:43.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.928+0000 7f3e4fcf3700 1 -- start start 2024-04-25T00:31:43.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.928+0000 7f3e4fcf3700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3e48101410 0x7f3e481a07a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:43.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.928+0000 7f3e4fcf3700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3e48101dd0 0x7f3e481a0ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:43.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.928+0000 7f3e4fcf3700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3e48107280 0x7f3e4819a960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:43.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.928+0000 7f3e4fcf3700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f3e48111520 con 0x7f3e48107280 2024-04-25T00:31:43.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.928+0000 7f3e4fcf3700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f3e481113a0 con 0x7f3e48101dd0 2024-04-25T00:31:43.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.928+0000 7f3e4fcf3700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f3e481116a0 con 0x7f3e48101410 2024-04-25T00:31:43.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.928+0000 7f3e4e290700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3e48107280 0x7f3e4819a960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:43.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.928+0000 7f3e4e290700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3e48107280 0x7f3e4819a960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:48710/0 (socket says 172.21.15.32:48710) 2024-04-25T00:31:43.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.928+0000 7f3e4e290700 1 -- 172.21.15.32:0/628290951 learned_addr learned my addr 172.21.15.32:0/628290951 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:43.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.928+0000 7f3e4d28e700 1 --2- 172.21.15.32:0/628290951 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3e48101dd0 0x7f3e481a0ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:43.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.928+0000 7f3e4da8f700 1 --2- 172.21.15.32:0/628290951 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3e48101410 0x7f3e481a07a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:43.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.928+0000 7f3e4e290700 1 -- 172.21.15.32:0/628290951 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3e48101410 msgr2=0x7f3e481a07a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:43.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.928+0000 7f3e4e290700 1 --2- 172.21.15.32:0/628290951 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3e48101410 0x7f3e481a07a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:43.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.928+0000 7f3e4e290700 1 -- 172.21.15.32:0/628290951 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3e48101dd0 msgr2=0x7f3e481a0ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:43.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.928+0000 7f3e4e290700 1 --2- 172.21.15.32:0/628290951 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3e48101dd0 0x7f3e481a0ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:43.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.928+0000 7f3e4e290700 1 -- 172.21.15.32:0/628290951 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3e4819b0c0 con 0x7f3e48107280 2024-04-25T00:31:43.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.929+0000 7f3e4e290700 1 --2- 172.21.15.32:0/628290951 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3e48107280 0x7f3e4819a960 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 crypto rx=0x7f3e400062a0 tx=0x7f3e4000be60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:43.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.929+0000 7f3e3effd700 1 -- 172.21.15.32:0/628290951 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3e40021070 con 0x7f3e48107280 2024-04-25T00:31:43.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.929+0000 7f3e3effd700 1 -- 172.21.15.32:0/628290951 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3e40007c80 con 0x7f3e48107280 2024-04-25T00:31:43.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.929+0000 7f3e4fcf3700 1 -- 172.21.15.32:0/628290951 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3e4819b350 con 0x7f3e48107280 2024-04-25T00:31:43.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.929+0000 7f3e3effd700 1 -- 172.21.15.32:0/628290951 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3e40005420 con 0x7f3e48107280 2024-04-25T00:31:43.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.929+0000 7f3e4fcf3700 1 -- 172.21.15.32:0/628290951 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f3e481a7670 con 0x7f3e48107280 2024-04-25T00:31:43.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.930+0000 7f3e4fcf3700 1 -- 172.21.15.32:0/628290951 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f3e4806b360 con 0x7f3e48107280 2024-04-25T00:31:43.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.931+0000 7f3e3effd700 1 -- 172.21.15.32:0/628290951 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f3e40024070 con 0x7f3e48107280 2024-04-25T00:31:43.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.932+0000 7f3e3effd700 1 --2- 172.21.15.32:0/628290951 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3e340a2cb0 0x7f3e340a5170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:43.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.932+0000 7f3e4da8f700 1 --2- 172.21.15.32:0/628290951 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3e340a2cb0 0x7f3e340a5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:43.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.932+0000 7f3e3effd700 1 -- 172.21.15.32:0/628290951 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(90..90 src has 1..90) ==== 9092+0+0 (secure 0 0 0) 0x7f3e4000f020 con 0x7f3e48107280 2024-04-25T00:31:43.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.932+0000 7f3e3effd700 1 --2- 172.21.15.32:0/628290951 >> [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708] conn(0x7f3e340ade10 0x7f3e340b02b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:43.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.932+0000 7f3e3effd700 1 -- 172.21.15.32:0/628290951 --> [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f3e4000ce60 con 0x7f3e340ade10 2024-04-25T00:31:43.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.932+0000 7f3e3effd700 1 -- 172.21.15.32:0/628290951 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_get_version_reply(handle=1 version=90) ==== 24+0+0 (secure 0 0 0) 0x7f3e400d0ee0 con 0x7f3e48107280 2024-04-25T00:31:43.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.932+0000 7f3e4d28e700 1 --2- 172.21.15.32:0/628290951 >> [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708] conn(0x7f3e340ade10 0x7f3e340b02b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:43.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.932+0000 7f3e4da8f700 1 --2- 172.21.15.32:0/628290951 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3e340a2cb0 0x7f3e340a5170 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7f3e4819baa0 tx=0x7f3e4400a380 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:43.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.932+0000 7f3e4d28e700 1 --2- 172.21.15.32:0/628290951 >> [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708] conn(0x7f3e340ade10 0x7f3e340b02b0 crc :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:43.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.933+0000 7f3e3effd700 1 -- 172.21.15.32:0/628290951 <== osd.3 v2:172.21.15.32:6810/1510492708 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f3e4000ce60 con 0x7f3e340ade10 2024-04-25T00:31:43.953 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.950+0000 7f3e4fcf3700 1 -- 172.21.15.32:0/628290951 --> [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f3e481028a0 con 0x7f3e340ade10 2024-04-25T00:31:43.953 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.950+0000 7f3e3effd700 1 -- 172.21.15.32:0/628290951 <== osd.3 v2:172.21.15.32:6810/1510492708 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7f3e481028a0 con 0x7f3e340ade10 2024-04-25T00:31:43.954 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.950+0000 7f3e4fcf3700 1 -- 172.21.15.32:0/628290951 >> [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708] conn(0x7f3e340ade10 msgr2=0x7f3e340b02b0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:43.954 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.950+0000 7f3e4fcf3700 1 --2- 172.21.15.32:0/628290951 >> [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708] conn(0x7f3e340ade10 0x7f3e340b02b0 crc :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:43.954 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.951+0000 7f3e4fcf3700 1 -- 172.21.15.32:0/628290951 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3e340a2cb0 msgr2=0x7f3e340a5170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:43.955 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.951+0000 7f3e4fcf3700 1 --2- 172.21.15.32:0/628290951 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3e340a2cb0 0x7f3e340a5170 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7f3e4819baa0 tx=0x7f3e4400a380 comp rx=0 tx=0).stop 2024-04-25T00:31:43.955 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.951+0000 7f3e4fcf3700 1 -- 172.21.15.32:0/628290951 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3e48107280 msgr2=0x7f3e4819a960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:43.955 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.951+0000 7f3e4fcf3700 1 --2- 172.21.15.32:0/628290951 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3e48107280 0x7f3e4819a960 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 crypto rx=0x7f3e400062a0 tx=0x7f3e4000be60 comp rx=0 tx=0).stop 2024-04-25T00:31:43.955 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.951+0000 7f3e4e290700 1 -- 172.21.15.32:0/628290951 reap_dead start 2024-04-25T00:31:43.955 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.951+0000 7f3e4fcf3700 1 -- 172.21.15.32:0/628290951 shutdown_connections 2024-04-25T00:31:43.955 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.951+0000 7f3e4fcf3700 1 -- 172.21.15.32:0/628290951 >> 172.21.15.32:0/628290951 conn(0x7f3e48078020 msgr2=0x7f3e4806a980 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:43.955 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.951+0000 7f3e4fcf3700 1 -- 172.21.15.32:0/628290951 shutdown_connections 2024-04-25T00:31:43.955 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:43.951+0000 7f3e4fcf3700 1 -- 172.21.15.32:0/628290951 wait complete. 2024-04-25T00:31:44.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [3, 6]}]: dispatch 2024-04-25T00:31:44.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [4, 7]}]: dispatch 2024-04-25T00:31:44.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [5, 9]}]: dispatch 2024-04-25T00:31:44.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.20", "id": [5, 9]}]: dispatch 2024-04-25T00:31:44.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.23", "id": [4, 8]}]: dispatch 2024-04-25T00:31:44.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2c", "id": [4, 9]}]: dispatch 2024-04-25T00:31:44.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [9, 10]}]: dispatch 2024-04-25T00:31:44.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [9, 10]}]: dispatch 2024-04-25T00:31:44.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [9, 8]}]: dispatch 2024-04-25T00:31:44.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:43 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [5, 8]}]: dispatch 2024-04-25T00:31:44.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [3, 6]}]: dispatch 2024-04-25T00:31:44.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [4, 7]}]: dispatch 2024-04-25T00:31:44.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [5, 9]}]: dispatch 2024-04-25T00:31:44.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.20", "id": [5, 9]}]: dispatch 2024-04-25T00:31:44.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.23", "id": [4, 8]}]: dispatch 2024-04-25T00:31:44.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2c", "id": [4, 9]}]: dispatch 2024-04-25T00:31:44.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [9, 10]}]: dispatch 2024-04-25T00:31:44.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [9, 10]}]: dispatch 2024-04-25T00:31:44.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [9, 8]}]: dispatch 2024-04-25T00:31:44.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:43 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [5, 8]}]: dispatch 2024-04-25T00:31:44.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:44 smithi032 ceph-mon[29357]: pgmap v319: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.5 KiB/s rd, 170 B/s wr, 7 op/s 2024-04-25T00:31:44.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [3, 6]}]': finished 2024-04-25T00:31:44.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [4, 7]}]': finished 2024-04-25T00:31:44.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [5, 9]}]': finished 2024-04-25T00:31:44.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.20", "id": [5, 9]}]': finished 2024-04-25T00:31:44.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.23", "id": [4, 8]}]': finished 2024-04-25T00:31:44.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2c", "id": [4, 9]}]': finished 2024-04-25T00:31:44.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [9, 10]}]': finished 2024-04-25T00:31:44.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [9, 10]}]': finished 2024-04-25T00:31:44.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [9, 8]}]': finished 2024-04-25T00:31:44.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [5, 8]}]': finished 2024-04-25T00:31:44.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:44 smithi032 ceph-mon[29357]: osdmap e90: 12 total, 12 up, 12 in 2024-04-25T00:31:45.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:44 smithi121 ceph-mon[32231]: pgmap v319: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.5 KiB/s rd, 170 B/s wr, 7 op/s 2024-04-25T00:31:45.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [3, 6]}]': finished 2024-04-25T00:31:45.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [4, 7]}]': finished 2024-04-25T00:31:45.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [5, 9]}]': finished 2024-04-25T00:31:45.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.20", "id": [5, 9]}]': finished 2024-04-25T00:31:45.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.23", "id": [4, 8]}]': finished 2024-04-25T00:31:45.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2c", "id": [4, 9]}]': finished 2024-04-25T00:31:45.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [9, 10]}]': finished 2024-04-25T00:31:45.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [9, 10]}]': finished 2024-04-25T00:31:45.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [9, 8]}]': finished 2024-04-25T00:31:45.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [5, 8]}]': finished 2024-04-25T00:31:45.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:44 smithi121 ceph-mon[32231]: osdmap e90: 12 total, 12 up, 12 in 2024-04-25T00:31:45.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:44 smithi130 ceph-mon[32525]: pgmap v319: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.5 KiB/s rd, 170 B/s wr, 7 op/s 2024-04-25T00:31:45.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [3, 6]}]': finished 2024-04-25T00:31:45.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [4, 7]}]': finished 2024-04-25T00:31:45.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [5, 9]}]': finished 2024-04-25T00:31:45.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.20", "id": [5, 9]}]': finished 2024-04-25T00:31:45.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.23", "id": [4, 8]}]': finished 2024-04-25T00:31:45.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2c", "id": [4, 9]}]': finished 2024-04-25T00:31:45.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [9, 10]}]': finished 2024-04-25T00:31:45.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [9, 10]}]': finished 2024-04-25T00:31:45.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [9, 8]}]': finished 2024-04-25T00:31:45.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [5, 8]}]': finished 2024-04-25T00:31:45.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:44 smithi130 ceph-mon[32525]: osdmap e90: 12 total, 12 up, 12 in 2024-04-25T00:31:45.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:45 smithi032 ceph-mon[29357]: osdmap e91: 12 total, 12 up, 12 in 2024-04-25T00:31:46.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:45 smithi121 ceph-mon[32231]: osdmap e91: 12 total, 12 up, 12 in 2024-04-25T00:31:46.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:45 smithi130 ceph-mon[32525]: osdmap e91: 12 total, 12 up, 12 in 2024-04-25T00:31:46.916 INFO:teuthology.orchestra.run.smithi032.stdout:107374182465 2024-04-25T00:31:46.916 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-25T00:31:46.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:46 smithi032 ceph-mon[29357]: pgmap v322: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.0 KiB/s rd, 255 B/s wr, 7 op/s 2024-04-25T00:31:46.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:46 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:31:47.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:46 smithi121 ceph-mon[32231]: pgmap v322: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.0 KiB/s rd, 255 B/s wr, 7 op/s 2024-04-25T00:31:47.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:46 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:31:47.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:46 smithi130 ceph-mon[32525]: pgmap v322: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.0 KiB/s rd, 255 B/s wr, 7 op/s 2024-04-25T00:31:47.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:46 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:31:48.224 INFO:teuthology.orchestra.run.smithi032.stdout:55834574920 2024-04-25T00:31:48.225 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-25T00:31:48.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:48 smithi032 ceph-mon[29357]: pgmap v323: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 1.9 KiB/s rd, 3 op/s 2024-04-25T00:31:48.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:48 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:31:48.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:48 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:31:48.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:48 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:31:48.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:48 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:31:49.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:48 smithi121 ceph-mon[32231]: pgmap v323: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 1.9 KiB/s rd, 3 op/s 2024-04-25T00:31:49.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:48 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:31:49.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:48 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:31:49.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:48 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:31:49.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:48 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:31:49.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:48 smithi130 ceph-mon[32525]: pgmap v323: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 1.9 KiB/s rd, 3 op/s 2024-04-25T00:31:49.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:48 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:31:49.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:48 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:31:49.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:48 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:31:49.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:48 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:31:49.341 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.337+0000 7f58216c6700 1 -- 172.21.15.32:0/2655602857 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f581c1075a0 msgr2=0x7f581c10f170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:49.341 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.337+0000 7f58216c6700 1 --2- 172.21.15.32:0/2655602857 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f581c1075a0 0x7f581c10f170 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7f5814009990 tx=0x7f5814009ca0 comp rx=0 tx=0).stop 2024-04-25T00:31:49.341 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.337+0000 7f58216c6700 1 -- 172.21.15.32:0/2655602857 shutdown_connections 2024-04-25T00:31:49.341 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.337+0000 7f58216c6700 1 --2- 172.21.15.32:0/2655602857 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f581c1075a0 0x7f581c10f170 unknown :-1 s=CLOSED pgs=299 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:49.341 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.337+0000 7f58216c6700 1 --2- 172.21.15.32:0/2655602857 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f581c102540 0x7f581c102920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:49.341 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.337+0000 7f58216c6700 1 --2- 172.21.15.32:0/2655602857 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f581c102ef0 0x7f581c106ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:49.341 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.337+0000 7f58216c6700 1 -- 172.21.15.32:0/2655602857 >> 172.21.15.32:0/2655602857 conn(0x7f581c0792c0 msgr2=0x7f581c0796d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:49.341 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.337+0000 7f58216c6700 1 -- 172.21.15.32:0/2655602857 shutdown_connections 2024-04-25T00:31:49.341 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.337+0000 7f58216c6700 1 -- 172.21.15.32:0/2655602857 wait complete. 2024-04-25T00:31:49.341 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.338+0000 7f58216c6700 1 Processor -- start 2024-04-25T00:31:49.341 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.338+0000 7f58216c6700 1 -- start start 2024-04-25T00:31:49.342 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.338+0000 7f58216c6700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f581c102540 0x7f581c19e660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:49.342 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.338+0000 7f58216c6700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f581c102ef0 0x7f581c19eba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:49.342 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.338+0000 7f58216c6700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f581c1075a0 0x7f581c1987e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:49.342 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.338+0000 7f58216c6700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f581c112580 con 0x7f581c102540 2024-04-25T00:31:49.342 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.338+0000 7f58216c6700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f581c112400 con 0x7f581c1075a0 2024-04-25T00:31:49.342 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.338+0000 7f58216c6700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f581c112700 con 0x7f581c102ef0 2024-04-25T00:31:49.342 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.339+0000 7f581affd700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f581c102540 0x7f581c19e660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:49.342 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.339+0000 7f581affd700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f581c102540 0x7f581c19e660 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:48736/0 (socket says 172.21.15.32:48736) 2024-04-25T00:31:49.342 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.339+0000 7f581affd700 1 -- 172.21.15.32:0/3880634469 learned_addr learned my addr 172.21.15.32:0/3880634469 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:49.342 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.339+0000 7f581b7fe700 1 --2- 172.21.15.32:0/3880634469 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f581c1075a0 0x7f581c1987e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:49.342 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.339+0000 7f581a7fc700 1 --2- 172.21.15.32:0/3880634469 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f581c102ef0 0x7f581c19eba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:49.342 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.339+0000 7f581affd700 1 -- 172.21.15.32:0/3880634469 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f581c102ef0 msgr2=0x7f581c19eba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:49.342 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.339+0000 7f581affd700 1 --2- 172.21.15.32:0/3880634469 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f581c102ef0 0x7f581c19eba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:49.343 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.339+0000 7f581affd700 1 -- 172.21.15.32:0/3880634469 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f581c1075a0 msgr2=0x7f581c1987e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:49.343 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.339+0000 7f581affd700 1 --2- 172.21.15.32:0/3880634469 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f581c1075a0 0x7f581c1987e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:49.343 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.339+0000 7f581affd700 1 -- 172.21.15.32:0/3880634469 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f581c198fc0 con 0x7f581c102540 2024-04-25T00:31:49.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.339+0000 7f581affd700 1 --2- 172.21.15.32:0/3880634469 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f581c102540 0x7f581c19e660 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 crypto rx=0x7f580c0060b0 tx=0x7f580c00d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:49.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.339+0000 7f5813fff700 1 -- 172.21.15.32:0/3880634469 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f580c01a070 con 0x7f581c102540 2024-04-25T00:31:49.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.339+0000 7f58216c6700 1 -- 172.21.15.32:0/3880634469 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f581c1992b0 con 0x7f581c102540 2024-04-25T00:31:49.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.339+0000 7f5813fff700 1 -- 172.21.15.32:0/3880634469 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f580c00ddd0 con 0x7f581c102540 2024-04-25T00:31:49.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.339+0000 7f5813fff700 1 -- 172.21.15.32:0/3880634469 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f580c0147a0 con 0x7f581c102540 2024-04-25T00:31:49.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.339+0000 7f58216c6700 1 -- 172.21.15.32:0/3880634469 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f581c1a54c0 con 0x7f581c102540 2024-04-25T00:31:49.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.340+0000 7f58216c6700 1 -- 172.21.15.32:0/3880634469 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f581c0629c0 con 0x7f581c102540 2024-04-25T00:31:49.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.342+0000 7f5813fff700 1 -- 172.21.15.32:0/3880634469 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f580c01d070 con 0x7f581c102540 2024-04-25T00:31:49.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.342+0000 7f5813fff700 1 --2- 172.21.15.32:0/3880634469 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f58080a2cb0 0x7f58080a5170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:49.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.343+0000 7f5813fff700 1 -- 172.21.15.32:0/3880634469 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f580c0cf620 con 0x7f581c102540 2024-04-25T00:31:49.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.343+0000 7f5813fff700 1 --2- 172.21.15.32:0/3880634469 >> [v2:172.21.15.121:6808/1286757826,v1:172.21.15.121:6809/1286757826] conn(0x7f58080ade10 0x7f58080b02b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:49.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.343+0000 7f5813fff700 1 -- 172.21.15.32:0/3880634469 --> [v2:172.21.15.121:6808/1286757826,v1:172.21.15.121:6809/1286757826] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f58080b0980 con 0x7f58080ade10 2024-04-25T00:31:49.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.343+0000 7f5813fff700 1 -- 172.21.15.32:0/3880634469 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_get_version_reply(handle=1 version=91) ==== 24+0+0 (secure 0 0 0) 0x7f580c0cfa70 con 0x7f581c102540 2024-04-25T00:31:49.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.343+0000 7f581a7fc700 1 --2- 172.21.15.32:0/3880634469 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f58080a2cb0 0x7f58080a5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:49.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.343+0000 7f581b7fe700 1 --2- 172.21.15.32:0/3880634469 >> [v2:172.21.15.121:6808/1286757826,v1:172.21.15.121:6809/1286757826] conn(0x7f58080ade10 0x7f58080b02b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:49.346 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.344+0000 7f581a7fc700 1 --2- 172.21.15.32:0/3880634469 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f58080a2cb0 0x7f58080a5170 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f5804005fd0 tx=0x7f5804005d90 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:49.346 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.344+0000 7f581b7fe700 1 --2- 172.21.15.32:0/3880634469 >> [v2:172.21.15.121:6808/1286757826,v1:172.21.15.121:6809/1286757826] conn(0x7f58080ade10 0x7f58080b02b0 crc :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:49.346 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.345+0000 7f5813fff700 1 -- 172.21.15.32:0/3880634469 <== osd.4 v2:172.21.15.121:6808/1286757826 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f58080b0980 con 0x7f58080ade10 2024-04-25T00:31:49.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.362+0000 7f58216c6700 1 -- 172.21.15.32:0/3880634469 --> [v2:172.21.15.121:6808/1286757826,v1:172.21.15.121:6809/1286757826] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f581c062dc0 con 0x7f58080ade10 2024-04-25T00:31:49.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.362+0000 7f5813fff700 1 -- 172.21.15.32:0/3880634469 <== osd.4 v2:172.21.15.121:6808/1286757826 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7f581c062dc0 con 0x7f58080ade10 2024-04-25T00:31:49.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.362+0000 7f58216c6700 1 -- 172.21.15.32:0/3880634469 >> [v2:172.21.15.121:6808/1286757826,v1:172.21.15.121:6809/1286757826] conn(0x7f58080ade10 msgr2=0x7f58080b02b0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:49.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.362+0000 7f58216c6700 1 --2- 172.21.15.32:0/3880634469 >> [v2:172.21.15.121:6808/1286757826,v1:172.21.15.121:6809/1286757826] conn(0x7f58080ade10 0x7f58080b02b0 crc :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:49.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.363+0000 7f58216c6700 1 -- 172.21.15.32:0/3880634469 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f58080a2cb0 msgr2=0x7f58080a5170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:49.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.363+0000 7f58216c6700 1 --2- 172.21.15.32:0/3880634469 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f58080a2cb0 0x7f58080a5170 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f5804005fd0 tx=0x7f5804005d90 comp rx=0 tx=0).stop 2024-04-25T00:31:49.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.363+0000 7f58216c6700 1 -- 172.21.15.32:0/3880634469 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f581c102540 msgr2=0x7f581c19e660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:49.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.363+0000 7f58216c6700 1 --2- 172.21.15.32:0/3880634469 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f581c102540 0x7f581c19e660 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 crypto rx=0x7f580c0060b0 tx=0x7f580c00d3b0 comp rx=0 tx=0).stop 2024-04-25T00:31:49.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.363+0000 7f581b7fe700 1 -- 172.21.15.32:0/3880634469 reap_dead start 2024-04-25T00:31:49.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.363+0000 7f58216c6700 1 -- 172.21.15.32:0/3880634469 shutdown_connections 2024-04-25T00:31:49.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.363+0000 7f58216c6700 1 -- 172.21.15.32:0/3880634469 >> 172.21.15.32:0/3880634469 conn(0x7f581c0792c0 msgr2=0x7f581c1002f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:49.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.363+0000 7f58216c6700 1 -- 172.21.15.32:0/3880634469 shutdown_connections 2024-04-25T00:31:49.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.363+0000 7f58216c6700 1 -- 172.21.15.32:0/3880634469 wait complete. 2024-04-25T00:31:50.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.002+0000 7f306f078700 1 -- 172.21.15.32:0/183278057 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3068107590 msgr2=0x7f306810f160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:50.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.002+0000 7f306f078700 1 --2- 172.21.15.32:0/183278057 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3068107590 0x7f306810f160 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f306000b0a0 tx=0x7f306000b3b0 comp rx=0 tx=0).stop 2024-04-25T00:31:50.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.002+0000 7f306f078700 1 -- 172.21.15.32:0/183278057 shutdown_connections 2024-04-25T00:31:50.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.002+0000 7f306f078700 1 --2- 172.21.15.32:0/183278057 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3068107590 0x7f306810f160 unknown :-1 s=CLOSED pgs=303 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:50.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.002+0000 7f306f078700 1 --2- 172.21.15.32:0/183278057 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3068102510 0x7f30681028f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:50.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.002+0000 7f306f078700 1 --2- 172.21.15.32:0/183278057 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3068102ec0 0x7f3068106eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:50.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.002+0000 7f306f078700 1 -- 172.21.15.32:0/183278057 >> 172.21.15.32:0/183278057 conn(0x7f30680793a0 msgr2=0x7f30680797b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:50.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.002+0000 7f306f078700 1 -- 172.21.15.32:0/183278057 shutdown_connections 2024-04-25T00:31:50.005 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.002+0000 7f306f078700 1 -- 172.21.15.32:0/183278057 wait complete. 2024-04-25T00:31:50.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.003+0000 7f306f078700 1 Processor -- start 2024-04-25T00:31:50.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.003+0000 7f306f078700 1 -- start start 2024-04-25T00:31:50.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.003+0000 7f306f078700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3068102510 0x7f306819e630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:50.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.003+0000 7f306f078700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3068102ec0 0x7f306819eb70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:50.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.003+0000 7f306f078700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3068107590 0x7f30681987b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:50.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.003+0000 7f306f078700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f3068112570 con 0x7f3068102ec0 2024-04-25T00:31:50.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.003+0000 7f306f078700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f30681123f0 con 0x7f3068107590 2024-04-25T00:31:50.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.003+0000 7f306f078700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f30681126f0 con 0x7f3068102510 2024-04-25T00:31:50.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.003+0000 7f3067fff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3068102ec0 0x7f306819eb70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:50.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.003+0000 7f3067fff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3068102ec0 0x7f306819eb70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:48760/0 (socket says 172.21.15.32:48760) 2024-04-25T00:31:50.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.003+0000 7f3067fff700 1 -- 172.21.15.32:0/3864512834 learned_addr learned my addr 172.21.15.32:0/3864512834 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:50.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.004+0000 7f306d615700 1 --2- 172.21.15.32:0/3864512834 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3068107590 0x7f30681987b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:50.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.004+0000 7f306ce14700 1 --2- 172.21.15.32:0/3864512834 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3068102510 0x7f306819e630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:50.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.004+0000 7f3067fff700 1 -- 172.21.15.32:0/3864512834 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3068102510 msgr2=0x7f306819e630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:50.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.004+0000 7f3067fff700 1 --2- 172.21.15.32:0/3864512834 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3068102510 0x7f306819e630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:50.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.004+0000 7f3067fff700 1 -- 172.21.15.32:0/3864512834 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3068107590 msgr2=0x7f30681987b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:50.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.004+0000 7f3067fff700 1 --2- 172.21.15.32:0/3864512834 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3068107590 0x7f30681987b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:50.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.004+0000 7f3067fff700 1 -- 172.21.15.32:0/3864512834 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3068198f90 con 0x7f3068102ec0 2024-04-25T00:31:50.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.004+0000 7f3067fff700 1 --2- 172.21.15.32:0/3864512834 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3068102ec0 0x7f306819eb70 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7f305000d950 tx=0x7f305000dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:50.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.004+0000 7f3065ffb700 1 -- 172.21.15.32:0/3864512834 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3050017070 con 0x7f3068102ec0 2024-04-25T00:31:50.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.004+0000 7f3065ffb700 1 -- 172.21.15.32:0/3864512834 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3050005020 con 0x7f3068102ec0 2024-04-25T00:31:50.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.004+0000 7f306f078700 1 -- 172.21.15.32:0/3864512834 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3068199280 con 0x7f3068102ec0 2024-04-25T00:31:50.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.004+0000 7f3065ffb700 1 -- 172.21.15.32:0/3864512834 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3050009b20 con 0x7f3068102ec0 2024-04-25T00:31:50.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.004+0000 7f306f078700 1 -- 172.21.15.32:0/3864512834 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f30681a5490 con 0x7f3068102ec0 2024-04-25T00:31:50.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.005+0000 7f306f078700 1 -- 172.21.15.32:0/3864512834 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f30680629c0 con 0x7f3068102ec0 2024-04-25T00:31:50.009 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.006+0000 7f3065ffb700 1 -- 172.21.15.32:0/3864512834 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f30500051d0 con 0x7f3068102ec0 2024-04-25T00:31:50.010 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.007+0000 7f3065ffb700 1 --2- 172.21.15.32:0/3864512834 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f30540a2cb0 0x7f30540a5170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:50.010 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.007+0000 7f306ce14700 1 --2- 172.21.15.32:0/3864512834 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f30540a2cb0 0x7f30540a5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:50.010 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.007+0000 7f3065ffb700 1 -- 172.21.15.32:0/3864512834 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f30500cf200 con 0x7f3068102ec0 2024-04-25T00:31:50.010 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.007+0000 7f3065ffb700 1 --2- 172.21.15.32:0/3864512834 >> [v2:172.21.15.130:6816/285467211,v1:172.21.15.130:6817/285467211] conn(0x7f30540ade10 0x7f30540b02b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:50.010 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.007+0000 7f3065ffb700 1 -- 172.21.15.32:0/3864512834 --> [v2:172.21.15.130:6816/285467211,v1:172.21.15.130:6817/285467211] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f30540b0980 con 0x7f30540ade10 2024-04-25T00:31:50.010 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.007+0000 7f3065ffb700 1 -- 172.21.15.32:0/3864512834 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_get_version_reply(handle=1 version=91) ==== 24+0+0 (secure 0 0 0) 0x7f30500cf5b0 con 0x7f3068102ec0 2024-04-25T00:31:50.010 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.007+0000 7f306ce14700 1 --2- 172.21.15.32:0/3864512834 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f30540a2cb0 0x7f30540a5170 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f3058005fd0 tx=0x7f3058005f60 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:50.010 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.008+0000 7f306d615700 1 --2- 172.21.15.32:0/3864512834 >> [v2:172.21.15.130:6816/285467211,v1:172.21.15.130:6817/285467211] conn(0x7f30540ade10 0x7f30540b02b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:50.010 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.008+0000 7f306d615700 1 --2- 172.21.15.32:0/3864512834 >> [v2:172.21.15.130:6816/285467211,v1:172.21.15.130:6817/285467211] conn(0x7f30540ade10 0x7f30540b02b0 crc :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.8 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:50.011 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.009+0000 7f3065ffb700 1 -- 172.21.15.32:0/3864512834 <== osd.8 v2:172.21.15.130:6816/285467211 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f30540b0980 con 0x7f30540ade10 2024-04-25T00:31:50.027 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.026+0000 7f306f078700 1 -- 172.21.15.32:0/3864512834 --> [v2:172.21.15.130:6816/285467211,v1:172.21.15.130:6817/285467211] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f3068062e40 con 0x7f30540ade10 2024-04-25T00:31:50.028 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.026+0000 7f3065ffb700 1 -- 172.21.15.32:0/3864512834 <== osd.8 v2:172.21.15.130:6816/285467211 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7f3068062e40 con 0x7f30540ade10 2024-04-25T00:31:50.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.027+0000 7f306f078700 1 -- 172.21.15.32:0/3864512834 >> [v2:172.21.15.130:6816/285467211,v1:172.21.15.130:6817/285467211] conn(0x7f30540ade10 msgr2=0x7f30540b02b0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:50.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.027+0000 7f306f078700 1 --2- 172.21.15.32:0/3864512834 >> [v2:172.21.15.130:6816/285467211,v1:172.21.15.130:6817/285467211] conn(0x7f30540ade10 0x7f30540b02b0 crc :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:50.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.027+0000 7f306f078700 1 -- 172.21.15.32:0/3864512834 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f30540a2cb0 msgr2=0x7f30540a5170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:50.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.027+0000 7f306f078700 1 --2- 172.21.15.32:0/3864512834 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f30540a2cb0 0x7f30540a5170 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f3058005fd0 tx=0x7f3058005f60 comp rx=0 tx=0).stop 2024-04-25T00:31:50.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.027+0000 7f306f078700 1 -- 172.21.15.32:0/3864512834 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3068102ec0 msgr2=0x7f306819eb70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:50.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.027+0000 7f306f078700 1 --2- 172.21.15.32:0/3864512834 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3068102ec0 0x7f306819eb70 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7f305000d950 tx=0x7f305000dc60 comp rx=0 tx=0).stop 2024-04-25T00:31:50.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.027+0000 7f306d615700 1 -- 172.21.15.32:0/3864512834 reap_dead start 2024-04-25T00:31:50.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.027+0000 7f306f078700 1 -- 172.21.15.32:0/3864512834 shutdown_connections 2024-04-25T00:31:50.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.027+0000 7f306f078700 1 -- 172.21.15.32:0/3864512834 >> 172.21.15.32:0/3864512834 conn(0x7f30680793a0 msgr2=0x7f30681002c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:50.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.027+0000 7f306f078700 1 -- 172.21.15.32:0/3864512834 shutdown_connections 2024-04-25T00:31:50.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.027+0000 7f306f078700 1 -- 172.21.15.32:0/3864512834 wait complete. 2024-04-25T00:31:50.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.678+0000 7f8ea09a0700 1 -- 172.21.15.32:0/212238498 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8e9c101150 msgr2=0x7f8e9c10c1e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:50.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.678+0000 7f8ea09a0700 1 --2- 172.21.15.32:0/212238498 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8e9c101150 0x7f8e9c10c1e0 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7f8e90009990 tx=0x7f8e90009ca0 comp rx=0 tx=0).stop 2024-04-25T00:31:50.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.678+0000 7f8ea09a0700 1 -- 172.21.15.32:0/212238498 shutdown_connections 2024-04-25T00:31:50.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.678+0000 7f8ea09a0700 1 --2- 172.21.15.32:0/212238498 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8e9c10c7b0 0x7f8e9c10ec00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:50.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.678+0000 7f8ea09a0700 1 --2- 172.21.15.32:0/212238498 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8e9c100830 0x7f8e9c100c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:50.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.678+0000 7f8ea09a0700 1 --2- 172.21.15.32:0/212238498 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8e9c101150 0x7f8e9c10c1e0 unknown :-1 s=CLOSED pgs=307 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:50.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.678+0000 7f8ea09a0700 1 -- 172.21.15.32:0/212238498 >> 172.21.15.32:0/212238498 conn(0x7f8e9c075ed0 msgr2=0x7f8e9c0762e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:50.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.678+0000 7f8ea09a0700 1 -- 172.21.15.32:0/212238498 shutdown_connections 2024-04-25T00:31:50.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.678+0000 7f8ea09a0700 1 -- 172.21.15.32:0/212238498 wait complete. 2024-04-25T00:31:50.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.678+0000 7f8ea09a0700 1 Processor -- start 2024-04-25T00:31:50.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.679+0000 7f8ea09a0700 1 -- start start 2024-04-25T00:31:50.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.679+0000 7f8ea09a0700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8e9c100830 0x7f8e9c1a0780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:50.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.679+0000 7f8ea09a0700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8e9c101150 0x7f8e9c1a0cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:50.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.679+0000 7f8ea09a0700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8e9c10c7b0 0x7f8e9c19a940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:50.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.679+0000 7f8ea09a0700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f8e9c1124e0 con 0x7f8e9c101150 2024-04-25T00:31:50.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.679+0000 7f8ea09a0700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f8e9c112360 con 0x7f8e9c100830 2024-04-25T00:31:50.702 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.679+0000 7f8ea09a0700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f8e9c112660 con 0x7f8e9c10c7b0 2024-04-25T00:31:50.702 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.679+0000 7f8e9affd700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8e9c101150 0x7f8e9c1a0cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:50.702 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.679+0000 7f8e9affd700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8e9c101150 0x7f8e9c1a0cc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:48798/0 (socket says 172.21.15.32:48798) 2024-04-25T00:31:50.702 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.679+0000 7f8e9affd700 1 -- 172.21.15.32:0/1984910187 learned_addr learned my addr 172.21.15.32:0/1984910187 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:50.702 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.679+0000 7f8e9bfff700 1 --2- 172.21.15.32:0/1984910187 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8e9c10c7b0 0x7f8e9c19a940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:50.702 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.679+0000 7f8e9b7fe700 1 --2- 172.21.15.32:0/1984910187 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8e9c100830 0x7f8e9c1a0780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:50.702 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.679+0000 7f8e9affd700 1 -- 172.21.15.32:0/1984910187 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8e9c10c7b0 msgr2=0x7f8e9c19a940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:50.702 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.679+0000 7f8e9affd700 1 --2- 172.21.15.32:0/1984910187 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8e9c10c7b0 0x7f8e9c19a940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:50.702 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.679+0000 7f8e9affd700 1 -- 172.21.15.32:0/1984910187 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8e9c100830 msgr2=0x7f8e9c1a0780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:50.702 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.679+0000 7f8e9affd700 1 --2- 172.21.15.32:0/1984910187 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8e9c100830 0x7f8e9c1a0780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:50.702 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.679+0000 7f8e9affd700 1 -- 172.21.15.32:0/1984910187 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8e9c19b0a0 con 0x7f8e9c101150 2024-04-25T00:31:50.702 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.680+0000 7f8e9affd700 1 --2- 172.21.15.32:0/1984910187 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8e9c101150 0x7f8e9c1a0cc0 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7f8e90000c00 tx=0x7f8e9000bb60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:50.702 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.680+0000 7f8e98ff9700 1 -- 172.21.15.32:0/1984910187 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8e90027070 con 0x7f8e9c101150 2024-04-25T00:31:50.703 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.680+0000 7f8ea09a0700 1 -- 172.21.15.32:0/1984910187 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8e9c19b330 con 0x7f8e9c101150 2024-04-25T00:31:50.703 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.680+0000 7f8e98ff9700 1 -- 172.21.15.32:0/1984910187 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8e90003d70 con 0x7f8e9c101150 2024-04-25T00:31:50.703 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.680+0000 7f8e98ff9700 1 -- 172.21.15.32:0/1984910187 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8e9000f460 con 0x7f8e9c101150 2024-04-25T00:31:50.703 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.680+0000 7f8ea09a0700 1 -- 172.21.15.32:0/1984910187 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8e9c1a7650 con 0x7f8e9c101150 2024-04-25T00:31:50.703 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.681+0000 7f8ea09a0700 1 -- 172.21.15.32:0/1984910187 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f8e9c100020 con 0x7f8e9c101150 2024-04-25T00:31:50.703 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.682+0000 7f8e98ff9700 1 -- 172.21.15.32:0/1984910187 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f8e9002a070 con 0x7f8e9c101150 2024-04-25T00:31:50.703 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.684+0000 7f8e98ff9700 1 --2- 172.21.15.32:0/1984910187 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8e840a2cb0 0x7f8e840a5170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:50.703 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.684+0000 7f8e9b7fe700 1 --2- 172.21.15.32:0/1984910187 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8e840a2cb0 0x7f8e840a5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:50.703 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.684+0000 7f8e98ff9700 1 -- 172.21.15.32:0/1984910187 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f8e900d0b70 con 0x7f8e9c101150 2024-04-25T00:31:50.703 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.684+0000 7f8e98ff9700 1 --2- 172.21.15.32:0/1984910187 >> [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027] conn(0x7f8e840ade10 0x7f8e840b02b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:50.703 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.684+0000 7f8e98ff9700 1 -- 172.21.15.32:0/1984910187 --> [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f8e9000ce90 con 0x7f8e840ade10 2024-04-25T00:31:50.703 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.684+0000 7f8e98ff9700 1 -- 172.21.15.32:0/1984910187 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_get_version_reply(handle=1 version=91) ==== 24+0+0 (secure 0 0 0) 0x7f8e900d0fc0 con 0x7f8e9c101150 2024-04-25T00:31:50.703 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.684+0000 7f8e9bfff700 1 --2- 172.21.15.32:0/1984910187 >> [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027] conn(0x7f8e840ade10 0x7f8e840b02b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:50.703 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.684+0000 7f8e9b7fe700 1 --2- 172.21.15.32:0/1984910187 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8e840a2cb0 0x7f8e840a5170 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f8e8c005950 tx=0x7f8e8c005eb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:50.704 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.684+0000 7f8e9bfff700 1 --2- 172.21.15.32:0/1984910187 >> [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027] conn(0x7f8e840ade10 0x7f8e840b02b0 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.6 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:50.704 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.684+0000 7f8e98ff9700 1 -- 172.21.15.32:0/1984910187 <== osd.6 v2:172.21.15.32:6818/3198957027 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f8e9000ce90 con 0x7f8e840ade10 2024-04-25T00:31:50.704 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.701+0000 7f8ea09a0700 1 -- 172.21.15.32:0/1984910187 --> [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f8e9c1017f0 con 0x7f8e840ade10 2024-04-25T00:31:50.704 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.701+0000 7f8e98ff9700 1 -- 172.21.15.32:0/1984910187 <== osd.6 v2:172.21.15.32:6818/3198957027 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7f8e9c1017f0 con 0x7f8e840ade10 2024-04-25T00:31:50.705 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.702+0000 7f8ea09a0700 1 -- 172.21.15.32:0/1984910187 >> [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027] conn(0x7f8e840ade10 msgr2=0x7f8e840b02b0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:50.705 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.702+0000 7f8ea09a0700 1 --2- 172.21.15.32:0/1984910187 >> [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027] conn(0x7f8e840ade10 0x7f8e840b02b0 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:50.705 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.702+0000 7f8ea09a0700 1 -- 172.21.15.32:0/1984910187 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8e840a2cb0 msgr2=0x7f8e840a5170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:50.705 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.702+0000 7f8ea09a0700 1 --2- 172.21.15.32:0/1984910187 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8e840a2cb0 0x7f8e840a5170 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f8e8c005950 tx=0x7f8e8c005eb0 comp rx=0 tx=0).stop 2024-04-25T00:31:50.705 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.702+0000 7f8ea09a0700 1 -- 172.21.15.32:0/1984910187 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8e9c101150 msgr2=0x7f8e9c1a0cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:50.705 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.702+0000 7f8ea09a0700 1 --2- 172.21.15.32:0/1984910187 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8e9c101150 0x7f8e9c1a0cc0 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7f8e90000c00 tx=0x7f8e9000bb60 comp rx=0 tx=0).stop 2024-04-25T00:31:50.705 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.702+0000 7f8e9bfff700 1 -- 172.21.15.32:0/1984910187 reap_dead start 2024-04-25T00:31:50.706 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.702+0000 7f8ea09a0700 1 -- 172.21.15.32:0/1984910187 shutdown_connections 2024-04-25T00:31:50.706 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.702+0000 7f8ea09a0700 1 -- 172.21.15.32:0/1984910187 >> 172.21.15.32:0/1984910187 conn(0x7f8e9c075ed0 msgr2=0x7f8e9c0ff5c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:50.706 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.702+0000 7f8ea09a0700 1 -- 172.21.15.32:0/1984910187 shutdown_connections 2024-04-25T00:31:50.706 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.702+0000 7f8ea09a0700 1 -- 172.21.15.32:0/1984910187 wait complete. 2024-04-25T00:31:50.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.174+0000 7ff85c336700 1 -- 172.21.15.32:0/3304871781 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff8541061b0 msgr2=0x7ff854106590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:50.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.174+0000 7ff85c336700 1 --2- 172.21.15.32:0/3304871781 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff8541061b0 0x7ff854106590 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7ff8500099c0 tx=0x7ff850009cd0 comp rx=0 tx=0).stop 2024-04-25T00:31:50.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.174+0000 7ff85c336700 1 -- 172.21.15.32:0/3304871781 shutdown_connections 2024-04-25T00:31:50.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.174+0000 7ff85c336700 1 --2- 172.21.15.32:0/3304871781 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff854101e40 0x7ff85410dfc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:50.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.174+0000 7ff85c336700 1 --2- 172.21.15.32:0/3304871781 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff8541061b0 0x7ff854106590 unknown :-1 s=CLOSED pgs=305 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:50.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.174+0000 7ff85c336700 1 --2- 172.21.15.32:0/3304871781 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff854101480 0x7ff854101900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:50.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.174+0000 7ff85c336700 1 -- 172.21.15.32:0/3304871781 >> 172.21.15.32:0/3304871781 conn(0x7ff854076fa0 msgr2=0x7ff8540773b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:50.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.174+0000 7ff85c336700 1 -- 172.21.15.32:0/3304871781 shutdown_connections 2024-04-25T00:31:50.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.174+0000 7ff85c336700 1 -- 172.21.15.32:0/3304871781 wait complete. 2024-04-25T00:31:50.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.175+0000 7ff85c336700 1 Processor -- start 2024-04-25T00:31:50.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.175+0000 7ff85c336700 1 -- start start 2024-04-25T00:31:50.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.176+0000 7ff85c336700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff854101480 0x7ff85419d560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:50.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.176+0000 7ff85c336700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff854101e40 0x7ff85419daa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:50.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.176+0000 7ff85c336700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff8541061b0 0x7ff854197630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:50.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.176+0000 7ff85c336700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7ff8541114d0 con 0x7ff854101e40 2024-04-25T00:31:50.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.176+0000 7ff85c336700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7ff854111350 con 0x7ff8541061b0 2024-04-25T00:31:50.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.176+0000 7ff85c336700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7ff854111650 con 0x7ff854101480 2024-04-25T00:31:50.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.176+0000 7ff8598d1700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff854101e40 0x7ff85419daa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:50.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.176+0000 7ff8598d1700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff854101e40 0x7ff85419daa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:48776/0 (socket says 172.21.15.32:48776) 2024-04-25T00:31:50.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.176+0000 7ff8598d1700 1 -- 172.21.15.32:0/1660642590 learned_addr learned my addr 172.21.15.32:0/1660642590 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:50.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.176+0000 7ff85a0d2700 1 --2- 172.21.15.32:0/1660642590 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff854101480 0x7ff85419d560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:50.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.176+0000 7ff85a8d3700 1 --2- 172.21.15.32:0/1660642590 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff8541061b0 0x7ff854197630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:50.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.176+0000 7ff8598d1700 1 -- 172.21.15.32:0/1660642590 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff854101480 msgr2=0x7ff85419d560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:50.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.176+0000 7ff8598d1700 1 --2- 172.21.15.32:0/1660642590 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff854101480 0x7ff85419d560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:50.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.176+0000 7ff8598d1700 1 -- 172.21.15.32:0/1660642590 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff8541061b0 msgr2=0x7ff854197630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:50.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.176+0000 7ff8598d1700 1 --2- 172.21.15.32:0/1660642590 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff8541061b0 0x7ff854197630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:50.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.176+0000 7ff8598d1700 1 -- 172.21.15.32:0/1660642590 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff854197f10 con 0x7ff854101e40 2024-04-25T00:31:50.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.176+0000 7ff8598d1700 1 --2- 172.21.15.32:0/1660642590 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff854101e40 0x7ff85419daa0 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7ff84400c9b0 tx=0x7ff84400ccc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:50.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.176+0000 7ff84b7fe700 1 -- 172.21.15.32:0/1660642590 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff844007b00 con 0x7ff854101e40 2024-04-25T00:31:50.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.176+0000 7ff84b7fe700 1 -- 172.21.15.32:0/1660642590 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff84400ce50 con 0x7ff854101e40 2024-04-25T00:31:50.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.176+0000 7ff85c336700 1 -- 172.21.15.32:0/1660642590 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff854198200 con 0x7ff854101e40 2024-04-25T00:31:50.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.177+0000 7ff84b7fe700 1 -- 172.21.15.32:0/1660642590 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff844018660 con 0x7ff854101e40 2024-04-25T00:31:50.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.177+0000 7ff85c336700 1 -- 172.21.15.32:0/1660642590 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff8541a44e0 con 0x7ff854101e40 2024-04-25T00:31:50.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.177+0000 7ff85c336700 1 -- 172.21.15.32:0/1660642590 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7ff8540ffd50 con 0x7ff854101e40 2024-04-25T00:31:50.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.179+0000 7ff84b7fe700 1 -- 172.21.15.32:0/1660642590 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7ff84400f450 con 0x7ff854101e40 2024-04-25T00:31:50.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.179+0000 7ff84b7fe700 1 --2- 172.21.15.32:0/1660642590 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff8400a2cb0 0x7ff8400a5170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:50.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.179+0000 7ff85a0d2700 1 --2- 172.21.15.32:0/1660642590 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff8400a2cb0 0x7ff8400a5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:50.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.179+0000 7ff84b7fe700 1 -- 172.21.15.32:0/1660642590 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7ff8440cf890 con 0x7ff854101e40 2024-04-25T00:31:50.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.180+0000 7ff84b7fe700 1 --2- 172.21.15.32:0/1660642590 >> [v2:172.21.15.32:6802/3984695193,v1:172.21.15.32:6803/3984695193] conn(0x7ff8400ade10 0x7ff8400b02b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:50.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.180+0000 7ff84b7fe700 1 -- 172.21.15.32:0/1660642590 --> [v2:172.21.15.32:6802/3984695193,v1:172.21.15.32:6803/3984695193] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7ff8400b0980 con 0x7ff8400ade10 2024-04-25T00:31:50.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.180+0000 7ff84b7fe700 1 -- 172.21.15.32:0/1660642590 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_get_version_reply(handle=1 version=91) ==== 24+0+0 (secure 0 0 0) 0x7ff8440cfce0 con 0x7ff854101e40 2024-04-25T00:31:50.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.180+0000 7ff85a8d3700 1 --2- 172.21.15.32:0/1660642590 >> [v2:172.21.15.32:6802/3984695193,v1:172.21.15.32:6803/3984695193] conn(0x7ff8400ade10 0x7ff8400b02b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:50.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.180+0000 7ff85a0d2700 1 --2- 172.21.15.32:0/1660642590 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff8400a2cb0 0x7ff8400a5170 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7ff850000c00 tx=0x7ff850005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:50.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.180+0000 7ff85a8d3700 1 --2- 172.21.15.32:0/1660642590 >> [v2:172.21.15.32:6802/3984695193,v1:172.21.15.32:6803/3984695193] conn(0x7ff8400ade10 0x7ff8400b02b0 crc :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:50.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.181+0000 7ff84b7fe700 1 -- 172.21.15.32:0/1660642590 <== osd.0 v2:172.21.15.32:6802/3984695193 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7ff8400b0980 con 0x7ff8400ade10 2024-04-25T00:31:50.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.198+0000 7ff85c336700 1 -- 172.21.15.32:0/1660642590 --> [v2:172.21.15.32:6802/3984695193,v1:172.21.15.32:6803/3984695193] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7ff8541028a0 con 0x7ff8400ade10 2024-04-25T00:31:50.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.198+0000 7ff84b7fe700 1 -- 172.21.15.32:0/1660642590 <== osd.0 v2:172.21.15.32:6802/3984695193 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (crc 0 0 0) 0x7ff8541028a0 con 0x7ff8400ade10 2024-04-25T00:31:50.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.198+0000 7ff85c336700 1 -- 172.21.15.32:0/1660642590 >> [v2:172.21.15.32:6802/3984695193,v1:172.21.15.32:6803/3984695193] conn(0x7ff8400ade10 msgr2=0x7ff8400b02b0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:50.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.198+0000 7ff85c336700 1 --2- 172.21.15.32:0/1660642590 >> [v2:172.21.15.32:6802/3984695193,v1:172.21.15.32:6803/3984695193] conn(0x7ff8400ade10 0x7ff8400b02b0 crc :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:50.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.198+0000 7ff85c336700 1 -- 172.21.15.32:0/1660642590 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff8400a2cb0 msgr2=0x7ff8400a5170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:50.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.198+0000 7ff85c336700 1 --2- 172.21.15.32:0/1660642590 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff8400a2cb0 0x7ff8400a5170 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7ff850000c00 tx=0x7ff850005fb0 comp rx=0 tx=0).stop 2024-04-25T00:31:50.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.198+0000 7ff85c336700 1 -- 172.21.15.32:0/1660642590 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff854101e40 msgr2=0x7ff85419daa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:50.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.198+0000 7ff85c336700 1 --2- 172.21.15.32:0/1660642590 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff854101e40 0x7ff85419daa0 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7ff84400c9b0 tx=0x7ff84400ccc0 comp rx=0 tx=0).stop 2024-04-25T00:31:50.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.198+0000 7ff85a8d3700 1 -- 172.21.15.32:0/1660642590 reap_dead start 2024-04-25T00:31:50.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.198+0000 7ff85c336700 1 -- 172.21.15.32:0/1660642590 shutdown_connections 2024-04-25T00:31:50.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.198+0000 7ff85c336700 1 -- 172.21.15.32:0/1660642590 >> 172.21.15.32:0/1660642590 conn(0x7ff854076fa0 msgr2=0x7ff8540ff3b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:50.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.198+0000 7ff85c336700 1 -- 172.21.15.32:0/1660642590 shutdown_connections 2024-04-25T00:31:50.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:50.199+0000 7ff85c336700 1 -- 172.21.15.32:0/1660642590 wait complete. 2024-04-25T00:31:50.915 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:50 smithi032 ceph-mon[29357]: pgmap v324: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 7 op/s; 14 B/s, 0 objects/s recovering 2024-04-25T00:31:51.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:50 smithi121 ceph-mon[32231]: pgmap v324: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 7 op/s; 14 B/s, 0 objects/s recovering 2024-04-25T00:31:51.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:50 smithi130 ceph-mon[32525]: pgmap v324: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 7 op/s; 14 B/s, 0 objects/s recovering 2024-04-25T00:31:51.556 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.897+0000 7fd8154e4700 1 -- 172.21.15.32:0/2904241383 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd810101af0 msgr2=0x7fd81010dfe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:51.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.897+0000 7fd8154e4700 1 --2- 172.21.15.32:0/2904241383 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd810101af0 0x7fd81010dfe0 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7fd80800b0a0 tx=0x7fd80800b3b0 comp rx=0 tx=0).stop 2024-04-25T00:31:51.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.897+0000 7fd8154e4700 1 -- 172.21.15.32:0/2904241383 shutdown_connections 2024-04-25T00:31:51.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.897+0000 7fd8154e4700 1 --2- 172.21.15.32:0/2904241383 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd810101af0 0x7fd81010dfe0 unknown :-1 s=CLOSED pgs=301 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:51.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.897+0000 7fd8154e4700 1 --2- 172.21.15.32:0/2904241383 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd8101071a0 0x7fd810107580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:51.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.897+0000 7fd8154e4700 1 --2- 172.21.15.32:0/2904241383 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd810101130 0x7fd8101015b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:51.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.897+0000 7fd8154e4700 1 -- 172.21.15.32:0/2904241383 >> 172.21.15.32:0/2904241383 conn(0x7fd8100780e0 msgr2=0x7fd8100784f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:51.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.897+0000 7fd8154e4700 1 -- 172.21.15.32:0/2904241383 shutdown_connections 2024-04-25T00:31:51.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.897+0000 7fd8154e4700 1 -- 172.21.15.32:0/2904241383 wait complete. 2024-04-25T00:31:51.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.898+0000 7fd8154e4700 1 Processor -- start 2024-04-25T00:31:51.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.898+0000 7fd8154e4700 1 -- start start 2024-04-25T00:31:51.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.899+0000 7fd8154e4700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd810101130 0x7fd81019e680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:51.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.899+0000 7fd8154e4700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd810101af0 0x7fd81019ebc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:51.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.899+0000 7fd8154e4700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd8101071a0 0x7fd810198750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:51.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.899+0000 7fd8154e4700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fd810111410 con 0x7fd810101130 2024-04-25T00:31:51.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.899+0000 7fd8154e4700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fd810111290 con 0x7fd8101071a0 2024-04-25T00:31:51.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.899+0000 7fd8154e4700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fd810111590 con 0x7fd810101af0 2024-04-25T00:31:51.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.899+0000 7fd80effd700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd810101130 0x7fd81019e680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:51.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.899+0000 7fd80e7fc700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd810101af0 0x7fd81019ebc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:51.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.899+0000 7fd80effd700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd810101130 0x7fd81019e680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:48752/0 (socket says 172.21.15.32:48752) 2024-04-25T00:31:51.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.899+0000 7fd80effd700 1 -- 172.21.15.32:0/1384325179 learned_addr learned my addr 172.21.15.32:0/1384325179 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:51.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.899+0000 7fd80f7fe700 1 --2- 172.21.15.32:0/1384325179 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd8101071a0 0x7fd810198750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:51.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.899+0000 7fd80effd700 1 -- 172.21.15.32:0/1384325179 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd810101af0 msgr2=0x7fd81019ebc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:51.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.899+0000 7fd80effd700 1 --2- 172.21.15.32:0/1384325179 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd810101af0 0x7fd81019ebc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:51.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.899+0000 7fd80effd700 1 -- 172.21.15.32:0/1384325179 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd8101071a0 msgr2=0x7fd810198750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:51.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.899+0000 7fd80effd700 1 --2- 172.21.15.32:0/1384325179 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd8101071a0 0x7fd810198750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:51.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.899+0000 7fd80effd700 1 -- 172.21.15.32:0/1384325179 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd810199030 con 0x7fd810101130 2024-04-25T00:31:51.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.899+0000 7fd80f7fe700 1 --2- 172.21.15.32:0/1384325179 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd8101071a0 0x7fd810198750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:31:51.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.899+0000 7fd80e7fc700 1 --2- 172.21.15.32:0/1384325179 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd810101af0 0x7fd81019ebc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:31:51.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.900+0000 7fd80effd700 1 --2- 172.21.15.32:0/1384325179 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd810101130 0x7fd81019e680 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7fd80000e9a0 tx=0x7fd80000ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:51.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.900+0000 7fd7f7fff700 1 -- 172.21.15.32:0/1384325179 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd80000ca10 con 0x7fd810101130 2024-04-25T00:31:51.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.900+0000 7fd8154e4700 1 -- 172.21.15.32:0/1384325179 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd810199320 con 0x7fd810101130 2024-04-25T00:31:51.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.900+0000 7fd7f7fff700 1 -- 172.21.15.32:0/1384325179 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd80000cbb0 con 0x7fd810101130 2024-04-25T00:31:51.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.900+0000 7fd7f7fff700 1 -- 172.21.15.32:0/1384325179 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd800018680 con 0x7fd810101130 2024-04-25T00:31:51.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.900+0000 7fd8154e4700 1 -- 172.21.15.32:0/1384325179 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd8101a5600 con 0x7fd810101130 2024-04-25T00:31:51.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.901+0000 7fd8154e4700 1 -- 172.21.15.32:0/1384325179 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7fd810105260 con 0x7fd810101130 2024-04-25T00:31:51.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.902+0000 7fd7f7fff700 1 -- 172.21.15.32:0/1384325179 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7fd800010430 con 0x7fd810101130 2024-04-25T00:31:51.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.903+0000 7fd7f7fff700 1 --2- 172.21.15.32:0/1384325179 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fd7f80a2cb0 0x7fd7f80a5170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:51.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.903+0000 7fd80e7fc700 1 --2- 172.21.15.32:0/1384325179 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fd7f80a2cb0 0x7fd7f80a5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:51.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.903+0000 7fd7f7fff700 1 -- 172.21.15.32:0/1384325179 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7fd8000cff90 con 0x7fd810101130 2024-04-25T00:31:51.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.903+0000 7fd7f7fff700 1 --2- 172.21.15.32:0/1384325179 >> [v2:172.21.15.130:6800/1393844110,v1:172.21.15.130:6801/1393844110] conn(0x7fd7f80ade10 0x7fd7f80b02b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:51.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.903+0000 7fd7f7fff700 1 -- 172.21.15.32:0/1384325179 --> [v2:172.21.15.130:6800/1393844110,v1:172.21.15.130:6801/1393844110] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7fd800019ec0 con 0x7fd7f80ade10 2024-04-25T00:31:51.560 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.903+0000 7fd7f7fff700 1 -- 172.21.15.32:0/1384325179 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_get_version_reply(handle=1 version=91) ==== 24+0+0 (secure 0 0 0) 0x7fd8000d0340 con 0x7fd810101130 2024-04-25T00:31:51.560 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.903+0000 7fd80e7fc700 1 --2- 172.21.15.32:0/1384325179 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fd7f80a2cb0 0x7fd7f80a5170 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7fd810199f10 tx=0x7fd80400b410 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:51.560 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.903+0000 7fd80f7fe700 1 --2- 172.21.15.32:0/1384325179 >> [v2:172.21.15.130:6800/1393844110,v1:172.21.15.130:6801/1393844110] conn(0x7fd7f80ade10 0x7fd7f80b02b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:51.560 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.904+0000 7fd80f7fe700 1 --2- 172.21.15.32:0/1384325179 >> [v2:172.21.15.130:6800/1393844110,v1:172.21.15.130:6801/1393844110] conn(0x7fd7f80ade10 0x7fd7f80b02b0 crc :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:51.560 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.905+0000 7fd7f7fff700 1 -- 172.21.15.32:0/1384325179 <== osd.2 v2:172.21.15.130:6800/1393844110 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7fd800019ec0 con 0x7fd7f80ade10 2024-04-25T00:31:51.560 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.921+0000 7fd8154e4700 1 -- 172.21.15.32:0/1384325179 --> [v2:172.21.15.130:6800/1393844110,v1:172.21.15.130:6801/1393844110] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7fd810102850 con 0x7fd7f80ade10 2024-04-25T00:31:51.560 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.922+0000 7fd7f7fff700 1 -- 172.21.15.32:0/1384325179 <== osd.2 v2:172.21.15.130:6800/1393844110 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (crc 0 0 0) 0x7fd810102850 con 0x7fd7f80ade10 2024-04-25T00:31:51.560 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.922+0000 7fd8154e4700 1 -- 172.21.15.32:0/1384325179 >> [v2:172.21.15.130:6800/1393844110,v1:172.21.15.130:6801/1393844110] conn(0x7fd7f80ade10 msgr2=0x7fd7f80b02b0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:51.561 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.922+0000 7fd8154e4700 1 --2- 172.21.15.32:0/1384325179 >> [v2:172.21.15.130:6800/1393844110,v1:172.21.15.130:6801/1393844110] conn(0x7fd7f80ade10 0x7fd7f80b02b0 crc :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:51.561 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.922+0000 7fd8154e4700 1 -- 172.21.15.32:0/1384325179 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fd7f80a2cb0 msgr2=0x7fd7f80a5170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:51.561 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.922+0000 7fd8154e4700 1 --2- 172.21.15.32:0/1384325179 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fd7f80a2cb0 0x7fd7f80a5170 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7fd810199f10 tx=0x7fd80400b410 comp rx=0 tx=0).stop 2024-04-25T00:31:51.561 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.922+0000 7fd8154e4700 1 -- 172.21.15.32:0/1384325179 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd810101130 msgr2=0x7fd81019e680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:51.561 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.922+0000 7fd8154e4700 1 --2- 172.21.15.32:0/1384325179 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd810101130 0x7fd81019e680 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7fd80000e9a0 tx=0x7fd80000ecb0 comp rx=0 tx=0).stop 2024-04-25T00:31:51.561 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.922+0000 7fd80f7fe700 1 -- 172.21.15.32:0/1384325179 reap_dead start 2024-04-25T00:31:51.561 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.922+0000 7fd8154e4700 1 -- 172.21.15.32:0/1384325179 shutdown_connections 2024-04-25T00:31:51.561 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.922+0000 7fd8154e4700 1 -- 172.21.15.32:0/1384325179 >> 172.21.15.32:0/1384325179 conn(0x7fd8100780e0 msgr2=0x7fd8101003a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:51.561 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.922+0000 7fd8154e4700 1 -- 172.21.15.32:0/1384325179 shutdown_connections 2024-04-25T00:31:51.561 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:49.922+0000 7fd8154e4700 1 -- 172.21.15.32:0/1384325179 wait complete. 2024-04-25T00:31:52.604 INFO:teuthology.orchestra.run.smithi032.stdout:77309411399 2024-04-25T00:31:52.604 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-25T00:31:52.856 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:52 smithi032 ceph-mon[29357]: pgmap v325: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 4 op/s; 14 B/s, 0 objects/s recovering 2024-04-25T00:31:53.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:52 smithi121 ceph-mon[32231]: pgmap v325: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 4 op/s; 14 B/s, 0 objects/s recovering 2024-04-25T00:31:53.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:52 smithi130 ceph-mon[32525]: pgmap v325: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 4 op/s; 14 B/s, 0 objects/s recovering 2024-04-25T00:31:53.272 INFO:teuthology.orchestra.run.smithi032.stdout:34359738449 2024-04-25T00:31:53.272 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-25T00:31:53.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.654+0000 7fb1be54d700 1 -- 172.21.15.32:0/4246738885 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb1b8100820 msgr2=0x7fb1b8100c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:53.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.654+0000 7fb1be54d700 1 --2- 172.21.15.32:0/4246738885 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb1b8100820 0x7fb1b8100c80 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7fb1ac009990 tx=0x7fb1ac009ca0 comp rx=0 tx=0).stop 2024-04-25T00:31:53.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.654+0000 7fb1be54d700 1 -- 172.21.15.32:0/4246738885 shutdown_connections 2024-04-25T00:31:53.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.654+0000 7fb1be54d700 1 --2- 172.21.15.32:0/4246738885 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb1b81011c0 0x7fb1b810f100 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:53.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.654+0000 7fb1be54d700 1 --2- 172.21.15.32:0/4246738885 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb1b81082c0 0x7fb1b81086a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:53.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.654+0000 7fb1be54d700 1 --2- 172.21.15.32:0/4246738885 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb1b8100820 0x7fb1b8100c80 unknown :-1 s=CLOSED pgs=309 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:53.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.654+0000 7fb1be54d700 1 -- 172.21.15.32:0/4246738885 >> 172.21.15.32:0/4246738885 conn(0x7fb1b80777a0 msgr2=0x7fb1b80feb50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:53.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.654+0000 7fb1be54d700 1 -- 172.21.15.32:0/4246738885 shutdown_connections 2024-04-25T00:31:53.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.655+0000 7fb1be54d700 1 -- 172.21.15.32:0/4246738885 wait complete. 2024-04-25T00:31:53.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.655+0000 7fb1be54d700 1 Processor -- start 2024-04-25T00:31:53.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.655+0000 7fb1be54d700 1 -- start start 2024-04-25T00:31:53.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.655+0000 7fb1be54d700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb1b8100820 0x7fb1b819e6a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:53.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.656+0000 7fb1be54d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb1b81011c0 0x7fb1b819ebe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:53.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.656+0000 7fb1be54d700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb1b81082c0 0x7fb1b81987c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:53.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.656+0000 7fb1be54d700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fb1b8112520 con 0x7fb1b81082c0 2024-04-25T00:31:53.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.656+0000 7fb1be54d700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fb1b81123a0 con 0x7fb1b8100820 2024-04-25T00:31:53.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.656+0000 7fb1be54d700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fb1b81126a0 con 0x7fb1b81011c0 2024-04-25T00:31:53.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.656+0000 7fb1bcaea700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb1b81082c0 0x7fb1b81987c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:53.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.656+0000 7fb1bcaea700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb1b81082c0 0x7fb1b81987c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:56340/0 (socket says 172.21.15.32:56340) 2024-04-25T00:31:53.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.656+0000 7fb1bcaea700 1 -- 172.21.15.32:0/1377002709 learned_addr learned my addr 172.21.15.32:0/1377002709 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:53.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.656+0000 7fb1b7fff700 1 --2- 172.21.15.32:0/1377002709 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb1b8100820 0x7fb1b819e6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:53.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.656+0000 7fb1b77fe700 1 --2- 172.21.15.32:0/1377002709 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb1b81011c0 0x7fb1b819ebe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:53.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.656+0000 7fb1bcaea700 1 -- 172.21.15.32:0/1377002709 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb1b81011c0 msgr2=0x7fb1b819ebe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:53.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.656+0000 7fb1bcaea700 1 --2- 172.21.15.32:0/1377002709 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb1b81011c0 0x7fb1b819ebe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:53.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.656+0000 7fb1bcaea700 1 -- 172.21.15.32:0/1377002709 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb1b8100820 msgr2=0x7fb1b819e6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:53.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.656+0000 7fb1bcaea700 1 --2- 172.21.15.32:0/1377002709 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb1b8100820 0x7fb1b819e6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:53.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.656+0000 7fb1bcaea700 1 -- 172.21.15.32:0/1377002709 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb1b81990a0 con 0x7fb1b81082c0 2024-04-25T00:31:53.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.656+0000 7fb1bcaea700 1 --2- 172.21.15.32:0/1377002709 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb1b81082c0 0x7fb1b81987c0 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x7fb1a800c900 tx=0x7fb1a800cc10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:53.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.657+0000 7fb1b57fa700 1 -- 172.21.15.32:0/1377002709 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb1a80078d0 con 0x7fb1b81082c0 2024-04-25T00:31:53.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.657+0000 7fb1b57fa700 1 -- 172.21.15.32:0/1377002709 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb1a800f450 con 0x7fb1b81082c0 2024-04-25T00:31:53.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.657+0000 7fb1b57fa700 1 -- 172.21.15.32:0/1377002709 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb1a8018660 con 0x7fb1b81082c0 2024-04-25T00:31:53.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.657+0000 7fb1be54d700 1 -- 172.21.15.32:0/1377002709 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb1b8199390 con 0x7fb1b81082c0 2024-04-25T00:31:53.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.657+0000 7fb1be54d700 1 -- 172.21.15.32:0/1377002709 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb1b81a5620 con 0x7fb1b81082c0 2024-04-25T00:31:53.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.658+0000 7fb1be54d700 1 -- 172.21.15.32:0/1377002709 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7fb1b80629c0 con 0x7fb1b81082c0 2024-04-25T00:31:53.661 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.660+0000 7fb1b57fa700 1 -- 172.21.15.32:0/1377002709 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7fb1a8007a70 con 0x7fb1b81082c0 2024-04-25T00:31:53.661 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.660+0000 7fb1b57fa700 1 --2- 172.21.15.32:0/1377002709 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb1a00a2cb0 0x7fb1a00a5170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:53.661 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.660+0000 7fb1b57fa700 1 -- 172.21.15.32:0/1377002709 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7fb1a80cf780 con 0x7fb1b81082c0 2024-04-25T00:31:53.661 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.660+0000 7fb1b7fff700 1 --2- 172.21.15.32:0/1377002709 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb1a00a2cb0 0x7fb1a00a5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:53.661 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.660+0000 7fb1b57fa700 1 --2- 172.21.15.32:0/1377002709 >> [v2:172.21.15.130:6808/909656116,v1:172.21.15.130:6809/909656116] conn(0x7fb1a00ade10 0x7fb1a00b02b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:53.661 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.660+0000 7fb1b57fa700 1 -- 172.21.15.32:0/1377002709 --> [v2:172.21.15.130:6808/909656116,v1:172.21.15.130:6809/909656116] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7fb1a800de50 con 0x7fb1a00ade10 2024-04-25T00:31:53.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.660+0000 7fb1b57fa700 1 -- 172.21.15.32:0/1377002709 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_get_version_reply(handle=1 version=91) ==== 24+0+0 (secure 0 0 0) 0x7fb1a80cfbd0 con 0x7fb1b81082c0 2024-04-25T00:31:53.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.661+0000 7fb1b77fe700 1 --2- 172.21.15.32:0/1377002709 >> [v2:172.21.15.130:6808/909656116,v1:172.21.15.130:6809/909656116] conn(0x7fb1a00ade10 0x7fb1a00b02b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:53.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.661+0000 7fb1b7fff700 1 --2- 172.21.15.32:0/1377002709 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb1a00a2cb0 0x7fb1a00a5170 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7fb1b0009550 tx=0x7fb1b00091b0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:53.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.661+0000 7fb1b77fe700 1 --2- 172.21.15.32:0/1377002709 >> [v2:172.21.15.130:6808/909656116,v1:172.21.15.130:6809/909656116] conn(0x7fb1a00ade10 0x7fb1a00b02b0 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:53.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.662+0000 7fb1b57fa700 1 -- 172.21.15.32:0/1377002709 <== osd.5 v2:172.21.15.130:6808/909656116 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7fb1a800de50 con 0x7fb1a00ade10 2024-04-25T00:31:53.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.679+0000 7fb1be54d700 1 -- 172.21.15.32:0/1377002709 --> [v2:172.21.15.130:6808/909656116,v1:172.21.15.130:6809/909656116] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7fb1b8062e40 con 0x7fb1a00ade10 2024-04-25T00:31:53.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.680+0000 7fb1b57fa700 1 -- 172.21.15.32:0/1377002709 <== osd.5 v2:172.21.15.130:6808/909656116 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7fb1b8062e40 con 0x7fb1a00ade10 2024-04-25T00:31:53.682 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.680+0000 7fb1be54d700 1 -- 172.21.15.32:0/1377002709 >> [v2:172.21.15.130:6808/909656116,v1:172.21.15.130:6809/909656116] conn(0x7fb1a00ade10 msgr2=0x7fb1a00b02b0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:53.682 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.681+0000 7fb1be54d700 1 --2- 172.21.15.32:0/1377002709 >> [v2:172.21.15.130:6808/909656116,v1:172.21.15.130:6809/909656116] conn(0x7fb1a00ade10 0x7fb1a00b02b0 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:53.682 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.681+0000 7fb1be54d700 1 -- 172.21.15.32:0/1377002709 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb1a00a2cb0 msgr2=0x7fb1a00a5170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:53.682 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.681+0000 7fb1be54d700 1 --2- 172.21.15.32:0/1377002709 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb1a00a2cb0 0x7fb1a00a5170 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7fb1b0009550 tx=0x7fb1b00091b0 comp rx=0 tx=0).stop 2024-04-25T00:31:53.682 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.681+0000 7fb1be54d700 1 -- 172.21.15.32:0/1377002709 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb1b81082c0 msgr2=0x7fb1b81987c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:53.682 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.681+0000 7fb1be54d700 1 --2- 172.21.15.32:0/1377002709 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb1b81082c0 0x7fb1b81987c0 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x7fb1a800c900 tx=0x7fb1a800cc10 comp rx=0 tx=0).stop 2024-04-25T00:31:53.682 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.681+0000 7fb1bcaea700 1 -- 172.21.15.32:0/1377002709 reap_dead start 2024-04-25T00:31:53.682 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.681+0000 7fb1be54d700 1 -- 172.21.15.32:0/1377002709 shutdown_connections 2024-04-25T00:31:53.682 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.681+0000 7fb1be54d700 1 -- 172.21.15.32:0/1377002709 >> 172.21.15.32:0/1377002709 conn(0x7fb1b80777a0 msgr2=0x7fb1b80fee70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:53.682 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.681+0000 7fb1be54d700 1 -- 172.21.15.32:0/1377002709 shutdown_connections 2024-04-25T00:31:53.683 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:53.681+0000 7fb1be54d700 1 -- 172.21.15.32:0/1377002709 wait complete. 2024-04-25T00:31:54.194 INFO:teuthology.orchestra.run.smithi032.stdout:197568495668 2024-04-25T00:31:54.195 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-25T00:31:54.487 INFO:teuthology.orchestra.run.smithi032.stdout:253403070505 2024-04-25T00:31:54.488 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.8 2024-04-25T00:31:54.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:54 smithi032 ceph-mon[29357]: pgmap v326: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.5 KiB/s rd, 6 op/s; 12 B/s, 0 objects/s recovering 2024-04-25T00:31:55.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:54 smithi121 ceph-mon[32231]: pgmap v326: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.5 KiB/s rd, 6 op/s; 12 B/s, 0 objects/s recovering 2024-04-25T00:31:55.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:54 smithi130 ceph-mon[32525]: pgmap v326: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.5 KiB/s rd, 6 op/s; 12 B/s, 0 objects/s recovering 2024-04-25T00:31:55.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.763+0000 7fdd682cf700 1 -- 172.21.15.32:0/1560159585 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdd60101af0 msgr2=0x7fdd6010dfe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:55.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.763+0000 7fdd682cf700 1 --2- 172.21.15.32:0/1560159585 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdd60101af0 0x7fdd6010dfe0 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7fdd5800b0a0 tx=0x7fdd5800b3b0 comp rx=0 tx=0).stop 2024-04-25T00:31:55.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.763+0000 7fdd682cf700 1 -- 172.21.15.32:0/1560159585 shutdown_connections 2024-04-25T00:31:55.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.763+0000 7fdd682cf700 1 --2- 172.21.15.32:0/1560159585 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdd60101af0 0x7fdd6010dfe0 unknown :-1 s=CLOSED pgs=311 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:55.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.763+0000 7fdd682cf700 1 --2- 172.21.15.32:0/1560159585 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdd601071a0 0x7fdd60107580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:55.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.763+0000 7fdd682cf700 1 --2- 172.21.15.32:0/1560159585 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdd60101130 0x7fdd601015b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:55.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.763+0000 7fdd682cf700 1 -- 172.21.15.32:0/1560159585 >> 172.21.15.32:0/1560159585 conn(0x7fdd600780e0 msgr2=0x7fdd600784f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:55.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.763+0000 7fdd682cf700 1 -- 172.21.15.32:0/1560159585 shutdown_connections 2024-04-25T00:31:55.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.763+0000 7fdd682cf700 1 -- 172.21.15.32:0/1560159585 wait complete. 2024-04-25T00:31:55.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.764+0000 7fdd682cf700 1 Processor -- start 2024-04-25T00:31:55.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.764+0000 7fdd682cf700 1 -- start start 2024-04-25T00:31:55.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.764+0000 7fdd682cf700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdd60101130 0x7fdd60102e10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:55.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.764+0000 7fdd682cf700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdd60101af0 0x7fdd60103350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:55.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.764+0000 7fdd682cf700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdd601071a0 0x7fdd60103890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:55.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.764+0000 7fdd682cf700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fdd60111430 con 0x7fdd60101af0 2024-04-25T00:31:55.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.764+0000 7fdd682cf700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fdd601112b0 con 0x7fdd601071a0 2024-04-25T00:31:55.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.764+0000 7fdd682cf700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fdd601115b0 con 0x7fdd60101130 2024-04-25T00:31:55.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.765+0000 7fdd6586a700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdd60101af0 0x7fdd60103350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:55.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.765+0000 7fdd6586a700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdd60101af0 0x7fdd60103350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:56352/0 (socket says 172.21.15.32:56352) 2024-04-25T00:31:55.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.765+0000 7fdd6606b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdd60101130 0x7fdd60102e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:55.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.765+0000 7fdd6586a700 1 -- 172.21.15.32:0/3199319360 learned_addr learned my addr 172.21.15.32:0/3199319360 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:55.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.765+0000 7fdd6686c700 1 --2- 172.21.15.32:0/3199319360 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdd601071a0 0x7fdd60103890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:55.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.765+0000 7fdd6606b700 1 -- 172.21.15.32:0/3199319360 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdd601071a0 msgr2=0x7fdd60103890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:55.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.765+0000 7fdd6606b700 1 --2- 172.21.15.32:0/3199319360 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdd601071a0 0x7fdd60103890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:55.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.765+0000 7fdd6606b700 1 -- 172.21.15.32:0/3199319360 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdd60101af0 msgr2=0x7fdd60103350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:55.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.765+0000 7fdd6606b700 1 --2- 172.21.15.32:0/3199319360 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdd60101af0 0x7fdd60103350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:55.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.765+0000 7fdd6606b700 1 -- 172.21.15.32:0/3199319360 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fdd601a6e50 con 0x7fdd60101130 2024-04-25T00:31:55.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.765+0000 7fdd6606b700 1 --2- 172.21.15.32:0/3199319360 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdd60101130 0x7fdd60102e10 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7fdd5c00e9f0 tx=0x7fdd5c00ed00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:55.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.766+0000 7fdd577fe700 1 -- 172.21.15.32:0/3199319360 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd5c004220 con 0x7fdd60101130 2024-04-25T00:31:55.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.766+0000 7fdd577fe700 1 -- 172.21.15.32:0/3199319360 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fdd5c004d10 con 0x7fdd60101130 2024-04-25T00:31:55.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.766+0000 7fdd682cf700 1 -- 172.21.15.32:0/3199319360 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fdd601a7140 con 0x7fdd60101130 2024-04-25T00:31:55.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.766+0000 7fdd682cf700 1 -- 172.21.15.32:0/3199319360 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fdd601a7680 con 0x7fdd60101130 2024-04-25T00:31:55.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.766+0000 7fdd577fe700 1 -- 172.21.15.32:0/3199319360 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd5c005020 con 0x7fdd60101130 2024-04-25T00:31:55.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.767+0000 7fdd682cf700 1 -- 172.21.15.32:0/3199319360 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7fdd44001060 con 0x7fdd60101130 2024-04-25T00:31:55.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.768+0000 7fdd577fe700 1 -- 172.21.15.32:0/3199319360 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7fdd5c042b10 con 0x7fdd60101130 2024-04-25T00:31:55.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.769+0000 7fdd577fe700 1 --2- 172.21.15.32:0/3199319360 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdd4c0a2d30 0x7fdd4c0a51f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:55.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.769+0000 7fdd577fe700 1 -- 172.21.15.32:0/3199319360 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7fdd5c0cf4c0 con 0x7fdd60101130 2024-04-25T00:31:55.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.769+0000 7fdd6586a700 1 --2- 172.21.15.32:0/3199319360 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdd4c0a2d30 0x7fdd4c0a51f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:55.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.769+0000 7fdd577fe700 1 --2- 172.21.15.32:0/3199319360 >> [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050] conn(0x7fdd4c0ade90 0x7fdd4c0b0330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:55.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.770+0000 7fdd577fe700 1 -- 172.21.15.32:0/3199319360 --> [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7fdd5c005520 con 0x7fdd4c0ade90 2024-04-25T00:31:55.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.770+0000 7fdd577fe700 1 -- 172.21.15.32:0/3199319360 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_get_version_reply(handle=1 version=91) ==== 24+0+0 (secure 0 0 0) 0x7fdd5c0cf870 con 0x7fdd60101130 2024-04-25T00:31:55.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.770+0000 7fdd6686c700 1 --2- 172.21.15.32:0/3199319360 >> [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050] conn(0x7fdd4c0ade90 0x7fdd4c0b0330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:55.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.770+0000 7fdd6586a700 1 --2- 172.21.15.32:0/3199319360 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdd4c0a2d30 0x7fdd4c0a51f0 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7fdd600ff850 tx=0x7fdd5000a400 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:55.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.770+0000 7fdd6686c700 1 --2- 172.21.15.32:0/3199319360 >> [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050] conn(0x7fdd4c0ade90 0x7fdd4c0b0330 crc :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.9 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:55.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.771+0000 7fdd577fe700 1 -- 172.21.15.32:0/3199319360 <== osd.9 v2:172.21.15.32:6826/4290214050 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7fdd5c005520 con 0x7fdd4c0ade90 2024-04-25T00:31:55.788 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.787+0000 7fdd682cf700 1 -- 172.21.15.32:0/3199319360 --> [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7fdd44002e90 con 0x7fdd4c0ade90 2024-04-25T00:31:55.788 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.788+0000 7fdd577fe700 1 -- 172.21.15.32:0/3199319360 <== osd.9 v2:172.21.15.32:6826/4290214050 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7fdd44002e90 con 0x7fdd4c0ade90 2024-04-25T00:31:55.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.788+0000 7fdd682cf700 1 -- 172.21.15.32:0/3199319360 >> [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050] conn(0x7fdd4c0ade90 msgr2=0x7fdd4c0b0330 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:55.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.788+0000 7fdd682cf700 1 --2- 172.21.15.32:0/3199319360 >> [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050] conn(0x7fdd4c0ade90 0x7fdd4c0b0330 crc :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:55.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.788+0000 7fdd682cf700 1 -- 172.21.15.32:0/3199319360 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdd4c0a2d30 msgr2=0x7fdd4c0a51f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:55.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.788+0000 7fdd682cf700 1 --2- 172.21.15.32:0/3199319360 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdd4c0a2d30 0x7fdd4c0a51f0 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7fdd600ff850 tx=0x7fdd5000a400 comp rx=0 tx=0).stop 2024-04-25T00:31:55.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.788+0000 7fdd682cf700 1 -- 172.21.15.32:0/3199319360 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdd60101130 msgr2=0x7fdd60102e10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:55.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.788+0000 7fdd682cf700 1 --2- 172.21.15.32:0/3199319360 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdd60101130 0x7fdd60102e10 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7fdd5c00e9f0 tx=0x7fdd5c00ed00 comp rx=0 tx=0).stop 2024-04-25T00:31:55.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.788+0000 7fdd6686c700 1 -- 172.21.15.32:0/3199319360 reap_dead start 2024-04-25T00:31:55.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.788+0000 7fdd682cf700 1 -- 172.21.15.32:0/3199319360 shutdown_connections 2024-04-25T00:31:55.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.788+0000 7fdd682cf700 1 -- 172.21.15.32:0/3199319360 >> 172.21.15.32:0/3199319360 conn(0x7fdd600780e0 msgr2=0x7fdd601002f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:55.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.788+0000 7fdd682cf700 1 -- 172.21.15.32:0/3199319360 shutdown_connections 2024-04-25T00:31:55.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:55.788+0000 7fdd682cf700 1 -- 172.21.15.32:0/3199319360 wait complete. 2024-04-25T00:31:56.780 INFO:teuthology.orchestra.run.smithi032.stdout:163208757306 2024-04-25T00:31:56.781 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-25T00:31:56.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:56 smithi032 ceph-mon[29357]: pgmap v327: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s; 11 B/s, 0 objects/s recovering 2024-04-25T00:31:57.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:56 smithi121 ceph-mon[32231]: pgmap v327: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s; 11 B/s, 0 objects/s recovering 2024-04-25T00:31:57.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:56 smithi130 ceph-mon[32525]: pgmap v327: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s; 11 B/s, 0 objects/s recovering 2024-04-25T00:31:57.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.159+0000 7faa32ace700 1 -- 172.21.15.32:0/504474210 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7faa2c107570 msgr2=0x7faa2c10f140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:57.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.159+0000 7faa32ace700 1 --2- 172.21.15.32:0/504474210 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7faa2c107570 0x7faa2c10f140 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x7faa2400b0a0 tx=0x7faa2400b3b0 comp rx=0 tx=0).stop 2024-04-25T00:31:57.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.159+0000 7faa32ace700 1 -- 172.21.15.32:0/504474210 shutdown_connections 2024-04-25T00:31:57.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.159+0000 7faa32ace700 1 --2- 172.21.15.32:0/504474210 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7faa2c107570 0x7faa2c10f140 unknown :-1 s=CLOSED pgs=312 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:57.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.159+0000 7faa32ace700 1 --2- 172.21.15.32:0/504474210 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7faa2c1024e0 0x7faa2c1028c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:57.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.159+0000 7faa32ace700 1 --2- 172.21.15.32:0/504474210 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faa2c102e90 0x7faa2c106e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:57.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.159+0000 7faa32ace700 1 -- 172.21.15.32:0/504474210 >> 172.21.15.32:0/504474210 conn(0x7faa2c079390 msgr2=0x7faa2c0797a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:57.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.160+0000 7faa32ace700 1 -- 172.21.15.32:0/504474210 shutdown_connections 2024-04-25T00:31:57.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.160+0000 7faa32ace700 1 -- 172.21.15.32:0/504474210 wait complete. 2024-04-25T00:31:57.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.160+0000 7faa32ace700 1 Processor -- start 2024-04-25T00:31:57.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.160+0000 7faa32ace700 1 -- start start 2024-04-25T00:31:57.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.160+0000 7faa32ace700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faa2c1024e0 0x7faa2c19e6b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:57.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.160+0000 7faa32ace700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7faa2c102e90 0x7faa2c19ebf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:57.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.160+0000 7faa32ace700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7faa2c107570 0x7faa2c198830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:57.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.160+0000 7faa32ace700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7faa2c1125b0 con 0x7faa2c107570 2024-04-25T00:31:57.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.160+0000 7faa32ace700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7faa2c112430 con 0x7faa2c102e90 2024-04-25T00:31:57.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.160+0000 7faa32ace700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7faa2c112730 con 0x7faa2c1024e0 2024-04-25T00:31:57.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.160+0000 7faa3106b700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7faa2c107570 0x7faa2c198830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:57.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.161+0000 7faa3106b700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7faa2c107570 0x7faa2c198830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:56368/0 (socket says 172.21.15.32:56368) 2024-04-25T00:31:57.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.161+0000 7faa3106b700 1 -- 172.21.15.32:0/2546243346 learned_addr learned my addr 172.21.15.32:0/2546243346 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:57.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.161+0000 7faa3086a700 1 --2- 172.21.15.32:0/2546243346 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faa2c1024e0 0x7faa2c19e6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:57.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.161+0000 7faa2bfff700 1 --2- 172.21.15.32:0/2546243346 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7faa2c102e90 0x7faa2c19ebf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:57.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.161+0000 7faa3106b700 1 -- 172.21.15.32:0/2546243346 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faa2c1024e0 msgr2=0x7faa2c19e6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:57.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.161+0000 7faa3106b700 1 --2- 172.21.15.32:0/2546243346 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faa2c1024e0 0x7faa2c19e6b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:57.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.161+0000 7faa3106b700 1 -- 172.21.15.32:0/2546243346 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7faa2c102e90 msgr2=0x7faa2c19ebf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:57.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.161+0000 7faa3106b700 1 --2- 172.21.15.32:0/2546243346 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7faa2c102e90 0x7faa2c19ebf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:57.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.161+0000 7faa3106b700 1 -- 172.21.15.32:0/2546243346 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7faa2c199010 con 0x7faa2c107570 2024-04-25T00:31:57.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.161+0000 7faa2bfff700 1 --2- 172.21.15.32:0/2546243346 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7faa2c102e90 0x7faa2c19ebf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:31:57.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.161+0000 7faa3106b700 1 --2- 172.21.15.32:0/2546243346 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7faa2c107570 0x7faa2c198830 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7faa240062a0 tx=0x7faa24004e20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:57.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.161+0000 7faa29ffb700 1 -- 172.21.15.32:0/2546243346 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faa24021070 con 0x7faa2c107570 2024-04-25T00:31:57.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.161+0000 7faa29ffb700 1 -- 172.21.15.32:0/2546243346 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7faa24007c80 con 0x7faa2c107570 2024-04-25T00:31:57.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.161+0000 7faa32ace700 1 -- 172.21.15.32:0/2546243346 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7faa2c1992a0 con 0x7faa2c107570 2024-04-25T00:31:57.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.161+0000 7faa29ffb700 1 -- 172.21.15.32:0/2546243346 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faa2401d450 con 0x7faa2c107570 2024-04-25T00:31:57.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.161+0000 7faa32ace700 1 -- 172.21.15.32:0/2546243346 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7faa2c1a5510 con 0x7faa2c107570 2024-04-25T00:31:57.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.162+0000 7faa32ace700 1 -- 172.21.15.32:0/2546243346 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7faa2c0629c0 con 0x7faa2c107570 2024-04-25T00:31:57.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.164+0000 7faa29ffb700 1 -- 172.21.15.32:0/2546243346 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7faa24004000 con 0x7faa2c107570 2024-04-25T00:31:57.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.164+0000 7faa29ffb700 1 --2- 172.21.15.32:0/2546243346 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7faa1c0a2d30 0x7faa1c0a51f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:57.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.164+0000 7faa29ffb700 1 -- 172.21.15.32:0/2546243346 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7faa2400f020 con 0x7faa2c107570 2024-04-25T00:31:57.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.164+0000 7faa3086a700 1 --2- 172.21.15.32:0/2546243346 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7faa1c0a2d30 0x7faa1c0a51f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:57.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.164+0000 7faa29ffb700 1 --2- 172.21.15.32:0/2546243346 >> [v2:172.21.15.121:6816/3677614004,v1:172.21.15.121:6817/3677614004] conn(0x7faa1c0ade90 0x7faa1c0b0330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:57.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.165+0000 7faa29ffb700 1 -- 172.21.15.32:0/2546243346 --> [v2:172.21.15.121:6816/3677614004,v1:172.21.15.121:6817/3677614004] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7faa24008ea0 con 0x7faa1c0ade90 2024-04-25T00:31:57.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.165+0000 7faa29ffb700 1 -- 172.21.15.32:0/2546243346 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_get_version_reply(handle=1 version=91) ==== 24+0+0 (secure 0 0 0) 0x7faa240d1120 con 0x7faa2c107570 2024-04-25T00:31:57.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.165+0000 7faa2bfff700 1 --2- 172.21.15.32:0/2546243346 >> [v2:172.21.15.121:6816/3677614004,v1:172.21.15.121:6817/3677614004] conn(0x7faa1c0ade90 0x7faa1c0b0330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:57.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.165+0000 7faa3086a700 1 --2- 172.21.15.32:0/2546243346 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7faa1c0a2d30 0x7faa1c0a51f0 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7faa20005950 tx=0x7faa200058e0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:57.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.165+0000 7faa2bfff700 1 --2- 172.21.15.32:0/2546243346 >> [v2:172.21.15.121:6816/3677614004,v1:172.21.15.121:6817/3677614004] conn(0x7faa1c0ade90 0x7faa1c0b0330 crc :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.7 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:57.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.166+0000 7faa29ffb700 1 -- 172.21.15.32:0/2546243346 <== osd.7 v2:172.21.15.121:6816/3677614004 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7faa24008ea0 con 0x7faa1c0ade90 2024-04-25T00:31:57.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.192+0000 7faa32ace700 1 -- 172.21.15.32:0/2546243346 --> [v2:172.21.15.121:6816/3677614004,v1:172.21.15.121:6817/3677614004] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7faa2c062dc0 con 0x7faa1c0ade90 2024-04-25T00:31:57.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.193+0000 7faa29ffb700 1 -- 172.21.15.32:0/2546243346 <== osd.7 v2:172.21.15.121:6816/3677614004 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7faa2c062dc0 con 0x7faa1c0ade90 2024-04-25T00:31:57.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.193+0000 7faa32ace700 1 -- 172.21.15.32:0/2546243346 >> [v2:172.21.15.121:6816/3677614004,v1:172.21.15.121:6817/3677614004] conn(0x7faa1c0ade90 msgr2=0x7faa1c0b0330 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:57.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.193+0000 7faa32ace700 1 --2- 172.21.15.32:0/2546243346 >> [v2:172.21.15.121:6816/3677614004,v1:172.21.15.121:6817/3677614004] conn(0x7faa1c0ade90 0x7faa1c0b0330 crc :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:57.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.193+0000 7faa32ace700 1 -- 172.21.15.32:0/2546243346 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7faa1c0a2d30 msgr2=0x7faa1c0a51f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:57.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.193+0000 7faa32ace700 1 --2- 172.21.15.32:0/2546243346 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7faa1c0a2d30 0x7faa1c0a51f0 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7faa20005950 tx=0x7faa200058e0 comp rx=0 tx=0).stop 2024-04-25T00:31:57.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.193+0000 7faa32ace700 1 -- 172.21.15.32:0/2546243346 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7faa2c107570 msgr2=0x7faa2c198830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:57.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.193+0000 7faa32ace700 1 --2- 172.21.15.32:0/2546243346 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7faa2c107570 0x7faa2c198830 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7faa240062a0 tx=0x7faa24004e20 comp rx=0 tx=0).stop 2024-04-25T00:31:57.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.193+0000 7faa3106b700 1 -- 172.21.15.32:0/2546243346 reap_dead start 2024-04-25T00:31:57.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.194+0000 7faa32ace700 1 -- 172.21.15.32:0/2546243346 shutdown_connections 2024-04-25T00:31:57.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.194+0000 7faa32ace700 1 -- 172.21.15.32:0/2546243346 >> 172.21.15.32:0/2546243346 conn(0x7faa2c079390 msgr2=0x7faa2c1002f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:57.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.194+0000 7faa32ace700 1 -- 172.21.15.32:0/2546243346 shutdown_connections 2024-04-25T00:31:57.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.194+0000 7faa32ace700 1 -- 172.21.15.32:0/2546243346 wait complete. 2024-04-25T00:31:57.651 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.648+0000 7f71fc3eb700 1 -- 172.21.15.32:0/1931417111 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71f4102ec0 msgr2=0x7f71f4106eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:57.651 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.648+0000 7f71fc3eb700 1 --2- 172.21.15.32:0/1931417111 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71f4102ec0 0x7f71f4106eb0 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7f71e4009990 tx=0x7f71e4009ca0 comp rx=0 tx=0).stop 2024-04-25T00:31:57.651 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.648+0000 7f71fc3eb700 1 -- 172.21.15.32:0/1931417111 shutdown_connections 2024-04-25T00:31:57.651 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.648+0000 7f71fc3eb700 1 --2- 172.21.15.32:0/1931417111 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f71f4107590 0x7f71f410f160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:57.651 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.648+0000 7f71fc3eb700 1 --2- 172.21.15.32:0/1931417111 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f71f4102510 0x7f71f41028f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:57.651 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.648+0000 7f71fc3eb700 1 --2- 172.21.15.32:0/1931417111 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71f4102ec0 0x7f71f4106eb0 unknown :-1 s=CLOSED pgs=314 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:57.651 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.648+0000 7f71fc3eb700 1 -- 172.21.15.32:0/1931417111 >> 172.21.15.32:0/1931417111 conn(0x7f71f40793a0 msgr2=0x7f71f40797b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:57.651 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.648+0000 7f71fc3eb700 1 -- 172.21.15.32:0/1931417111 shutdown_connections 2024-04-25T00:31:57.651 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.648+0000 7f71fc3eb700 1 -- 172.21.15.32:0/1931417111 wait complete. 2024-04-25T00:31:57.651 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.649+0000 7f71fc3eb700 1 Processor -- start 2024-04-25T00:31:57.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.649+0000 7f71fc3eb700 1 -- start start 2024-04-25T00:31:57.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.649+0000 7f71fc3eb700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f71f4102510 0x7f71f41a0830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:57.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.649+0000 7f71fc3eb700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71f4102ec0 0x7f71f41a0d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:57.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.649+0000 7f71fc3eb700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f71f4107590 0x7f71f419a9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:57.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.649+0000 7f71fc3eb700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f71f4112530 con 0x7f71f4102ec0 2024-04-25T00:31:57.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.649+0000 7f71fc3eb700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f71f41123b0 con 0x7f71f4107590 2024-04-25T00:31:57.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.649+0000 7f71fc3eb700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f71f41126b0 con 0x7f71f4102510 2024-04-25T00:31:57.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.649+0000 7f71f9986700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71f4102ec0 0x7f71f41a0d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:57.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.649+0000 7f71f9986700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71f4102ec0 0x7f71f41a0d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:56384/0 (socket says 172.21.15.32:56384) 2024-04-25T00:31:57.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.649+0000 7f71f9986700 1 -- 172.21.15.32:0/1412235466 learned_addr learned my addr 172.21.15.32:0/1412235466 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:31:57.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.650+0000 7f71fa187700 1 --2- 172.21.15.32:0/1412235466 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f71f4102510 0x7f71f41a0830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:57.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.650+0000 7f71fa988700 1 --2- 172.21.15.32:0/1412235466 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f71f4107590 0x7f71f419a9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:57.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.650+0000 7f71f9986700 1 -- 172.21.15.32:0/1412235466 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f71f4102510 msgr2=0x7f71f41a0830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:57.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.650+0000 7f71f9986700 1 --2- 172.21.15.32:0/1412235466 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f71f4102510 0x7f71f41a0830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:57.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.650+0000 7f71f9986700 1 -- 172.21.15.32:0/1412235466 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f71f4107590 msgr2=0x7f71f419a9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:57.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.650+0000 7f71f9986700 1 --2- 172.21.15.32:0/1412235466 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f71f4107590 0x7f71f419a9f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:57.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.650+0000 7f71f9986700 1 -- 172.21.15.32:0/1412235466 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f71f419b150 con 0x7f71f4102ec0 2024-04-25T00:31:57.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.650+0000 7f71f9986700 1 --2- 172.21.15.32:0/1412235466 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71f4102ec0 0x7f71f41a0d70 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7f71e4000c00 tx=0x7f71e400bb60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:57.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.650+0000 7f71eb7fe700 1 -- 172.21.15.32:0/1412235466 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f71e4027070 con 0x7f71f4102ec0 2024-04-25T00:31:57.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.650+0000 7f71fc3eb700 1 -- 172.21.15.32:0/1412235466 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f71f419b3e0 con 0x7f71f4102ec0 2024-04-25T00:31:57.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.650+0000 7f71eb7fe700 1 -- 172.21.15.32:0/1412235466 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f71e4022470 con 0x7f71f4102ec0 2024-04-25T00:31:57.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.650+0000 7f71eb7fe700 1 -- 172.21.15.32:0/1412235466 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f71e400f6c0 con 0x7f71f4102ec0 2024-04-25T00:31:57.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.650+0000 7f71fc3eb700 1 -- 172.21.15.32:0/1412235466 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f71f41090a0 con 0x7f71f4102ec0 2024-04-25T00:31:57.654 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.651+0000 7f71fc3eb700 1 -- 172.21.15.32:0/1412235466 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f71f4075780 con 0x7f71f4102ec0 2024-04-25T00:31:57.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.652+0000 7f71eb7fe700 1 -- 172.21.15.32:0/1412235466 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f71e402a070 con 0x7f71f4102ec0 2024-04-25T00:31:57.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.653+0000 7f71eb7fe700 1 --2- 172.21.15.32:0/1412235466 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f71e00a2cb0 0x7f71e00a5170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:57.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.653+0000 7f71fa187700 1 --2- 172.21.15.32:0/1412235466 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f71e00a2cb0 0x7f71e00a5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:57.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.653+0000 7f71eb7fe700 1 -- 172.21.15.32:0/1412235466 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f71e40d0c20 con 0x7f71f4102ec0 2024-04-25T00:31:57.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.653+0000 7f71eb7fe700 1 --2- 172.21.15.32:0/1412235466 >> [v2:172.21.15.121:6824/3217408278,v1:172.21.15.121:6825/3217408278] conn(0x7f71e00ade10 0x7f71e00b02b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:31:57.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.653+0000 7f71eb7fe700 1 -- 172.21.15.32:0/1412235466 --> [v2:172.21.15.121:6824/3217408278,v1:172.21.15.121:6825/3217408278] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f71e400b0b0 con 0x7f71e00ade10 2024-04-25T00:31:57.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.653+0000 7f71eb7fe700 1 -- 172.21.15.32:0/1412235466 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_get_version_reply(handle=1 version=91) ==== 24+0+0 (secure 0 0 0) 0x7f71e40d1070 con 0x7f71f4102ec0 2024-04-25T00:31:57.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.653+0000 7f71fa187700 1 --2- 172.21.15.32:0/1412235466 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f71e00a2cb0 0x7f71e00a5170 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f71f0005950 tx=0x7f71f00058e0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:57.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.654+0000 7f71fa988700 1 --2- 172.21.15.32:0/1412235466 >> [v2:172.21.15.121:6824/3217408278,v1:172.21.15.121:6825/3217408278] conn(0x7f71e00ade10 0x7f71e00b02b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:31:57.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.654+0000 7f71fa988700 1 --2- 172.21.15.32:0/1412235466 >> [v2:172.21.15.121:6824/3217408278,v1:172.21.15.121:6825/3217408278] conn(0x7f71e00ade10 0x7f71e00b02b0 crc :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.10 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:31:57.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.655+0000 7f71eb7fe700 1 -- 172.21.15.32:0/1412235466 <== osd.10 v2:172.21.15.121:6824/3217408278 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f71e400b0b0 con 0x7f71e00ade10 2024-04-25T00:31:57.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.673+0000 7f71fc3eb700 1 -- 172.21.15.32:0/1412235466 --> [v2:172.21.15.121:6824/3217408278,v1:172.21.15.121:6825/3217408278] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f71f4103990 con 0x7f71e00ade10 2024-04-25T00:31:57.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.674+0000 7f71eb7fe700 1 -- 172.21.15.32:0/1412235466 <== osd.10 v2:172.21.15.121:6824/3217408278 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7f71f4103990 con 0x7f71e00ade10 2024-04-25T00:31:57.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.674+0000 7f71fc3eb700 1 -- 172.21.15.32:0/1412235466 >> [v2:172.21.15.121:6824/3217408278,v1:172.21.15.121:6825/3217408278] conn(0x7f71e00ade10 msgr2=0x7f71e00b02b0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:57.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.674+0000 7f71fc3eb700 1 --2- 172.21.15.32:0/1412235466 >> [v2:172.21.15.121:6824/3217408278,v1:172.21.15.121:6825/3217408278] conn(0x7f71e00ade10 0x7f71e00b02b0 crc :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:31:57.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.674+0000 7f71fc3eb700 1 -- 172.21.15.32:0/1412235466 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f71e00a2cb0 msgr2=0x7f71e00a5170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:57.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.674+0000 7f71fc3eb700 1 --2- 172.21.15.32:0/1412235466 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f71e00a2cb0 0x7f71e00a5170 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f71f0005950 tx=0x7f71f00058e0 comp rx=0 tx=0).stop 2024-04-25T00:31:57.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.674+0000 7f71fc3eb700 1 -- 172.21.15.32:0/1412235466 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71f4102ec0 msgr2=0x7f71f41a0d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:31:57.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.674+0000 7f71fc3eb700 1 --2- 172.21.15.32:0/1412235466 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71f4102ec0 0x7f71f41a0d70 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7f71e4000c00 tx=0x7f71e400bb60 comp rx=0 tx=0).stop 2024-04-25T00:31:57.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.674+0000 7f71fa988700 1 -- 172.21.15.32:0/1412235466 reap_dead start 2024-04-25T00:31:57.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.674+0000 7f71fc3eb700 1 -- 172.21.15.32:0/1412235466 shutdown_connections 2024-04-25T00:31:57.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.674+0000 7f71fc3eb700 1 -- 172.21.15.32:0/1412235466 >> 172.21.15.32:0/1412235466 conn(0x7f71f40793a0 msgr2=0x7f71f4100460 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:31:57.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.674+0000 7f71fc3eb700 1 -- 172.21.15.32:0/1412235466 shutdown_connections 2024-04-25T00:31:57.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:31:57.674+0000 7f71fc3eb700 1 -- 172.21.15.32:0/1412235466 wait complete. 2024-04-25T00:31:58.113 INFO:teuthology.orchestra.run.smithi032.stdout:219043332143 2024-04-25T00:31:58.114 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-25T00:31:59.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:31:58 smithi121 ceph-mon[32231]: pgmap v328: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s; 9 B/s, 0 objects/s recovering 2024-04-25T00:31:59.093 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:31:58 smithi032 ceph-mon[29357]: pgmap v328: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s; 9 B/s, 0 objects/s recovering 2024-04-25T00:31:59.123 INFO:teuthology.orchestra.run.smithi032.stdout:141733920830 2024-04-25T00:31:59.123 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-25T00:31:59.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:31:58 smithi130 ceph-mon[32525]: pgmap v328: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s; 9 B/s, 0 objects/s recovering 2024-04-25T00:31:59.616 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:31:59.621 INFO:teuthology.orchestra.run.smithi032.stdout:309237645345 2024-04-25T00:31:59.622 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.10 2024-04-25T00:32:00.440 INFO:teuthology.orchestra.run.smithi032.stdout:287762808869 2024-04-25T00:32:00.440 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.9 2024-04-25T00:32:00.824 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:32:00.831 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:32:00.831 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:32:00.836 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:32:00.840 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:32:00.845 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:32:00.853 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:32:01.014 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:00 smithi032 ceph-mon[29357]: pgmap v329: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 7 op/s; 9 B/s, 0 objects/s recovering 2024-04-25T00:32:01.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:00 smithi121 ceph-mon[32231]: pgmap v329: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 7 op/s; 9 B/s, 0 objects/s recovering 2024-04-25T00:32:01.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:00 smithi130 ceph-mon[32525]: pgmap v329: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 7 op/s; 9 B/s, 0 objects/s recovering 2024-04-25T00:32:03.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:02 smithi121 ceph-mon[32231]: pgmap v330: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:32:03.112 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:02 smithi032 ceph-mon[29357]: pgmap v330: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:32:03.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:02 smithi130 ceph-mon[32525]: pgmap v330: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:32:03.333 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:32:05.051 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:04 smithi032 ceph-mon[29357]: pgmap v331: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 6 op/s 2024-04-25T00:32:05.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:04 smithi121 ceph-mon[32231]: pgmap v331: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 6 op/s 2024-04-25T00:32:05.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:04 smithi130 ceph-mon[32525]: pgmap v331: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 6 op/s 2024-04-25T00:32:07.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:06 smithi121 ceph-mon[32231]: pgmap v332: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.6 KiB/s rd, 4 op/s 2024-04-25T00:32:07.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:06 smithi130 ceph-mon[32525]: pgmap v332: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.6 KiB/s rd, 4 op/s 2024-04-25T00:32:07.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:06 smithi032 ceph-mon[29357]: pgmap v332: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.6 KiB/s rd, 4 op/s 2024-04-25T00:32:07.975 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:07 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3491678311' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-25T00:32:08.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:07 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3491678311' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-25T00:32:08.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:07 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3491678311' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-25T00:32:08.975 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:08 smithi032 ceph-mon[29357]: pgmap v333: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.3 KiB/s rd, 4 op/s 2024-04-25T00:32:09.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:08 smithi121 ceph-mon[32231]: pgmap v333: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.3 KiB/s rd, 4 op/s 2024-04-25T00:32:09.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:08 smithi130 ceph-mon[32525]: pgmap v333: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.3 KiB/s rd, 4 op/s 2024-04-25T00:32:09.821 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.441+0000 7f546279c700 1 -- 172.21.15.32:0/507996781 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f545c101af0 msgr2=0x7f545c10dfe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:09.821 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.441+0000 7f546279c700 1 --2- 172.21.15.32:0/507996781 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f545c101af0 0x7f545c10dfe0 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7f545400b0a0 tx=0x7f545400b3b0 comp rx=0 tx=0).stop 2024-04-25T00:32:09.821 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.441+0000 7f546279c700 1 -- 172.21.15.32:0/507996781 shutdown_connections 2024-04-25T00:32:09.821 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.441+0000 7f546279c700 1 --2- 172.21.15.32:0/507996781 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f545c101af0 0x7f545c10dfe0 unknown :-1 s=CLOSED pgs=316 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:09.821 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.441+0000 7f546279c700 1 --2- 172.21.15.32:0/507996781 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f545c1071a0 0x7f545c107580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:09.821 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.441+0000 7f546279c700 1 --2- 172.21.15.32:0/507996781 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f545c101130 0x7f545c1015b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:09.821 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.441+0000 7f546279c700 1 -- 172.21.15.32:0/507996781 >> 172.21.15.32:0/507996781 conn(0x7f545c0780e0 msgr2=0x7f545c0784f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:09.821 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.441+0000 7f546279c700 1 -- 172.21.15.32:0/507996781 shutdown_connections 2024-04-25T00:32:09.821 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.441+0000 7f546279c700 1 -- 172.21.15.32:0/507996781 wait complete. 2024-04-25T00:32:09.821 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.442+0000 7f546279c700 1 Processor -- start 2024-04-25T00:32:09.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.442+0000 7f546279c700 1 -- start start 2024-04-25T00:32:09.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.442+0000 7f546279c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f545c101130 0x7f545c19d600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:09.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.442+0000 7f546279c700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f545c101af0 0x7f545c19db40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:09.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.442+0000 7f546279c700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f545c1071a0 0x7f545c1976d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:09.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.442+0000 7f546279c700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f545c111410 con 0x7f545c101af0 2024-04-25T00:32:09.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.442+0000 7f546279c700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f545c111290 con 0x7f545c1071a0 2024-04-25T00:32:09.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.442+0000 7f546279c700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f545c111590 con 0x7f545c101130 2024-04-25T00:32:09.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.442+0000 7f545b7fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f545c101af0 0x7f545c19db40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:09.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.443+0000 7f545b7fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f545c101af0 0x7f545c19db40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:52582/0 (socket says 172.21.15.32:52582) 2024-04-25T00:32:09.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.443+0000 7f545b7fe700 1 -- 172.21.15.32:0/3491678311 learned_addr learned my addr 172.21.15.32:0/3491678311 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:09.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.443+0000 7f545bfff700 1 --2- 172.21.15.32:0/3491678311 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f545c101130 0x7f545c19d600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:09.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.443+0000 7f5460d39700 1 --2- 172.21.15.32:0/3491678311 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f545c1071a0 0x7f545c1976d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:09.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.443+0000 7f545b7fe700 1 -- 172.21.15.32:0/3491678311 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f545c101130 msgr2=0x7f545c19d600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:09.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.443+0000 7f545b7fe700 1 --2- 172.21.15.32:0/3491678311 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f545c101130 0x7f545c19d600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:09.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.443+0000 7f545b7fe700 1 -- 172.21.15.32:0/3491678311 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f545c1071a0 msgr2=0x7f545c1976d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:09.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.443+0000 7f545b7fe700 1 --2- 172.21.15.32:0/3491678311 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f545c1071a0 0x7f545c1976d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:09.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.443+0000 7f545b7fe700 1 -- 172.21.15.32:0/3491678311 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f545c197fb0 con 0x7f545c101af0 2024-04-25T00:32:09.823 INFO:teuthology.orchestra.run.smithi032.stdout:163208757308 2024-04-25T00:32:09.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.443+0000 7f5460d39700 1 --2- 172.21.15.32:0/3491678311 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f545c1071a0 0x7f545c1976d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:32:09.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.443+0000 7f545b7fe700 1 --2- 172.21.15.32:0/3491678311 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f545c101af0 0x7f545c19db40 secure :-1 s=READY pgs=317 cs=0 l=1 rev1=1 crypto rx=0x7f5450009970 tx=0x7f5450009c80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:09.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.443+0000 7f54597fa700 1 -- 172.21.15.32:0/3491678311 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f545001c070 con 0x7f545c101af0 2024-04-25T00:32:09.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.443+0000 7f54597fa700 1 -- 172.21.15.32:0/3491678311 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f545000d260 con 0x7f545c101af0 2024-04-25T00:32:09.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.443+0000 7f54597fa700 1 -- 172.21.15.32:0/3491678311 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5450017ad0 con 0x7f545c101af0 2024-04-25T00:32:09.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.443+0000 7f546279c700 1 -- 172.21.15.32:0/3491678311 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f545c198240 con 0x7f545c101af0 2024-04-25T00:32:09.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.443+0000 7f546279c700 1 -- 172.21.15.32:0/3491678311 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f545c06a0e0 con 0x7f545c101af0 2024-04-25T00:32:09.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.444+0000 7f546279c700 1 -- 172.21.15.32:0/3491678311 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f545c102850 con 0x7f545c101af0 2024-04-25T00:32:09.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.445+0000 7f54597fa700 1 -- 172.21.15.32:0/3491678311 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f545001f070 con 0x7f545c101af0 2024-04-25T00:32:09.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.446+0000 7f54597fa700 1 --2- 172.21.15.32:0/3491678311 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f54440a2d30 0x7f54440a51f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:09.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.446+0000 7f545bfff700 1 --2- 172.21.15.32:0/3491678311 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f54440a2d30 0x7f54440a51f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:09.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.446+0000 7f54597fa700 1 -- 172.21.15.32:0/3491678311 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f54500d0650 con 0x7f545c101af0 2024-04-25T00:32:09.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.447+0000 7f545bfff700 1 --2- 172.21.15.32:0/3491678311 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f54440a2d30 0x7f54440a51f0 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f544c007b60 tx=0x7f544c0073d0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:09.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.451+0000 7f54597fa700 1 -- 172.21.15.32:0/3491678311 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f545009d090 con 0x7f545c101af0 2024-04-25T00:32:09.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.668+0000 7f546279c700 1 -- 172.21.15.32:0/3491678311 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 5} v 0) -- 0x7f545c198c00 con 0x7f545c101af0 2024-04-25T00:32:09.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.669+0000 7f54597fa700 1 -- 172.21.15.32:0/3491678311 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 5}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f545009c990 con 0x7f545c101af0 2024-04-25T00:32:09.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.671+0000 7f546279c700 1 -- 172.21.15.32:0/3491678311 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f54440a2d30 msgr2=0x7f54440a51f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:09.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.671+0000 7f546279c700 1 --2- 172.21.15.32:0/3491678311 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f54440a2d30 0x7f54440a51f0 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f544c007b60 tx=0x7f544c0073d0 comp rx=0 tx=0).stop 2024-04-25T00:32:09.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.671+0000 7f546279c700 1 -- 172.21.15.32:0/3491678311 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f545c101af0 msgr2=0x7f545c19db40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:09.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.671+0000 7f546279c700 1 --2- 172.21.15.32:0/3491678311 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f545c101af0 0x7f545c19db40 secure :-1 s=READY pgs=317 cs=0 l=1 rev1=1 crypto rx=0x7f5450009970 tx=0x7f5450009c80 comp rx=0 tx=0).stop 2024-04-25T00:32:09.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.671+0000 7f546279c700 1 -- 172.21.15.32:0/3491678311 shutdown_connections 2024-04-25T00:32:09.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.671+0000 7f546279c700 1 --2- 172.21.15.32:0/3491678311 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f54440a2d30 0x7f54440a51f0 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:09.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.671+0000 7f546279c700 1 --2- 172.21.15.32:0/3491678311 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f545c1071a0 0x7f545c1976d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:09.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.671+0000 7f546279c700 1 --2- 172.21.15.32:0/3491678311 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f545c101130 0x7f545c19d600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:09.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.671+0000 7f546279c700 1 --2- 172.21.15.32:0/3491678311 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f545c101af0 0x7f545c19db40 unknown :-1 s=CLOSED pgs=317 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:09.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.671+0000 7f546279c700 1 -- 172.21.15.32:0/3491678311 >> 172.21.15.32:0/3491678311 conn(0x7f545c0780e0 msgr2=0x7f545c0ffbe0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:09.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.671+0000 7f546279c700 1 -- 172.21.15.32:0/3491678311 shutdown_connections 2024-04-25T00:32:09.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:07.671+0000 7f546279c700 1 -- 172.21.15.32:0/3491678311 wait complete. 2024-04-25T00:32:10.474 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757306 got 163208757308 for osd.5 2024-04-25T00:32:10.475 DEBUG:teuthology.parallel:result is None 2024-04-25T00:32:10.845 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.548+0000 7f9da8746700 1 -- 172.21.15.32:0/872402897 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9da006b550 msgr2=0x7f9da0106ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:10.845 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.548+0000 7f9da8746700 1 --2- 172.21.15.32:0/872402897 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9da006b550 0x7f9da0106ed0 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7f9d900099c0 tx=0x7f9d90009cd0 comp rx=0 tx=0).stop 2024-04-25T00:32:10.845 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.548+0000 7f9da8746700 1 -- 172.21.15.32:0/872402897 shutdown_connections 2024-04-25T00:32:10.845 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.548+0000 7f9da8746700 1 --2- 172.21.15.32:0/872402897 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9da0107600 0x7f9da010f1a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:10.845 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.548+0000 7f9da8746700 1 --2- 172.21.15.32:0/872402897 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9da006aba0 0x7f9da006af80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:10.845 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.548+0000 7f9da8746700 1 --2- 172.21.15.32:0/872402897 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9da006b550 0x7f9da0106ed0 unknown :-1 s=CLOSED pgs=318 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:10.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.548+0000 7f9da8746700 1 -- 172.21.15.32:0/872402897 >> 172.21.15.32:0/872402897 conn(0x7f9da0077e90 msgr2=0x7f9da00782a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:10.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.548+0000 7f9da8746700 1 -- 172.21.15.32:0/872402897 shutdown_connections 2024-04-25T00:32:10.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.548+0000 7f9da8746700 1 -- 172.21.15.32:0/872402897 wait complete. 2024-04-25T00:32:10.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.548+0000 7f9da8746700 1 Processor -- start 2024-04-25T00:32:10.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.548+0000 7f9da8746700 1 -- start start 2024-04-25T00:32:10.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.549+0000 7f9da8746700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9da006aba0 0x7f9da0101520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:10.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.549+0000 7f9da8746700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9da006b550 0x7f9da0101a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:10.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.549+0000 7f9da8746700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9da0107600 0x7f9da0101fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:10.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.549+0000 7f9da8746700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f9da0112570 con 0x7f9da006b550 2024-04-25T00:32:10.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.549+0000 7f9da8746700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f9da01123f0 con 0x7f9da0107600 2024-04-25T00:32:10.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.549+0000 7f9da8746700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f9da01126f0 con 0x7f9da006aba0 2024-04-25T00:32:10.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.549+0000 7f9da5ce1700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9da006b550 0x7f9da0101a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:10.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.549+0000 7f9da5ce1700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9da006b550 0x7f9da0101a60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:52588/0 (socket says 172.21.15.32:52588) 2024-04-25T00:32:10.847 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.549+0000 7f9da5ce1700 1 -- 172.21.15.32:0/1728186229 learned_addr learned my addr 172.21.15.32:0/1728186229 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:10.847 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.549+0000 7f9da64e2700 1 --2- 172.21.15.32:0/1728186229 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9da006aba0 0x7f9da0101520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:10.847 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.549+0000 7f9da6ce3700 1 --2- 172.21.15.32:0/1728186229 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9da0107600 0x7f9da0101fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:10.847 INFO:teuthology.orchestra.run.smithi032.stdout:77309411403 2024-04-25T00:32:10.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.549+0000 7f9da5ce1700 1 -- 172.21.15.32:0/1728186229 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9da006aba0 msgr2=0x7f9da0101520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:10.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.549+0000 7f9da5ce1700 1 --2- 172.21.15.32:0/1728186229 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9da006aba0 0x7f9da0101520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:10.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.549+0000 7f9da5ce1700 1 -- 172.21.15.32:0/1728186229 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9da0107600 msgr2=0x7f9da0101fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:10.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.549+0000 7f9da5ce1700 1 --2- 172.21.15.32:0/1728186229 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9da0107600 0x7f9da0101fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:10.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.549+0000 7f9da5ce1700 1 -- 172.21.15.32:0/1728186229 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9da01a6f30 con 0x7f9da006b550 2024-04-25T00:32:10.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.549+0000 7f9da5ce1700 1 --2- 172.21.15.32:0/1728186229 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9da006b550 0x7f9da0101a60 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7f9d900098e0 tx=0x7f9d900047b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:10.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.549+0000 7f9d977fe700 1 -- 172.21.15.32:0/1728186229 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9d90027070 con 0x7f9da006b550 2024-04-25T00:32:10.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.550+0000 7f9da8746700 1 -- 172.21.15.32:0/1728186229 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9da01a71c0 con 0x7f9da006b550 2024-04-25T00:32:10.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.550+0000 7f9d977fe700 1 -- 172.21.15.32:0/1728186229 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9d90003e40 con 0x7f9da006b550 2024-04-25T00:32:10.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.550+0000 7f9d977fe700 1 -- 172.21.15.32:0/1728186229 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9d90022c00 con 0x7f9da006b550 2024-04-25T00:32:10.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.550+0000 7f9da8746700 1 -- 172.21.15.32:0/1728186229 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9da01a7670 con 0x7f9da006b550 2024-04-25T00:32:10.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.550+0000 7f9da8746700 1 -- 172.21.15.32:0/1728186229 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9da0068f20 con 0x7f9da006b550 2024-04-25T00:32:10.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.552+0000 7f9d977fe700 1 -- 172.21.15.32:0/1728186229 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f9d9002a070 con 0x7f9da006b550 2024-04-25T00:32:10.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.553+0000 7f9d977fe700 1 --2- 172.21.15.32:0/1728186229 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f9d8c0a2d30 0x7f9d8c0a51f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:10.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.553+0000 7f9d977fe700 1 -- 172.21.15.32:0/1728186229 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f9d900d0bf0 con 0x7f9da006b550 2024-04-25T00:32:10.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.553+0000 7f9da64e2700 1 --2- 172.21.15.32:0/1728186229 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f9d8c0a2d30 0x7f9d8c0a51f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:10.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.554+0000 7f9da64e2700 1 --2- 172.21.15.32:0/1728186229 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f9d8c0a2d30 0x7f9d8c0a51f0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f9d9c005950 tx=0x7f9d9c00b410 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:10.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.558+0000 7f9d977fe700 1 -- 172.21.15.32:0/1728186229 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f9d90031070 con 0x7f9da006b550 2024-04-25T00:32:10.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.766+0000 7f9da8746700 1 -- 172.21.15.32:0/1728186229 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7f9da002db30 con 0x7f9da006b550 2024-04-25T00:32:10.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.768+0000 7f9d977fe700 1 -- 172.21.15.32:0/1728186229 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f9d90013070 con 0x7f9da006b550 2024-04-25T00:32:10.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.770+0000 7f9da8746700 1 -- 172.21.15.32:0/1728186229 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f9d8c0a2d30 msgr2=0x7f9d8c0a51f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:10.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.770+0000 7f9da8746700 1 --2- 172.21.15.32:0/1728186229 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f9d8c0a2d30 0x7f9d8c0a51f0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f9d9c005950 tx=0x7f9d9c00b410 comp rx=0 tx=0).stop 2024-04-25T00:32:10.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.770+0000 7f9da8746700 1 -- 172.21.15.32:0/1728186229 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9da006b550 msgr2=0x7f9da0101a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:10.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.770+0000 7f9da8746700 1 --2- 172.21.15.32:0/1728186229 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9da006b550 0x7f9da0101a60 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7f9d900098e0 tx=0x7f9d900047b0 comp rx=0 tx=0).stop 2024-04-25T00:32:10.850 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.770+0000 7f9da8746700 1 -- 172.21.15.32:0/1728186229 shutdown_connections 2024-04-25T00:32:10.850 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.770+0000 7f9da8746700 1 --2- 172.21.15.32:0/1728186229 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f9d8c0a2d30 0x7f9d8c0a51f0 unknown :-1 s=CLOSED pgs=208 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:10.850 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.770+0000 7f9da8746700 1 --2- 172.21.15.32:0/1728186229 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9da0107600 0x7f9da0101fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:10.850 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.770+0000 7f9da8746700 1 --2- 172.21.15.32:0/1728186229 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9da006aba0 0x7f9da0101520 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:10.850 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.770+0000 7f9da8746700 1 --2- 172.21.15.32:0/1728186229 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9da006b550 0x7f9da0101a60 unknown :-1 s=CLOSED pgs=319 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:10.850 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.770+0000 7f9da8746700 1 -- 172.21.15.32:0/1728186229 >> 172.21.15.32:0/1728186229 conn(0x7f9da0077e90 msgr2=0x7f9da0100c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:10.850 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.770+0000 7f9da8746700 1 -- 172.21.15.32:0/1728186229 shutdown_connections 2024-04-25T00:32:10.850 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:09.771+0000 7f9da8746700 1 -- 172.21.15.32:0/1728186229 wait complete. 2024-04-25T00:32:10.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:10 smithi032 ceph-mon[29357]: pgmap v334: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.6 KiB/s rd, 7 op/s 2024-04-25T00:32:10.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:10 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1728186229' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-25T00:32:11.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:10 smithi121 ceph-mon[32231]: pgmap v334: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.6 KiB/s rd, 7 op/s 2024-04-25T00:32:11.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:10 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1728186229' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-25T00:32:11.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:10 smithi130 ceph-mon[32525]: pgmap v334: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.6 KiB/s rd, 7 op/s 2024-04-25T00:32:11.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:10 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1728186229' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-25T00:32:11.535 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411399 got 77309411403 for osd.2 2024-04-25T00:32:11.535 DEBUG:teuthology.parallel:result is None 2024-04-25T00:32:11.851 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:32:12.877 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:12 smithi032 ceph-mon[29357]: pgmap v335: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.3 KiB/s rd, 4 op/s 2024-04-25T00:32:13.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:12 smithi121 ceph-mon[32231]: pgmap v335: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.3 KiB/s rd, 4 op/s 2024-04-25T00:32:13.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:12 smithi130 ceph-mon[32525]: pgmap v335: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.3 KiB/s rd, 4 op/s 2024-04-25T00:32:13.966 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.965+0000 7f2d03f85700 1 -- 172.21.15.32:0/183021881 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2cfc108410 msgr2=0x7f2cfc1087f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:13.966 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.965+0000 7f2d03f85700 1 --2- 172.21.15.32:0/183021881 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2cfc108410 0x7f2cfc1087f0 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7f2cf8009990 tx=0x7f2cf8009ca0 comp rx=0 tx=0).stop 2024-04-25T00:32:13.966 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.965+0000 7f2d03f85700 1 -- 172.21.15.32:0/183021881 shutdown_connections 2024-04-25T00:32:13.966 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.965+0000 7f2d03f85700 1 --2- 172.21.15.32:0/183021881 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2cfc101620 0x7f2cfc10f120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:13.966 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.965+0000 7f2d03f85700 1 --2- 172.21.15.32:0/183021881 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2cfc108410 0x7f2cfc1087f0 unknown :-1 s=CLOSED pgs=322 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:13.966 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.965+0000 7f2d03f85700 1 --2- 172.21.15.32:0/183021881 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f2cfc100c60 0x7f2cfc1010e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:13.966 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.965+0000 7f2d03f85700 1 -- 172.21.15.32:0/183021881 >> 172.21.15.32:0/183021881 conn(0x7f2cfc0763a0 msgr2=0x7f2cfc0767b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:13.966 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.965+0000 7f2d03f85700 1 -- 172.21.15.32:0/183021881 shutdown_connections 2024-04-25T00:32:13.966 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.965+0000 7f2d03f85700 1 -- 172.21.15.32:0/183021881 wait complete. 2024-04-25T00:32:13.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.966+0000 7f2d03f85700 1 Processor -- start 2024-04-25T00:32:13.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.966+0000 7f2d03f85700 1 -- start start 2024-04-25T00:32:13.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.966+0000 7f2d03f85700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2cfc100c60 0x7f2cfc101e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:13.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.966+0000 7f2d03f85700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f2cfc101620 0x7f2cfc102380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:13.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.966+0000 7f2d03f85700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2cfc108410 0x7f2cfc1028c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:13.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.966+0000 7f2d03f85700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f2cfc112530 con 0x7f2cfc108410 2024-04-25T00:32:13.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.966+0000 7f2d03f85700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f2cfc1123b0 con 0x7f2cfc101620 2024-04-25T00:32:13.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.966+0000 7f2d03f85700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f2cfc1126b0 con 0x7f2cfc100c60 2024-04-25T00:32:13.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.966+0000 7f2d02522700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2cfc108410 0x7f2cfc1028c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:13.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.966+0000 7f2d02522700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2cfc108410 0x7f2cfc1028c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40262/0 (socket says 172.21.15.32:40262) 2024-04-25T00:32:13.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.966+0000 7f2d02522700 1 -- 172.21.15.32:0/3352862211 learned_addr learned my addr 172.21.15.32:0/3352862211 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:13.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.966+0000 7f2d01d21700 1 --2- 172.21.15.32:0/3352862211 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2cfc100c60 0x7f2cfc101e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:13.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.966+0000 7f2d01520700 1 --2- 172.21.15.32:0/3352862211 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f2cfc101620 0x7f2cfc102380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:13.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.966+0000 7f2d02522700 1 -- 172.21.15.32:0/3352862211 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2cfc100c60 msgr2=0x7f2cfc101e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:13.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.966+0000 7f2d02522700 1 --2- 172.21.15.32:0/3352862211 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2cfc100c60 0x7f2cfc101e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:13.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.966+0000 7f2d02522700 1 -- 172.21.15.32:0/3352862211 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f2cfc101620 msgr2=0x7f2cfc102380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:13.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.966+0000 7f2d02522700 1 --2- 172.21.15.32:0/3352862211 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f2cfc101620 0x7f2cfc102380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:13.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.966+0000 7f2d02522700 1 -- 172.21.15.32:0/3352862211 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2cfc1a6e90 con 0x7f2cfc108410 2024-04-25T00:32:13.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.966+0000 7f2d01520700 1 --2- 172.21.15.32:0/3352862211 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f2cfc101620 0x7f2cfc102380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:32:13.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.967+0000 7f2d02522700 1 --2- 172.21.15.32:0/3352862211 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2cfc108410 0x7f2cfc1028c0 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7f2cf4007ee0 tx=0x7f2cf400d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:13.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.967+0000 7f2cf2ffd700 1 -- 172.21.15.32:0/3352862211 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2cf400dbd0 con 0x7f2cfc108410 2024-04-25T00:32:13.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.967+0000 7f2d03f85700 1 -- 172.21.15.32:0/3352862211 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2cfc1a7180 con 0x7f2cfc108410 2024-04-25T00:32:13.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.967+0000 7f2cf2ffd700 1 -- 172.21.15.32:0/3352862211 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f2cf400dd70 con 0x7f2cfc108410 2024-04-25T00:32:13.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.967+0000 7f2cf2ffd700 1 -- 172.21.15.32:0/3352862211 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2cf4005a60 con 0x7f2cfc108410 2024-04-25T00:32:13.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.967+0000 7f2d03f85700 1 -- 172.21.15.32:0/3352862211 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2cfc1a76c0 con 0x7f2cfc108410 2024-04-25T00:32:13.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.968+0000 7f2d03f85700 1 -- 172.21.15.32:0/3352862211 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2cfc10a850 con 0x7f2cfc108410 2024-04-25T00:32:13.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.969+0000 7f2cf2ffd700 1 -- 172.21.15.32:0/3352862211 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f2cf401a020 con 0x7f2cfc108410 2024-04-25T00:32:13.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.970+0000 7f2cf2ffd700 1 --2- 172.21.15.32:0/3352862211 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f2ce80a2cb0 0x7f2ce80a5170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:13.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.970+0000 7f2d01d21700 1 --2- 172.21.15.32:0/3352862211 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f2ce80a2cb0 0x7f2ce80a5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:13.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.970+0000 7f2cf2ffd700 1 -- 172.21.15.32:0/3352862211 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f2cf40cec00 con 0x7f2cfc108410 2024-04-25T00:32:13.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.970+0000 7f2d01d21700 1 --2- 172.21.15.32:0/3352862211 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f2ce80a2cb0 0x7f2ce80a5170 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f2cf800b5c0 tx=0x7f2cf8005960 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:13.974 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.974+0000 7f2cf2ffd700 1 -- 172.21.15.32:0/3352862211 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f2cf4010070 con 0x7f2cfc108410 2024-04-25T00:32:14.055 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:13 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2143959787' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-25T00:32:14.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:13 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2143959787' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-25T00:32:14.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:13 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2143959787' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-25T00:32:14.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:14.182+0000 7f2d03f85700 1 -- 172.21.15.32:0/3352862211 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 11} v 0) -- 0x7f2cfc10a230 con 0x7f2cfc108410 2024-04-25T00:32:14.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:14.183+0000 7f2cf2ffd700 1 -- 172.21.15.32:0/3352862211 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 11}]=0 v0) ==== 75+0+13 (secure 0 0 0) 0x7f2cf409b6a0 con 0x7f2cfc108410 2024-04-25T00:32:14.184 INFO:teuthology.orchestra.run.smithi032.stdout:343597383710 2024-04-25T00:32:14.185 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:14.185+0000 7f2d03f85700 1 -- 172.21.15.32:0/3352862211 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f2ce80a2cb0 msgr2=0x7f2ce80a5170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:14.185 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:14.185+0000 7f2d03f85700 1 --2- 172.21.15.32:0/3352862211 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f2ce80a2cb0 0x7f2ce80a5170 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f2cf800b5c0 tx=0x7f2cf8005960 comp rx=0 tx=0).stop 2024-04-25T00:32:14.185 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:14.185+0000 7f2d03f85700 1 -- 172.21.15.32:0/3352862211 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2cfc108410 msgr2=0x7f2cfc1028c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:14.185 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:14.185+0000 7f2d03f85700 1 --2- 172.21.15.32:0/3352862211 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2cfc108410 0x7f2cfc1028c0 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7f2cf4007ee0 tx=0x7f2cf400d3b0 comp rx=0 tx=0).stop 2024-04-25T00:32:14.186 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:14.185+0000 7f2d03f85700 1 -- 172.21.15.32:0/3352862211 shutdown_connections 2024-04-25T00:32:14.186 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:14.185+0000 7f2d03f85700 1 --2- 172.21.15.32:0/3352862211 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f2ce80a2cb0 0x7f2ce80a5170 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:14.186 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:14.185+0000 7f2d03f85700 1 --2- 172.21.15.32:0/3352862211 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2cfc108410 0x7f2cfc1028c0 unknown :-1 s=CLOSED pgs=323 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:14.186 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:14.185+0000 7f2d03f85700 1 --2- 172.21.15.32:0/3352862211 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2cfc100c60 0x7f2cfc101e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:14.186 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:14.185+0000 7f2d03f85700 1 --2- 172.21.15.32:0/3352862211 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f2cfc101620 0x7f2cfc102380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:14.186 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:14.185+0000 7f2d03f85700 1 -- 172.21.15.32:0/3352862211 >> 172.21.15.32:0/3352862211 conn(0x7f2cfc0763a0 msgr2=0x7f2cfc0ff7b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:14.186 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:14.185+0000 7f2d03f85700 1 -- 172.21.15.32:0/3352862211 shutdown_connections 2024-04-25T00:32:14.186 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:14.185+0000 7f2d03f85700 1 -- 172.21.15.32:0/3352862211 wait complete. 2024-04-25T00:32:15.035 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:14 smithi032 ceph-mon[29357]: pgmap v336: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 6 op/s 2024-04-25T00:32:15.035 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:14 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3352862211' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 11}]: dispatch 2024-04-25T00:32:15.045 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:32:15.045 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:32:15.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.057+0000 7fc450b0a700 1 -- 172.21.15.32:0/1557995451 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc44c101ae0 msgr2=0x7fc44c10dfd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:15.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.057+0000 7fc450b0a700 1 --2- 172.21.15.32:0/1557995451 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc44c101ae0 0x7fc44c10dfd0 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 crypto rx=0x7fc44400b0a0 tx=0x7fc44400b3b0 comp rx=0 tx=0).stop 2024-04-25T00:32:15.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.057+0000 7fc450b0a700 1 -- 172.21.15.32:0/1557995451 shutdown_connections 2024-04-25T00:32:15.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.057+0000 7fc450b0a700 1 --2- 172.21.15.32:0/1557995451 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc44c101ae0 0x7fc44c10dfd0 unknown :-1 s=CLOSED pgs=324 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:15.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.057+0000 7fc450b0a700 1 --2- 172.21.15.32:0/1557995451 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc44c107190 0x7fc44c107570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:15.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.057+0000 7fc450b0a700 1 --2- 172.21.15.32:0/1557995451 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc44c101120 0x7fc44c1015a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:15.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.057+0000 7fc450b0a700 1 -- 172.21.15.32:0/1557995451 >> 172.21.15.32:0/1557995451 conn(0x7fc44c0780a0 msgr2=0x7fc44c0784b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:15.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.058+0000 7fc450b0a700 1 -- 172.21.15.32:0/1557995451 shutdown_connections 2024-04-25T00:32:15.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.058+0000 7fc450b0a700 1 -- 172.21.15.32:0/1557995451 wait complete. 2024-04-25T00:32:15.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.059+0000 7fc450b0a700 1 Processor -- start 2024-04-25T00:32:15.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.059+0000 7fc450b0a700 1 -- start start 2024-04-25T00:32:15.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.059+0000 7fc450b0a700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc44c101120 0x7fc44c19ad10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:15.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.059+0000 7fc450b0a700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc44c101ae0 0x7fc44c19b250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:15.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.059+0000 7fc450b0a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc44c19f5e0 0x7fc44c19fa60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:15.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.059+0000 7fc450b0a700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fc44c1125a0 con 0x7fc44c101120 2024-04-25T00:32:15.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.059+0000 7fc450b0a700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fc44c112420 con 0x7fc44c101ae0 2024-04-25T00:32:15.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.059+0000 7fc450b0a700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fc44c112720 con 0x7fc44c19f5e0 2024-04-25T00:32:15.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.060+0000 7fc44a59c700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc44c101120 0x7fc44c19ad10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:15.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.060+0000 7fc44a59c700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc44c101120 0x7fc44c19ad10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40280/0 (socket says 172.21.15.32:40280) 2024-04-25T00:32:15.062 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.060+0000 7fc44a59c700 1 -- 172.21.15.32:0/1950735309 learned_addr learned my addr 172.21.15.32:0/1950735309 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:15.062 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.060+0000 7fc44ad9d700 1 --2- 172.21.15.32:0/1950735309 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc44c19f5e0 0x7fc44c19fa60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:15.062 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.060+0000 7fc449d9b700 1 --2- 172.21.15.32:0/1950735309 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc44c101ae0 0x7fc44c19b250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:15.062 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.060+0000 7fc44a59c700 1 -- 172.21.15.32:0/1950735309 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc44c19f5e0 msgr2=0x7fc44c19fa60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:15.062 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.060+0000 7fc44a59c700 1 --2- 172.21.15.32:0/1950735309 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc44c19f5e0 0x7fc44c19fa60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:15.062 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.060+0000 7fc44a59c700 1 -- 172.21.15.32:0/1950735309 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc44c101ae0 msgr2=0x7fc44c19b250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:15.062 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.060+0000 7fc44a59c700 1 --2- 172.21.15.32:0/1950735309 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc44c101ae0 0x7fc44c19b250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:15.062 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.060+0000 7fc44a59c700 1 -- 172.21.15.32:0/1950735309 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc44c1a0100 con 0x7fc44c101120 2024-04-25T00:32:15.062 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.060+0000 7fc449d9b700 1 --2- 172.21.15.32:0/1950735309 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc44c101ae0 0x7fc44c19b250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:32:15.062 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.060+0000 7fc44a59c700 1 --2- 172.21.15.32:0/1950735309 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc44c101120 0x7fc44c19ad10 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7fc44000b500 tx=0x7fc44000b810 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:15.062 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.061+0000 7fc43b7fe700 1 -- 172.21.15.32:0/1950735309 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc440015020 con 0x7fc44c101120 2024-04-25T00:32:15.063 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.061+0000 7fc450b0a700 1 -- 172.21.15.32:0/1950735309 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc44c1a03f0 con 0x7fc44c101120 2024-04-25T00:32:15.063 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.061+0000 7fc450b0a700 1 -- 172.21.15.32:0/1950735309 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc44c1a7bd0 con 0x7fc44c101120 2024-04-25T00:32:15.063 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.061+0000 7fc43b7fe700 1 -- 172.21.15.32:0/1950735309 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc440011d10 con 0x7fc44c101120 2024-04-25T00:32:15.063 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.062+0000 7fc43b7fe700 1 -- 172.21.15.32:0/1950735309 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc440011a30 con 0x7fc44c101120 2024-04-25T00:32:15.063 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.063+0000 7fc450b0a700 1 -- 172.21.15.32:0/1950735309 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc42c005330 con 0x7fc44c101120 2024-04-25T00:32:15.065 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.064+0000 7fc43b7fe700 1 -- 172.21.15.32:0/1950735309 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7fc440018020 con 0x7fc44c101120 2024-04-25T00:32:15.065 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.065+0000 7fc43b7fe700 1 --2- 172.21.15.32:0/1950735309 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc4340a2cb0 0x7fc4340a5170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:15.066 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.065+0000 7fc43b7fe700 1 -- 172.21.15.32:0/1950735309 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7fc4400ceae0 con 0x7fc44c101120 2024-04-25T00:32:15.066 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.065+0000 7fc449d9b700 1 --2- 172.21.15.32:0/1950735309 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc4340a2cb0 0x7fc4340a5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:15.066 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.066+0000 7fc449d9b700 1 --2- 172.21.15.32:0/1950735309 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc4340a2cb0 0x7fc4340a5170 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7fc44c078a10 tx=0x7fc43c005f50 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:15.073 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.073+0000 7fc43b7fe700 1 -- 172.21.15.32:0/1950735309 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fc44001f030 con 0x7fc44c101120 2024-04-25T00:32:15.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:14 smithi121 ceph-mon[32231]: pgmap v336: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 6 op/s 2024-04-25T00:32:15.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:14 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3352862211' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 11}]: dispatch 2024-04-25T00:32:15.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:14 smithi130 ceph-mon[32525]: pgmap v336: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 6 op/s 2024-04-25T00:32:15.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:14 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3352862211' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 11}]: dispatch 2024-04-25T00:32:15.285 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.284+0000 7fc450b0a700 1 -- 172.21.15.32:0/1950735309 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7fc42c0058f0 con 0x7fc44c101120 2024-04-25T00:32:15.285 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.285+0000 7fc43b7fe700 1 -- 172.21.15.32:0/1950735309 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fc44009b690 con 0x7fc44c101120 2024-04-25T00:32:15.285 INFO:teuthology.orchestra.run.smithi032.stdout:34359738454 2024-04-25T00:32:15.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.287+0000 7fc450b0a700 1 -- 172.21.15.32:0/1950735309 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc4340a2cb0 msgr2=0x7fc4340a5170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:15.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.287+0000 7fc450b0a700 1 --2- 172.21.15.32:0/1950735309 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc4340a2cb0 0x7fc4340a5170 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7fc44c078a10 tx=0x7fc43c005f50 comp rx=0 tx=0).stop 2024-04-25T00:32:15.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.287+0000 7fc450b0a700 1 -- 172.21.15.32:0/1950735309 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc44c101120 msgr2=0x7fc44c19ad10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:15.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.287+0000 7fc450b0a700 1 --2- 172.21.15.32:0/1950735309 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc44c101120 0x7fc44c19ad10 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7fc44000b500 tx=0x7fc44000b810 comp rx=0 tx=0).stop 2024-04-25T00:32:15.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.287+0000 7fc450b0a700 1 -- 172.21.15.32:0/1950735309 shutdown_connections 2024-04-25T00:32:15.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.287+0000 7fc450b0a700 1 --2- 172.21.15.32:0/1950735309 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc4340a2cb0 0x7fc4340a5170 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:15.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.287+0000 7fc450b0a700 1 --2- 172.21.15.32:0/1950735309 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc44c19f5e0 0x7fc44c19fa60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:15.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.287+0000 7fc450b0a700 1 --2- 172.21.15.32:0/1950735309 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc44c101120 0x7fc44c19ad10 unknown :-1 s=CLOSED pgs=325 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:15.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.287+0000 7fc450b0a700 1 --2- 172.21.15.32:0/1950735309 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc44c101ae0 0x7fc44c19b250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:15.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.287+0000 7fc450b0a700 1 -- 172.21.15.32:0/1950735309 >> 172.21.15.32:0/1950735309 conn(0x7fc44c0780a0 msgr2=0x7fc44c10c5c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:15.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.287+0000 7fc450b0a700 1 -- 172.21.15.32:0/1950735309 shutdown_connections 2024-04-25T00:32:15.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:15.287+0000 7fc450b0a700 1 -- 172.21.15.32:0/1950735309 wait complete. 2024-04-25T00:32:15.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.893+0000 7f72a6c55700 1 -- 172.21.15.32:0/3116105275 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f72a006a980 msgr2=0x7f72a006ade0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:15.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.893+0000 7f72a6c55700 1 --2- 172.21.15.32:0/3116105275 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f72a006a980 0x7f72a006ade0 secure :-1 s=READY pgs=320 cs=0 l=1 rev1=1 crypto rx=0x7f7288009990 tx=0x7f7288009ca0 comp rx=0 tx=0).stop 2024-04-25T00:32:15.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.893+0000 7f72a6c55700 1 -- 172.21.15.32:0/3116105275 shutdown_connections 2024-04-25T00:32:15.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.893+0000 7f72a6c55700 1 --2- 172.21.15.32:0/3116105275 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f72a006b320 0x7f72a010f0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:15.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.893+0000 7f72a6c55700 1 --2- 172.21.15.32:0/3116105275 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f72a01082b0 0x7f72a0108690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:15.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.893+0000 7f72a6c55700 1 --2- 172.21.15.32:0/3116105275 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f72a006a980 0x7f72a006ade0 unknown :-1 s=CLOSED pgs=320 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:15.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.893+0000 7f72a6c55700 1 -- 172.21.15.32:0/3116105275 >> 172.21.15.32:0/3116105275 conn(0x7f72a0078100 msgr2=0x7f72a0078510 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:15.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.893+0000 7f72a6c55700 1 -- 172.21.15.32:0/3116105275 shutdown_connections 2024-04-25T00:32:15.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.893+0000 7f72a6c55700 1 -- 172.21.15.32:0/3116105275 wait complete. 2024-04-25T00:32:15.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.894+0000 7f72a6c55700 1 Processor -- start 2024-04-25T00:32:15.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.894+0000 7f72a6c55700 1 -- start start 2024-04-25T00:32:15.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.894+0000 7f72a6c55700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f72a006a980 0x7f72a019e6a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:15.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.894+0000 7f72a6c55700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f72a006b320 0x7f72a019ebe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:15.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.894+0000 7f72a6c55700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f72a01082b0 0x7f72a0198770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:15.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.894+0000 7f72a6c55700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f72a0112510 con 0x7f72a01082b0 2024-04-25T00:32:15.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.894+0000 7f72a6c55700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f72a0112390 con 0x7f72a006a980 2024-04-25T00:32:15.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.894+0000 7f72a6c55700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f72a0112690 con 0x7f72a006b320 2024-04-25T00:32:15.661 INFO:teuthology.orchestra.run.smithi032.stdout:107374182470 2024-04-25T00:32:15.661 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.894+0000 7f72a51f2700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f72a01082b0 0x7f72a0198770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:15.661 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.894+0000 7f72a51f2700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f72a01082b0 0x7f72a0198770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40250/0 (socket says 172.21.15.32:40250) 2024-04-25T00:32:15.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.894+0000 7f72a51f2700 1 -- 172.21.15.32:0/2143959787 learned_addr learned my addr 172.21.15.32:0/2143959787 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:15.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.894+0000 7f72a49f1700 1 --2- 172.21.15.32:0/2143959787 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f72a006a980 0x7f72a019e6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:15.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.894+0000 7f729ffff700 1 --2- 172.21.15.32:0/2143959787 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f72a006b320 0x7f72a019ebe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:15.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.894+0000 7f72a51f2700 1 -- 172.21.15.32:0/2143959787 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f72a006b320 msgr2=0x7f72a019ebe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:15.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.894+0000 7f72a51f2700 1 --2- 172.21.15.32:0/2143959787 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f72a006b320 0x7f72a019ebe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:15.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.894+0000 7f72a51f2700 1 -- 172.21.15.32:0/2143959787 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f72a006a980 msgr2=0x7f72a019e6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:15.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.894+0000 7f72a51f2700 1 --2- 172.21.15.32:0/2143959787 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f72a006a980 0x7f72a019e6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:15.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.894+0000 7f72a51f2700 1 -- 172.21.15.32:0/2143959787 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f72a0199050 con 0x7f72a01082b0 2024-04-25T00:32:15.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.895+0000 7f72a51f2700 1 --2- 172.21.15.32:0/2143959787 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f72a01082b0 0x7f72a0198770 secure :-1 s=READY pgs=321 cs=0 l=1 rev1=1 crypto rx=0x7f729800c900 tx=0x7f729800cc10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:15.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.895+0000 7f729dffb700 1 -- 172.21.15.32:0/2143959787 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f72980078d0 con 0x7f72a01082b0 2024-04-25T00:32:15.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.895+0000 7f72a6c55700 1 -- 172.21.15.32:0/2143959787 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f72a0199340 con 0x7f72a01082b0 2024-04-25T00:32:15.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.895+0000 7f729dffb700 1 -- 172.21.15.32:0/2143959787 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f729800f450 con 0x7f72a01082b0 2024-04-25T00:32:15.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.895+0000 7f729dffb700 1 -- 172.21.15.32:0/2143959787 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7298018660 con 0x7f72a01082b0 2024-04-25T00:32:15.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.895+0000 7f72a6c55700 1 -- 172.21.15.32:0/2143959787 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f72a01a5620 con 0x7f72a01082b0 2024-04-25T00:32:15.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.896+0000 7f72a6c55700 1 -- 172.21.15.32:0/2143959787 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f72a0103960 con 0x7f72a01082b0 2024-04-25T00:32:15.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.897+0000 7f729dffb700 1 -- 172.21.15.32:0/2143959787 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f7298007a70 con 0x7f72a01082b0 2024-04-25T00:32:15.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.898+0000 7f729dffb700 1 --2- 172.21.15.32:0/2143959787 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f728c0a2cb0 0x7f728c0a5170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:15.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.898+0000 7f72a49f1700 1 --2- 172.21.15.32:0/2143959787 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f728c0a2cb0 0x7f728c0a5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:15.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.898+0000 7f729dffb700 1 -- 172.21.15.32:0/2143959787 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f72980cf3c0 con 0x7f72a01082b0 2024-04-25T00:32:15.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.898+0000 7f72a49f1700 1 --2- 172.21.15.32:0/2143959787 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f728c0a2cb0 0x7f728c0a5170 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f7290009550 tx=0x7f72900091b0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:15.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:12.902+0000 7f729dffb700 1 -- 172.21.15.32:0/2143959787 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f7298014030 con 0x7f72a01082b0 2024-04-25T00:32:15.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.112+0000 7f72a6c55700 1 -- 172.21.15.32:0/2143959787 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7f72a0199ca0 con 0x7f72a01082b0 2024-04-25T00:32:15.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.112+0000 7f729dffb700 1 -- 172.21.15.32:0/2143959787 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f729809be00 con 0x7f72a01082b0 2024-04-25T00:32:15.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.114+0000 7f72a6c55700 1 -- 172.21.15.32:0/2143959787 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f728c0a2cb0 msgr2=0x7f728c0a5170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:15.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.114+0000 7f72a6c55700 1 --2- 172.21.15.32:0/2143959787 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f728c0a2cb0 0x7f728c0a5170 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f7290009550 tx=0x7f72900091b0 comp rx=0 tx=0).stop 2024-04-25T00:32:15.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.114+0000 7f72a6c55700 1 -- 172.21.15.32:0/2143959787 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f72a01082b0 msgr2=0x7f72a0198770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:15.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.114+0000 7f72a6c55700 1 --2- 172.21.15.32:0/2143959787 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f72a01082b0 0x7f72a0198770 secure :-1 s=READY pgs=321 cs=0 l=1 rev1=1 crypto rx=0x7f729800c900 tx=0x7f729800cc10 comp rx=0 tx=0).stop 2024-04-25T00:32:15.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.114+0000 7f72a6c55700 1 -- 172.21.15.32:0/2143959787 shutdown_connections 2024-04-25T00:32:15.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.114+0000 7f72a6c55700 1 --2- 172.21.15.32:0/2143959787 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f728c0a2cb0 0x7f728c0a5170 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:15.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.114+0000 7f72a6c55700 1 --2- 172.21.15.32:0/2143959787 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f72a01082b0 0x7f72a0198770 unknown :-1 s=CLOSED pgs=321 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:15.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.114+0000 7f72a6c55700 1 --2- 172.21.15.32:0/2143959787 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f72a006a980 0x7f72a019e6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:15.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.114+0000 7f72a6c55700 1 --2- 172.21.15.32:0/2143959787 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f72a006b320 0x7f72a019ebe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:15.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.114+0000 7f72a6c55700 1 -- 172.21.15.32:0/2143959787 >> 172.21.15.32:0/2143959787 conn(0x7f72a0078100 msgr2=0x7f72a00ffd30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:15.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.114+0000 7f72a6c55700 1 -- 172.21.15.32:0/2143959787 shutdown_connections 2024-04-25T00:32:15.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:13.114+0000 7f72a6c55700 1 -- 172.21.15.32:0/2143959787 wait complete. 2024-04-25T00:32:15.945 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:15 smithi032 ceph-mon[29357]: pgmap v337: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-25T00:32:15.945 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:15 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1950735309' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-25T00:32:16.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:15 smithi121 ceph-mon[32231]: pgmap v337: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-25T00:32:16.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:15 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1950735309' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-25T00:32:16.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:15 smithi130 ceph-mon[32525]: pgmap v337: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-25T00:32:16.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:15 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1950735309' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-25T00:32:16.503 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182465 got 107374182470 for osd.3 2024-04-25T00:32:16.503 DEBUG:teuthology.parallel:result is None 2024-04-25T00:32:16.954 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738449 got 34359738454 for osd.0 2024-04-25T00:32:16.954 DEBUG:teuthology.parallel:result is None 2024-04-25T00:32:18.074 INFO:tasks.cephadm.ceph_manager.ceph:need seq 343597383703 got 343597383710 for osd.11 2024-04-25T00:32:18.074 DEBUG:teuthology.parallel:result is None 2024-04-25T00:32:18.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:17 smithi032 ceph-mon[29357]: pgmap v338: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:32:18.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:17 smithi121 ceph-mon[32231]: pgmap v338: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:32:18.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:17 smithi130 ceph-mon[32525]: pgmap v338: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:32:18.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.959+0000 7f7a08092700 1 -- 172.21.15.32:0/1240333900 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7a0006a930 msgr2=0x7f7a0006ad90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:18.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.959+0000 7f7a08092700 1 --2- 172.21.15.32:0/1240333900 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7a0006a930 0x7f7a0006ad90 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 crypto rx=0x7f79f0009990 tx=0x7f79f0009ca0 comp rx=0 tx=0).stop 2024-04-25T00:32:18.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.959+0000 7f7a08092700 1 -- 172.21.15.32:0/1240333900 shutdown_connections 2024-04-25T00:32:18.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.959+0000 7f7a08092700 1 --2- 172.21.15.32:0/1240333900 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7a0006b2d0 0x7f7a0010f120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:18.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.959+0000 7f7a08092700 1 --2- 172.21.15.32:0/1240333900 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7a001082e0 0x7f7a001086c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:18.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.959+0000 7f7a08092700 1 --2- 172.21.15.32:0/1240333900 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7a0006a930 0x7f7a0006ad90 unknown :-1 s=CLOSED pgs=328 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:18.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.959+0000 7f7a08092700 1 -- 172.21.15.32:0/1240333900 >> 172.21.15.32:0/1240333900 conn(0x7f7a00078170 msgr2=0x7f7a00078580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:18.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.959+0000 7f7a08092700 1 -- 172.21.15.32:0/1240333900 shutdown_connections 2024-04-25T00:32:18.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.959+0000 7f7a08092700 1 -- 172.21.15.32:0/1240333900 wait complete. 2024-04-25T00:32:18.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.959+0000 7f7a08092700 1 Processor -- start 2024-04-25T00:32:18.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.959+0000 7f7a08092700 1 -- start start 2024-04-25T00:32:18.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.960+0000 7f7a08092700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7a0006a930 0x7f7a0019e680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:18.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.960+0000 7f7a08092700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7a0006b2d0 0x7f7a0019ebc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:18.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.960+0000 7f7a08092700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7a001082e0 0x7f7a00198800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:18.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.960+0000 7f7a08092700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f7a00112540 con 0x7f7a0006a930 2024-04-25T00:32:18.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.960+0000 7f7a08092700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f7a001123c0 con 0x7f7a001082e0 2024-04-25T00:32:18.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.960+0000 7f7a08092700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f7a001126c0 con 0x7f7a0006b2d0 2024-04-25T00:32:18.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.960+0000 7f7a05e2e700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7a0006a930 0x7f7a0019e680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.960+0000 7f7a05e2e700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7a0006a930 0x7f7a0019e680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40322/0 (socket says 172.21.15.32:40322) 2024-04-25T00:32:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.960+0000 7f7a05e2e700 1 -- 172.21.15.32:0/3351035745 learned_addr learned my addr 172.21.15.32:0/3351035745 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.960+0000 7f7a0662f700 1 --2- 172.21.15.32:0/3351035745 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7a001082e0 0x7f7a00198800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.960+0000 7f7a0562d700 1 --2- 172.21.15.32:0/3351035745 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7a0006b2d0 0x7f7a0019ebc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.960+0000 7f7a05e2e700 1 -- 172.21.15.32:0/3351035745 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7a0006b2d0 msgr2=0x7f7a0019ebc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.960+0000 7f7a05e2e700 1 --2- 172.21.15.32:0/3351035745 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7a0006b2d0 0x7f7a0019ebc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.960+0000 7f7a05e2e700 1 -- 172.21.15.32:0/3351035745 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7a001082e0 msgr2=0x7f7a00198800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.960+0000 7f7a05e2e700 1 --2- 172.21.15.32:0/3351035745 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7a001082e0 0x7f7a00198800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.960+0000 7f7a05e2e700 1 -- 172.21.15.32:0/3351035745 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7a00198fe0 con 0x7f7a0006a930 2024-04-25T00:32:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.960+0000 7f7a0562d700 1 --2- 172.21.15.32:0/3351035745 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7a0006b2d0 0x7f7a0019ebc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:32:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.960+0000 7f7a0662f700 1 --2- 172.21.15.32:0/3351035745 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7a001082e0 0x7f7a00198800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:32:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.960+0000 7f7a05e2e700 1 --2- 172.21.15.32:0/3351035745 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7a0006a930 0x7f7a0019e680 secure :-1 s=READY pgs=329 cs=0 l=1 rev1=1 crypto rx=0x7f79fc00e9a0 tx=0x7f79fc00ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.961+0000 7f79f6ffd700 1 -- 172.21.15.32:0/3351035745 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f79fc00cbd0 con 0x7f7a0006a930 2024-04-25T00:32:18.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.961+0000 7f79f6ffd700 1 -- 172.21.15.32:0/3351035745 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f79fc00cd70 con 0x7f7a0006a930 2024-04-25T00:32:18.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.961+0000 7f79f6ffd700 1 -- 172.21.15.32:0/3351035745 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f79fc018750 con 0x7f7a0006a930 2024-04-25T00:32:18.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.961+0000 7f7a08092700 1 -- 172.21.15.32:0/3351035745 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7a001992d0 con 0x7f7a0006a930 2024-04-25T00:32:18.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.961+0000 7f7a08092700 1 -- 172.21.15.32:0/3351035745 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7a001a54e0 con 0x7f7a0006a930 2024-04-25T00:32:18.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.962+0000 7f7a08092700 1 -- 172.21.15.32:0/3351035745 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7a00103990 con 0x7f7a0006a930 2024-04-25T00:32:18.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.963+0000 7f79f6ffd700 1 -- 172.21.15.32:0/3351035745 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f79fc010430 con 0x7f7a0006a930 2024-04-25T00:32:18.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.964+0000 7f79f6ffd700 1 --2- 172.21.15.32:0/3351035745 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f79ec0a2d30 0x7f79ec0a51f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:18.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.964+0000 7f7a0562d700 1 --2- 172.21.15.32:0/3351035745 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f79ec0a2d30 0x7f79ec0a51f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:18.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.964+0000 7f79f6ffd700 1 -- 172.21.15.32:0/3351035745 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f79fc0cf6f0 con 0x7f7a0006a930 2024-04-25T00:32:18.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.964+0000 7f7a0562d700 1 --2- 172.21.15.32:0/3351035745 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f79ec0a2d30 0x7f79ec0a51f0 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f79f000b2e0 tx=0x7f79f001a040 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:18.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.968+0000 7f79f6ffd700 1 -- 172.21.15.32:0/3351035745 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f79fc09c190 con 0x7f7a0006a930 2024-04-25T00:32:19.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.861+0000 7efdceb43700 1 -- 172.21.15.32:0/961724811 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efdc8101770 msgr2=0x7efdc8101b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:19.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.861+0000 7efdceb43700 1 --2- 172.21.15.32:0/961724811 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efdc8101770 0x7efdc8101b50 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7efdb80099c0 tx=0x7efdb8009cd0 comp rx=0 tx=0).stop 2024-04-25T00:32:19.086 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.861+0000 7efdceb43700 1 -- 172.21.15.32:0/961724811 shutdown_connections 2024-04-25T00:32:19.086 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.861+0000 7efdceb43700 1 --2- 172.21.15.32:0/961724811 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efdc81064c0 0x7efdc810e060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:19.086 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.861+0000 7efdceb43700 1 --2- 172.21.15.32:0/961724811 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efdc8101770 0x7efdc8101b50 unknown :-1 s=CLOSED pgs=326 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:19.086 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.862+0000 7efdceb43700 1 --2- 172.21.15.32:0/961724811 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efdc8102120 0x7efdc8105d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:19.086 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.862+0000 7efdceb43700 1 -- 172.21.15.32:0/961724811 >> 172.21.15.32:0/961724811 conn(0x7efdc80780e0 msgr2=0x7efdc80784f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:19.086 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.862+0000 7efdceb43700 1 -- 172.21.15.32:0/961724811 shutdown_connections 2024-04-25T00:32:19.086 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.862+0000 7efdceb43700 1 -- 172.21.15.32:0/961724811 wait complete. 2024-04-25T00:32:19.086 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.862+0000 7efdceb43700 1 Processor -- start 2024-04-25T00:32:19.086 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.862+0000 7efdceb43700 1 -- start start 2024-04-25T00:32:19.086 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.862+0000 7efdceb43700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efdc8101770 0x7efdc819e6e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:19.086 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.862+0000 7efdceb43700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efdc8102120 0x7efdc819ec20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:19.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.862+0000 7efdceb43700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efdc81064c0 0x7efdc81988a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:19.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.862+0000 7efdceb43700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7efdc81114a0 con 0x7efdc8102120 2024-04-25T00:32:19.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.862+0000 7efdceb43700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7efdc8111320 con 0x7efdc8101770 2024-04-25T00:32:19.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.862+0000 7efdceb43700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7efdc8111620 con 0x7efdc81064c0 2024-04-25T00:32:19.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.863+0000 7efdc7fff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efdc8102120 0x7efdc819ec20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:19.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.863+0000 7efdc7fff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efdc8102120 0x7efdc819ec20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40292/0 (socket says 172.21.15.32:40292) 2024-04-25T00:32:19.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.863+0000 7efdc7fff700 1 -- 172.21.15.32:0/4129626336 learned_addr learned my addr 172.21.15.32:0/4129626336 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:19.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.863+0000 7efdcc8df700 1 --2- 172.21.15.32:0/4129626336 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efdc8101770 0x7efdc819e6e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:19.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.863+0000 7efdcd0e0700 1 --2- 172.21.15.32:0/4129626336 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efdc81064c0 0x7efdc81988a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:19.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.863+0000 7efdc7fff700 1 -- 172.21.15.32:0/4129626336 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efdc81064c0 msgr2=0x7efdc81988a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:19.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.863+0000 7efdc7fff700 1 --2- 172.21.15.32:0/4129626336 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efdc81064c0 0x7efdc81988a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:19.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.863+0000 7efdc7fff700 1 -- 172.21.15.32:0/4129626336 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efdc8101770 msgr2=0x7efdc819e6e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:19.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.863+0000 7efdc7fff700 1 --2- 172.21.15.32:0/4129626336 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efdc8101770 0x7efdc819e6e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:19.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.863+0000 7efdc7fff700 1 -- 172.21.15.32:0/4129626336 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7efdc8199000 con 0x7efdc8102120 2024-04-25T00:32:19.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.863+0000 7efdcd0e0700 1 --2- 172.21.15.32:0/4129626336 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efdc81064c0 0x7efdc81988a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:32:19.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.863+0000 7efdc7fff700 1 --2- 172.21.15.32:0/4129626336 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efdc8102120 0x7efdc819ec20 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7efdbc00e940 tx=0x7efdbc00ec50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:19.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.863+0000 7efdc5ffb700 1 -- 172.21.15.32:0/4129626336 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efdbc00cbd0 con 0x7efdc8102120 2024-04-25T00:32:19.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.863+0000 7efdc5ffb700 1 -- 172.21.15.32:0/4129626336 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7efdbc00ee90 con 0x7efdc8102120 2024-04-25T00:32:19.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.863+0000 7efdceb43700 1 -- 172.21.15.32:0/4129626336 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7efdc81992f0 con 0x7efdc8102120 2024-04-25T00:32:19.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.863+0000 7efdc5ffb700 1 -- 172.21.15.32:0/4129626336 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efdbc018660 con 0x7efdc8102120 2024-04-25T00:32:19.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.864+0000 7efdceb43700 1 -- 172.21.15.32:0/4129626336 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7efdc81a55b0 con 0x7efdc8102120 2024-04-25T00:32:19.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.865+0000 7efdceb43700 1 -- 172.21.15.32:0/4129626336 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7efdc8102870 con 0x7efdc8102120 2024-04-25T00:32:19.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.866+0000 7efdc5ffb700 1 -- 172.21.15.32:0/4129626336 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7efdbc010430 con 0x7efdc8102120 2024-04-25T00:32:19.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.866+0000 7efdc5ffb700 1 --2- 172.21.15.32:0/4129626336 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7efdb00a2d30 0x7efdb00a51f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:19.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.866+0000 7efdcc8df700 1 --2- 172.21.15.32:0/4129626336 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7efdb00a2d30 0x7efdb00a51f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:19.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.866+0000 7efdc5ffb700 1 -- 172.21.15.32:0/4129626336 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7efdbc0cf700 con 0x7efdc8102120 2024-04-25T00:32:19.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.867+0000 7efdcc8df700 1 --2- 172.21.15.32:0/4129626336 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7efdb00a2d30 0x7efdb00a51f0 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7efdc81999e0 tx=0x7efdb8005a90 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:19.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:18.871+0000 7efdc5ffb700 1 -- 172.21.15.32:0/4129626336 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7efdbc09c1a0 con 0x7efdc8102120 2024-04-25T00:32:19.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.084+0000 7efdceb43700 1 -- 172.21.15.32:0/4129626336 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 8} v 0) -- 0x7efdc81a57e0 con 0x7efdc8102120 2024-04-25T00:32:19.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.085+0000 7efdc5ffb700 1 -- 172.21.15.32:0/4129626336 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 8}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7efdbc09baa0 con 0x7efdc8102120 2024-04-25T00:32:19.089 INFO:teuthology.orchestra.run.smithi032.stdout:253403070510 2024-04-25T00:32:19.090 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.088+0000 7efdceb43700 1 -- 172.21.15.32:0/4129626336 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7efdb00a2d30 msgr2=0x7efdb00a51f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:19.090 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.088+0000 7efdceb43700 1 --2- 172.21.15.32:0/4129626336 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7efdb00a2d30 0x7efdb00a51f0 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7efdc81999e0 tx=0x7efdb8005a90 comp rx=0 tx=0).stop 2024-04-25T00:32:19.090 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.088+0000 7efdceb43700 1 -- 172.21.15.32:0/4129626336 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efdc8102120 msgr2=0x7efdc819ec20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:19.090 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.088+0000 7efdceb43700 1 --2- 172.21.15.32:0/4129626336 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efdc8102120 0x7efdc819ec20 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7efdbc00e940 tx=0x7efdbc00ec50 comp rx=0 tx=0).stop 2024-04-25T00:32:19.090 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.088+0000 7efdceb43700 1 -- 172.21.15.32:0/4129626336 shutdown_connections 2024-04-25T00:32:19.090 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.088+0000 7efdceb43700 1 --2- 172.21.15.32:0/4129626336 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7efdb00a2d30 0x7efdb00a51f0 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:19.090 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.088+0000 7efdceb43700 1 --2- 172.21.15.32:0/4129626336 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efdc81064c0 0x7efdc81988a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:19.090 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.088+0000 7efdceb43700 1 --2- 172.21.15.32:0/4129626336 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efdc8101770 0x7efdc819e6e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:19.090 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.088+0000 7efdceb43700 1 --2- 172.21.15.32:0/4129626336 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efdc8102120 0x7efdc819ec20 unknown :-1 s=CLOSED pgs=327 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:19.090 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.088+0000 7efdceb43700 1 -- 172.21.15.32:0/4129626336 >> 172.21.15.32:0/4129626336 conn(0x7efdc80780e0 msgr2=0x7efdc8100cb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:19.090 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.088+0000 7efdceb43700 1 -- 172.21.15.32:0/4129626336 shutdown_connections 2024-04-25T00:32:19.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.088+0000 7efdceb43700 1 -- 172.21.15.32:0/4129626336 wait complete. 2024-04-25T00:32:19.197 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.196+0000 7f7a08092700 1 -- 172.21.15.32:0/3351035745 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7f7a0002db30 con 0x7f7a0006a930 2024-04-25T00:32:19.197 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.197+0000 7f79f6ffd700 1 -- 172.21.15.32:0/3351035745 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f79fc09ba90 con 0x7f7a0006a930 2024-04-25T00:32:19.197 INFO:teuthology.orchestra.run.smithi032.stdout:55834574929 2024-04-25T00:32:19.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.199+0000 7f7a08092700 1 -- 172.21.15.32:0/3351035745 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f79ec0a2d30 msgr2=0x7f79ec0a51f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:19.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.199+0000 7f7a08092700 1 --2- 172.21.15.32:0/3351035745 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f79ec0a2d30 0x7f79ec0a51f0 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f79f000b2e0 tx=0x7f79f001a040 comp rx=0 tx=0).stop 2024-04-25T00:32:19.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.199+0000 7f7a08092700 1 -- 172.21.15.32:0/3351035745 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7a0006a930 msgr2=0x7f7a0019e680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:19.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.199+0000 7f7a08092700 1 --2- 172.21.15.32:0/3351035745 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7a0006a930 0x7f7a0019e680 secure :-1 s=READY pgs=329 cs=0 l=1 rev1=1 crypto rx=0x7f79fc00e9a0 tx=0x7f79fc00ecb0 comp rx=0 tx=0).stop 2024-04-25T00:32:19.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.199+0000 7f7a08092700 1 -- 172.21.15.32:0/3351035745 shutdown_connections 2024-04-25T00:32:19.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.199+0000 7f7a08092700 1 --2- 172.21.15.32:0/3351035745 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f79ec0a2d30 0x7f79ec0a51f0 unknown :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:19.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.199+0000 7f7a08092700 1 --2- 172.21.15.32:0/3351035745 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7a001082e0 0x7f7a00198800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:19.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.199+0000 7f7a08092700 1 --2- 172.21.15.32:0/3351035745 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7a0006a930 0x7f7a0019e680 unknown :-1 s=CLOSED pgs=329 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:19.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.199+0000 7f7a08092700 1 --2- 172.21.15.32:0/3351035745 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7a0006b2d0 0x7f7a0019ebc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:19.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.199+0000 7f7a08092700 1 -- 172.21.15.32:0/3351035745 >> 172.21.15.32:0/3351035745 conn(0x7f7a00078170 msgr2=0x7f7a000ffdf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:19.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.199+0000 7f7a08092700 1 -- 172.21.15.32:0/3351035745 shutdown_connections 2024-04-25T00:32:19.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:19.199+0000 7f7a08092700 1 -- 172.21.15.32:0/3351035745 wait complete. 2024-04-25T00:32:20.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.032+0000 7f014b842700 1 -- 172.21.15.32:0/289314627 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01441024e0 msgr2=0x7f01441028c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:20.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.032+0000 7f014b842700 1 --2- 172.21.15.32:0/289314627 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01441024e0 0x7f01441028c0 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7f0140009990 tx=0x7f0140009ca0 comp rx=0 tx=0).stop 2024-04-25T00:32:20.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.032+0000 7f014b842700 1 -- 172.21.15.32:0/289314627 shutdown_connections 2024-04-25T00:32:20.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.032+0000 7f014b842700 1 --2- 172.21.15.32:0/289314627 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0144107570 0x7f014410f140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:20.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.032+0000 7f014b842700 1 --2- 172.21.15.32:0/289314627 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01441024e0 0x7f01441028c0 unknown :-1 s=CLOSED pgs=330 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:20.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.032+0000 7f014b842700 1 --2- 172.21.15.32:0/289314627 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0144102e90 0x7f0144106e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:20.036 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.032+0000 7f014b842700 1 -- 172.21.15.32:0/289314627 >> 172.21.15.32:0/289314627 conn(0x7f0144079390 msgr2=0x7f01440797a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:20.036 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.032+0000 7f014b842700 1 -- 172.21.15.32:0/289314627 shutdown_connections 2024-04-25T00:32:20.036 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.032+0000 7f014b842700 1 -- 172.21.15.32:0/289314627 wait complete. 2024-04-25T00:32:20.036 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.033+0000 7f014b842700 1 Processor -- start 2024-04-25T00:32:20.036 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.033+0000 7f014b842700 1 -- start start 2024-04-25T00:32:20.036 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.033+0000 7f014b842700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01441024e0 0x7f014419e6a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:20.036 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.033+0000 7f014b842700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0144102e90 0x7f014419ebe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:20.036 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.033+0000 7f014b842700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0144107570 0x7f0144198820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:20.036 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.033+0000 7f014b842700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f0144112560 con 0x7f01441024e0 2024-04-25T00:32:20.036 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.033+0000 7f014b842700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f01441123e0 con 0x7f0144102e90 2024-04-25T00:32:20.036 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.033+0000 7f014b842700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f01441126e0 con 0x7f0144107570 2024-04-25T00:32:20.036 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.033+0000 7f01495de700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01441024e0 0x7f014419e6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:20.036 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.033+0000 7f01495de700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01441024e0 0x7f014419e6a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40336/0 (socket says 172.21.15.32:40336) 2024-04-25T00:32:20.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.034+0000 7f01495de700 1 -- 172.21.15.32:0/369462224 learned_addr learned my addr 172.21.15.32:0/369462224 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:20.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.034+0000 7f0148ddd700 1 --2- 172.21.15.32:0/369462224 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0144102e90 0x7f014419ebe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:20.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.034+0000 7f01495de700 1 -- 172.21.15.32:0/369462224 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0144107570 msgr2=0x7f0144198820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:20.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.034+0000 7f01495de700 1 --2- 172.21.15.32:0/369462224 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0144107570 0x7f0144198820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:20.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.034+0000 7f01495de700 1 -- 172.21.15.32:0/369462224 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0144102e90 msgr2=0x7f014419ebe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:20.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.034+0000 7f01495de700 1 --2- 172.21.15.32:0/369462224 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0144102e90 0x7f014419ebe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:20.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.034+0000 7f01495de700 1 -- 172.21.15.32:0/369462224 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0144199000 con 0x7f01441024e0 2024-04-25T00:32:20.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.034+0000 7f0148ddd700 1 --2- 172.21.15.32:0/369462224 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0144102e90 0x7f014419ebe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:32:20.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.034+0000 7f01495de700 1 --2- 172.21.15.32:0/369462224 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01441024e0 0x7f014419e6a0 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7f0140000c00 tx=0x7f01400047b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:20.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.034+0000 7f013a7fc700 1 -- 172.21.15.32:0/369462224 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0140027070 con 0x7f01441024e0 2024-04-25T00:32:20.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.034+0000 7f014b842700 1 -- 172.21.15.32:0/369462224 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0144199290 con 0x7f01441024e0 2024-04-25T00:32:20.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.034+0000 7f013a7fc700 1 -- 172.21.15.32:0/369462224 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0140004050 con 0x7f01441024e0 2024-04-25T00:32:20.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.034+0000 7f013a7fc700 1 -- 172.21.15.32:0/369462224 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f014000f460 con 0x7f01441024e0 2024-04-25T00:32:20.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.034+0000 7f014b842700 1 -- 172.21.15.32:0/369462224 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f01441a5500 con 0x7f01441024e0 2024-04-25T00:32:20.038 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.035+0000 7f014b842700 1 -- 172.21.15.32:0/369462224 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0144103970 con 0x7f01441024e0 2024-04-25T00:32:20.038 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.037+0000 7f013a7fc700 1 -- 172.21.15.32:0/369462224 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f014002a070 con 0x7f01441024e0 2024-04-25T00:32:20.038 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.037+0000 7f013a7fc700 1 --2- 172.21.15.32:0/369462224 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f01300a2cb0 0x7f01300a5170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:20.038 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.037+0000 7f0148ddd700 1 --2- 172.21.15.32:0/369462224 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f01300a2cb0 0x7f01300a5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:20.039 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.037+0000 7f013a7fc700 1 -- 172.21.15.32:0/369462224 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f01400d0900 con 0x7f01441024e0 2024-04-25T00:32:20.039 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.038+0000 7f0148ddd700 1 --2- 172.21.15.32:0/369462224 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f01300a2cb0 0x7f01300a5170 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7f0134005fd0 tx=0x7f0134005ee0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:20.041 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.041+0000 7f013a7fc700 1 -- 172.21.15.32:0/369462224 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f0140031070 con 0x7f01441024e0 2024-04-25T00:32:20.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:19 smithi032 ceph-mon[29357]: pgmap v339: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 7 op/s 2024-04-25T00:32:20.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:19 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/4129626336' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 8}]: dispatch 2024-04-25T00:32:20.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:19 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3351035745' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-25T00:32:20.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.252+0000 7f014b842700 1 -- 172.21.15.32:0/369462224 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 10} v 0) -- 0x7f0144199d50 con 0x7f01441024e0 2024-04-25T00:32:20.253 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.252+0000 7f013a7fc700 1 -- 172.21.15.32:0/369462224 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 10}]=0 v0) ==== 75+0+13 (secure 0 0 0) 0x7f0140013070 con 0x7f01441024e0 2024-04-25T00:32:20.253 INFO:teuthology.orchestra.run.smithi032.stdout:309237645349 2024-04-25T00:32:20.255 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.254+0000 7f014b842700 1 -- 172.21.15.32:0/369462224 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f01300a2cb0 msgr2=0x7f01300a5170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:20.255 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.254+0000 7f014b842700 1 --2- 172.21.15.32:0/369462224 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f01300a2cb0 0x7f01300a5170 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7f0134005fd0 tx=0x7f0134005ee0 comp rx=0 tx=0).stop 2024-04-25T00:32:20.255 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.254+0000 7f014b842700 1 -- 172.21.15.32:0/369462224 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01441024e0 msgr2=0x7f014419e6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:20.255 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.254+0000 7f014b842700 1 --2- 172.21.15.32:0/369462224 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01441024e0 0x7f014419e6a0 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7f0140000c00 tx=0x7f01400047b0 comp rx=0 tx=0).stop 2024-04-25T00:32:20.256 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.255+0000 7f014b842700 1 -- 172.21.15.32:0/369462224 shutdown_connections 2024-04-25T00:32:20.256 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.255+0000 7f014b842700 1 --2- 172.21.15.32:0/369462224 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f01300a2cb0 0x7f01300a5170 unknown :-1 s=CLOSED pgs=214 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:20.256 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.255+0000 7f014b842700 1 --2- 172.21.15.32:0/369462224 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0144107570 0x7f0144198820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:20.256 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.255+0000 7f014b842700 1 --2- 172.21.15.32:0/369462224 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01441024e0 0x7f014419e6a0 unknown :-1 s=CLOSED pgs=331 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:20.256 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.255+0000 7f014b842700 1 --2- 172.21.15.32:0/369462224 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0144102e90 0x7f014419ebe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:20.256 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.255+0000 7f014b842700 1 -- 172.21.15.32:0/369462224 >> 172.21.15.32:0/369462224 conn(0x7f0144079390 msgr2=0x7f0144101a40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:20.256 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.255+0000 7f014b842700 1 -- 172.21.15.32:0/369462224 shutdown_connections 2024-04-25T00:32:20.256 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.255+0000 7f014b842700 1 -- 172.21.15.32:0/369462224 wait complete. 2024-04-25T00:32:20.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:19 smithi121 ceph-mon[32231]: pgmap v339: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 7 op/s 2024-04-25T00:32:20.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:19 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/4129626336' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 8}]: dispatch 2024-04-25T00:32:20.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:19 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3351035745' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-25T00:32:20.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:19 smithi130 ceph-mon[32525]: pgmap v339: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 7 op/s 2024-04-25T00:32:20.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:19 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/4129626336' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 8}]: dispatch 2024-04-25T00:32:20.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:19 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3351035745' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-25T00:32:20.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.823+0000 7f9fd3649700 1 -- 172.21.15.32:0/1924205774 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fcc06a8e0 msgr2=0x7f9fcc10f0f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:20.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.823+0000 7f9fd3649700 1 --2- 172.21.15.32:0/1924205774 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fcc06a8e0 0x7f9fcc10f0f0 secure :-1 s=READY pgs=332 cs=0 l=1 rev1=1 crypto rx=0x7f9fc400b0a0 tx=0x7f9fc400b3b0 comp rx=0 tx=0).stop 2024-04-25T00:32:20.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.823+0000 7f9fd3649700 1 -- 172.21.15.32:0/1924205774 shutdown_connections 2024-04-25T00:32:20.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.823+0000 7f9fd3649700 1 --2- 172.21.15.32:0/1924205774 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fcc06a8e0 0x7f9fcc10f0f0 unknown :-1 s=CLOSED pgs=332 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:20.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.823+0000 7f9fd3649700 1 --2- 172.21.15.32:0/1924205774 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9fcc1082b0 0x7f9fcc108690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:20.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.823+0000 7f9fd3649700 1 --2- 172.21.15.32:0/1924205774 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9fcc069f40 0x7f9fcc06a3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:20.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.823+0000 7f9fd3649700 1 -- 172.21.15.32:0/1924205774 >> 172.21.15.32:0/1924205774 conn(0x7f9fcc0780e0 msgr2=0x7f9fcc0784f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:20.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.823+0000 7f9fd3649700 1 -- 172.21.15.32:0/1924205774 shutdown_connections 2024-04-25T00:32:20.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.823+0000 7f9fd3649700 1 -- 172.21.15.32:0/1924205774 wait complete. 2024-04-25T00:32:20.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.823+0000 7f9fd3649700 1 Processor -- start 2024-04-25T00:32:20.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.823+0000 7f9fd3649700 1 -- start start 2024-04-25T00:32:20.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.824+0000 7f9fd3649700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fcc069f40 0x7f9fcc1a07f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:20.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.824+0000 7f9fd3649700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9fcc06a8e0 0x7f9fcc1a0d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:20.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.824+0000 7f9fd3649700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9fcc1082b0 0x7f9fcc19a9b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:20.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.824+0000 7f9fd3649700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f9fcc112500 con 0x7f9fcc069f40 2024-04-25T00:32:20.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.824+0000 7f9fd3649700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f9fcc112380 con 0x7f9fcc06a8e0 2024-04-25T00:32:20.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.824+0000 7f9fd3649700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f9fcc112680 con 0x7f9fcc1082b0 2024-04-25T00:32:20.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.824+0000 7f9fd13e5700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fcc069f40 0x7f9fcc1a07f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:20.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.824+0000 7f9fd13e5700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fcc069f40 0x7f9fcc1a07f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40362/0 (socket says 172.21.15.32:40362) 2024-04-25T00:32:20.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.824+0000 7f9fd13e5700 1 -- 172.21.15.32:0/3314804308 learned_addr learned my addr 172.21.15.32:0/3314804308 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:20.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.824+0000 7f9fd0be4700 1 --2- 172.21.15.32:0/3314804308 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9fcc06a8e0 0x7f9fcc1a0d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:20.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.824+0000 7f9fd1be6700 1 --2- 172.21.15.32:0/3314804308 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9fcc1082b0 0x7f9fcc19a9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:20.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.824+0000 7f9fd13e5700 1 -- 172.21.15.32:0/3314804308 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9fcc1082b0 msgr2=0x7f9fcc19a9b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:20.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.824+0000 7f9fd13e5700 1 --2- 172.21.15.32:0/3314804308 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9fcc1082b0 0x7f9fcc19a9b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:20.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.824+0000 7f9fd13e5700 1 -- 172.21.15.32:0/3314804308 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9fcc06a8e0 msgr2=0x7f9fcc1a0d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:20.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.824+0000 7f9fd13e5700 1 --2- 172.21.15.32:0/3314804308 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9fcc06a8e0 0x7f9fcc1a0d30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:20.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.824+0000 7f9fd13e5700 1 -- 172.21.15.32:0/3314804308 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9fcc19b110 con 0x7f9fcc069f40 2024-04-25T00:32:20.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.824+0000 7f9fd0be4700 1 --2- 172.21.15.32:0/3314804308 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9fcc06a8e0 0x7f9fcc1a0d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:32:20.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.824+0000 7f9fd13e5700 1 --2- 172.21.15.32:0/3314804308 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fcc069f40 0x7f9fcc1a07f0 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7f9fc800c920 tx=0x7f9fc800cce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:20.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.825+0000 7f9fc27fc700 1 -- 172.21.15.32:0/3314804308 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9fc8007b00 con 0x7f9fcc069f40 2024-04-25T00:32:20.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.825+0000 7f9fd3649700 1 -- 172.21.15.32:0/3314804308 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9fcc19b400 con 0x7f9fcc069f40 2024-04-25T00:32:20.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.825+0000 7f9fc27fc700 1 -- 172.21.15.32:0/3314804308 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9fc8004d10 con 0x7f9fcc069f40 2024-04-25T00:32:20.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.825+0000 7f9fc27fc700 1 -- 172.21.15.32:0/3314804308 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9fc80056e0 con 0x7f9fcc069f40 2024-04-25T00:32:20.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.825+0000 7f9fd3649700 1 -- 172.21.15.32:0/3314804308 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9fcc1a76c0 con 0x7f9fcc069f40 2024-04-25T00:32:20.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.826+0000 7f9fd3649700 1 -- 172.21.15.32:0/3314804308 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9fcc068f20 con 0x7f9fcc069f40 2024-04-25T00:32:20.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.827+0000 7f9fc27fc700 1 -- 172.21.15.32:0/3314804308 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f9fc8007ca0 con 0x7f9fcc069f40 2024-04-25T00:32:20.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.828+0000 7f9fc27fc700 1 --2- 172.21.15.32:0/3314804308 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f9fb80a2cb0 0x7f9fb80a5170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:20.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.828+0000 7f9fd0be4700 1 --2- 172.21.15.32:0/3314804308 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f9fb80a2cb0 0x7f9fb80a5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:20.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.828+0000 7f9fc27fc700 1 -- 172.21.15.32:0/3314804308 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f9fc80cf4d0 con 0x7f9fcc069f40 2024-04-25T00:32:20.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.828+0000 7f9fd0be4700 1 --2- 172.21.15.32:0/3314804308 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f9fb80a2cb0 0x7f9fb80a5170 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f9fcc103210 tx=0x7f9fbc00b5f0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:20.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:20.831+0000 7f9fc27fc700 1 -- 172.21.15.32:0/3314804308 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f9fc8014030 con 0x7f9fcc069f40 2024-04-25T00:32:20.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:20 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/369462224' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 10}]: dispatch 2024-04-25T00:32:21.039 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:21.039+0000 7f9fd3649700 1 -- 172.21.15.32:0/3314804308 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 6} v 0) -- 0x7f9fcc1a79b0 con 0x7f9fcc069f40 2024-04-25T00:32:21.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:21.039+0000 7f9fc27fc700 1 -- 172.21.15.32:0/3314804308 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 6}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f9fc809bf70 con 0x7f9fcc069f40 2024-04-25T00:32:21.040 INFO:teuthology.orchestra.run.smithi032.stdout:197568495674 2024-04-25T00:32:21.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:21.041+0000 7f9fd3649700 1 -- 172.21.15.32:0/3314804308 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f9fb80a2cb0 msgr2=0x7f9fb80a5170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:21.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:21.041+0000 7f9fd3649700 1 --2- 172.21.15.32:0/3314804308 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f9fb80a2cb0 0x7f9fb80a5170 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f9fcc103210 tx=0x7f9fbc00b5f0 comp rx=0 tx=0).stop 2024-04-25T00:32:21.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:21.041+0000 7f9fd3649700 1 -- 172.21.15.32:0/3314804308 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fcc069f40 msgr2=0x7f9fcc1a07f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:21.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:21.041+0000 7f9fd3649700 1 --2- 172.21.15.32:0/3314804308 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fcc069f40 0x7f9fcc1a07f0 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7f9fc800c920 tx=0x7f9fc800cce0 comp rx=0 tx=0).stop 2024-04-25T00:32:21.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:21.041+0000 7f9fd3649700 1 -- 172.21.15.32:0/3314804308 shutdown_connections 2024-04-25T00:32:21.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:21.041+0000 7f9fd3649700 1 --2- 172.21.15.32:0/3314804308 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f9fb80a2cb0 0x7f9fb80a5170 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:21.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:21.041+0000 7f9fd3649700 1 --2- 172.21.15.32:0/3314804308 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9fcc1082b0 0x7f9fcc19a9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:21.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:21.041+0000 7f9fd3649700 1 --2- 172.21.15.32:0/3314804308 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fcc069f40 0x7f9fcc1a07f0 unknown :-1 s=CLOSED pgs=333 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:21.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:21.041+0000 7f9fd3649700 1 --2- 172.21.15.32:0/3314804308 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9fcc06a8e0 0x7f9fcc1a0d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:21.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:21.041+0000 7f9fd3649700 1 -- 172.21.15.32:0/3314804308 >> 172.21.15.32:0/3314804308 conn(0x7f9fcc0780e0 msgr2=0x7f9fcc10d4b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:21.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:21.042+0000 7f9fd3649700 1 -- 172.21.15.32:0/3314804308 shutdown_connections 2024-04-25T00:32:21.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:21.042+0000 7f9fd3649700 1 -- 172.21.15.32:0/3314804308 wait complete. 2024-04-25T00:32:21.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:20 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/369462224' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 10}]: dispatch 2024-04-25T00:32:21.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:20 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/369462224' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 10}]: dispatch 2024-04-25T00:32:22.235 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:21 smithi032 ceph-mon[29357]: pgmap v340: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:32:22.235 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:21 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3314804308' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-25T00:32:22.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:21 smithi121 ceph-mon[32231]: pgmap v340: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:32:22.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:21 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3314804308' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-25T00:32:22.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:21 smithi130 ceph-mon[32525]: pgmap v340: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:32:22.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:21 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3314804308' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-25T00:32:22.849 INFO:tasks.cephadm.ceph_manager.ceph:need seq 309237645345 got 309237645349 for osd.10 2024-04-25T00:32:22.849 DEBUG:teuthology.parallel:result is None 2024-04-25T00:32:23.437 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.402+0000 7fc53bcec700 1 -- 172.21.15.32:0/3353341136 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc5341024e0 msgr2=0x7fc5341028c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:23.437 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.402+0000 7fc53bcec700 1 --2- 172.21.15.32:0/3353341136 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc5341024e0 0x7fc5341028c0 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7fc530009990 tx=0x7fc530009ca0 comp rx=0 tx=0).stop 2024-04-25T00:32:23.437 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.402+0000 7fc53bcec700 1 -- 172.21.15.32:0/3353341136 shutdown_connections 2024-04-25T00:32:23.437 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.402+0000 7fc53bcec700 1 --2- 172.21.15.32:0/3353341136 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc534107570 0x7fc53410f140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:23.437 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.402+0000 7fc53bcec700 1 --2- 172.21.15.32:0/3353341136 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc5341024e0 0x7fc5341028c0 unknown :-1 s=CLOSED pgs=334 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:23.437 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.402+0000 7fc53bcec700 1 --2- 172.21.15.32:0/3353341136 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc534102e90 0x7fc534106e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:23.437 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.402+0000 7fc53bcec700 1 -- 172.21.15.32:0/3353341136 >> 172.21.15.32:0/3353341136 conn(0x7fc534079390 msgr2=0x7fc5340797a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:23.437 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.402+0000 7fc53bcec700 1 -- 172.21.15.32:0/3353341136 shutdown_connections 2024-04-25T00:32:23.437 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.402+0000 7fc53bcec700 1 -- 172.21.15.32:0/3353341136 wait complete. 2024-04-25T00:32:23.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.403+0000 7fc53bcec700 1 Processor -- start 2024-04-25T00:32:23.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.403+0000 7fc53bcec700 1 -- start start 2024-04-25T00:32:23.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.403+0000 7fc53bcec700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc5341024e0 0x7fc53419e690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:23.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.403+0000 7fc53bcec700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc534102e90 0x7fc53419ebd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:23.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.403+0000 7fc53bcec700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc534107570 0x7fc534198810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:23.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.403+0000 7fc53bcec700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fc534112530 con 0x7fc5341024e0 2024-04-25T00:32:23.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.403+0000 7fc53bcec700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fc5341123b0 con 0x7fc534107570 2024-04-25T00:32:23.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.403+0000 7fc53bcec700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fc5341126b0 con 0x7fc534102e90 2024-04-25T00:32:23.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.403+0000 7fc539a88700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc5341024e0 0x7fc53419e690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:23.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.403+0000 7fc539a88700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc5341024e0 0x7fc53419e690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:41928/0 (socket says 172.21.15.32:41928) 2024-04-25T00:32:23.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.403+0000 7fc539a88700 1 -- 172.21.15.32:0/3166931617 learned_addr learned my addr 172.21.15.32:0/3166931617 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:23.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.403+0000 7fc539287700 1 --2- 172.21.15.32:0/3166931617 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc534102e90 0x7fc53419ebd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:23.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.404+0000 7fc53a289700 1 --2- 172.21.15.32:0/3166931617 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc534107570 0x7fc534198810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:23.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.404+0000 7fc539a88700 1 -- 172.21.15.32:0/3166931617 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc534102e90 msgr2=0x7fc53419ebd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:23.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.404+0000 7fc539a88700 1 --2- 172.21.15.32:0/3166931617 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc534102e90 0x7fc53419ebd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:23.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.404+0000 7fc539a88700 1 -- 172.21.15.32:0/3166931617 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc534107570 msgr2=0x7fc534198810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:23.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.404+0000 7fc539a88700 1 --2- 172.21.15.32:0/3166931617 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc534107570 0x7fc534198810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:23.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.404+0000 7fc539a88700 1 -- 172.21.15.32:0/3166931617 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc534198ff0 con 0x7fc5341024e0 2024-04-25T00:32:23.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.404+0000 7fc539a88700 1 --2- 172.21.15.32:0/3166931617 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc5341024e0 0x7fc53419e690 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7fc530000c00 tx=0x7fc5300047b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:23.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.404+0000 7fc52affd700 1 -- 172.21.15.32:0/3166931617 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc530027070 con 0x7fc5341024e0 2024-04-25T00:32:23.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.404+0000 7fc52affd700 1 -- 172.21.15.32:0/3166931617 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc530003780 con 0x7fc5341024e0 2024-04-25T00:32:23.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.404+0000 7fc52affd700 1 -- 172.21.15.32:0/3166931617 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc530022dd0 con 0x7fc5341024e0 2024-04-25T00:32:23.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.404+0000 7fc53bcec700 1 -- 172.21.15.32:0/3166931617 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc534199280 con 0x7fc5341024e0 2024-04-25T00:32:23.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.404+0000 7fc53bcec700 1 -- 172.21.15.32:0/3166931617 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc5341a54f0 con 0x7fc5341024e0 2024-04-25T00:32:23.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.405+0000 7fc53bcec700 1 -- 172.21.15.32:0/3166931617 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc53410a790 con 0x7fc5341024e0 2024-04-25T00:32:23.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.406+0000 7fc52affd700 1 -- 172.21.15.32:0/3166931617 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7fc53002a070 con 0x7fc5341024e0 2024-04-25T00:32:23.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.407+0000 7fc52affd700 1 --2- 172.21.15.32:0/3166931617 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc5200a2cb0 0x7fc5200a5170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:23.440 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.407+0000 7fc52affd700 1 -- 172.21.15.32:0/3166931617 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7fc5300d0210 con 0x7fc5341024e0 2024-04-25T00:32:23.440 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.407+0000 7fc539287700 1 --2- 172.21.15.32:0/3166931617 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc5200a2cb0 0x7fc5200a5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:23.440 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.407+0000 7fc539287700 1 --2- 172.21.15.32:0/3166931617 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc5200a2cb0 0x7fc5200a5170 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7fc534101e50 tx=0x7fc5240073d0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:23.440 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.412+0000 7fc52affd700 1 -- 172.21.15.32:0/3166931617 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fc530013070 con 0x7fc5341024e0 2024-04-25T00:32:23.644 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.643+0000 7fc53bcec700 1 -- 172.21.15.32:0/3166931617 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 7} v 0) -- 0x7fc534199d40 con 0x7fc5341024e0 2024-04-25T00:32:23.644 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.643+0000 7fc52affd700 1 -- 172.21.15.32:0/3166931617 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 7}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7fc53009cdd0 con 0x7fc5341024e0 2024-04-25T00:32:23.644 INFO:teuthology.orchestra.run.smithi032.stdout:219043332148 2024-04-25T00:32:23.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.645+0000 7fc53bcec700 1 -- 172.21.15.32:0/3166931617 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc5200a2cb0 msgr2=0x7fc5200a5170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:23.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.645+0000 7fc53bcec700 1 --2- 172.21.15.32:0/3166931617 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc5200a2cb0 0x7fc5200a5170 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7fc534101e50 tx=0x7fc5240073d0 comp rx=0 tx=0).stop 2024-04-25T00:32:23.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.645+0000 7fc53bcec700 1 -- 172.21.15.32:0/3166931617 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc5341024e0 msgr2=0x7fc53419e690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:23.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.645+0000 7fc53bcec700 1 --2- 172.21.15.32:0/3166931617 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc5341024e0 0x7fc53419e690 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7fc530000c00 tx=0x7fc5300047b0 comp rx=0 tx=0).stop 2024-04-25T00:32:23.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.645+0000 7fc53bcec700 1 -- 172.21.15.32:0/3166931617 shutdown_connections 2024-04-25T00:32:23.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.645+0000 7fc53bcec700 1 --2- 172.21.15.32:0/3166931617 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc5200a2cb0 0x7fc5200a5170 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:23.647 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.645+0000 7fc53bcec700 1 --2- 172.21.15.32:0/3166931617 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc534107570 0x7fc534198810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:23.647 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.645+0000 7fc53bcec700 1 --2- 172.21.15.32:0/3166931617 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc5341024e0 0x7fc53419e690 unknown :-1 s=CLOSED pgs=335 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:23.647 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.645+0000 7fc53bcec700 1 --2- 172.21.15.32:0/3166931617 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc534102e90 0x7fc53419ebd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:23.647 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.646+0000 7fc53bcec700 1 -- 172.21.15.32:0/3166931617 >> 172.21.15.32:0/3166931617 conn(0x7fc534079390 msgr2=0x7fc5341086f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:23.647 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.646+0000 7fc53bcec700 1 -- 172.21.15.32:0/3166931617 shutdown_connections 2024-04-25T00:32:23.647 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:23.646+0000 7fc53bcec700 1 -- 172.21.15.32:0/3166931617 wait complete. 2024-04-25T00:32:23.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:23 smithi032 ceph-mon[29357]: pgmap v341: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 7 op/s 2024-04-25T00:32:23.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:23 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3166931617' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-25T00:32:24.070 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574920 got 55834574929 for osd.1 2024-04-25T00:32:24.070 DEBUG:teuthology.parallel:result is None 2024-04-25T00:32:24.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:23 smithi121 ceph-mon[32231]: pgmap v341: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 7 op/s 2024-04-25T00:32:24.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:23 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3166931617' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-25T00:32:24.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:23 smithi130 ceph-mon[32525]: pgmap v341: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 7 op/s 2024-04-25T00:32:24.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:23 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3166931617' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-25T00:32:24.495 INFO:tasks.cephadm.ceph_manager.ceph:need seq 253403070505 got 253403070510 for osd.8 2024-04-25T00:32:24.496 DEBUG:teuthology.parallel:result is None 2024-04-25T00:32:25.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:25 smithi032 ceph-mon[29357]: pgmap v342: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-25T00:32:26.243 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495668 got 197568495674 for osd.6 2024-04-25T00:32:26.243 DEBUG:teuthology.parallel:result is None 2024-04-25T00:32:26.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:25 smithi121 ceph-mon[32231]: pgmap v342: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-25T00:32:26.357 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.034+0000 7f7aac17c700 1 -- 172.21.15.32:0/948091373 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7aa41024e0 msgr2=0x7f7aa41028c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:26.357 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.034+0000 7f7aac17c700 1 --2- 172.21.15.32:0/948091373 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7aa41024e0 0x7f7aa41028c0 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 crypto rx=0x7f7aa0009990 tx=0x7f7aa0009ca0 comp rx=0 tx=0).stop 2024-04-25T00:32:26.357 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.035+0000 7f7aac17c700 1 -- 172.21.15.32:0/948091373 shutdown_connections 2024-04-25T00:32:26.357 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.035+0000 7f7aac17c700 1 --2- 172.21.15.32:0/948091373 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7aa4107570 0x7f7aa410f140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:26.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.035+0000 7f7aac17c700 1 --2- 172.21.15.32:0/948091373 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7aa41024e0 0x7f7aa41028c0 unknown :-1 s=CLOSED pgs=336 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:26.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.035+0000 7f7aac17c700 1 --2- 172.21.15.32:0/948091373 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7aa4102e90 0x7f7aa4106e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:26.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.035+0000 7f7aac17c700 1 -- 172.21.15.32:0/948091373 >> 172.21.15.32:0/948091373 conn(0x7f7aa4079390 msgr2=0x7f7aa40797a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:26.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.035+0000 7f7aac17c700 1 -- 172.21.15.32:0/948091373 shutdown_connections 2024-04-25T00:32:26.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.035+0000 7f7aac17c700 1 -- 172.21.15.32:0/948091373 wait complete. 2024-04-25T00:32:26.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.035+0000 7f7aac17c700 1 Processor -- start 2024-04-25T00:32:26.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.035+0000 7f7aac17c700 1 -- start start 2024-04-25T00:32:26.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.035+0000 7f7aac17c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7aa41024e0 0x7f7aa4074530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:26.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.036+0000 7f7aac17c700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7aa4102e90 0x7f7aa4074a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:26.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.036+0000 7f7aac17c700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7aa4107570 0x7f7aa4074fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:26.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.036+0000 7f7aac17c700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f7aa4112530 con 0x7f7aa4102e90 2024-04-25T00:32:26.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.036+0000 7f7aac17c700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f7aa41123b0 con 0x7f7aa4107570 2024-04-25T00:32:26.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.036+0000 7f7aac17c700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f7aa41126b0 con 0x7f7aa41024e0 2024-04-25T00:32:26.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.036+0000 7f7aa9717700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7aa4102e90 0x7f7aa4074a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:26.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.036+0000 7f7aa9717700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7aa4102e90 0x7f7aa4074a70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:41938/0 (socket says 172.21.15.32:41938) 2024-04-25T00:32:26.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.036+0000 7f7aa9717700 1 -- 172.21.15.32:0/1163906092 learned_addr learned my addr 172.21.15.32:0/1163906092 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:26.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.036+0000 7f7aaa719700 1 --2- 172.21.15.32:0/1163906092 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7aa4107570 0x7f7aa4074fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:26.359 INFO:teuthology.orchestra.run.smithi032.stdout:141733920837 2024-04-25T00:32:26.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.036+0000 7f7aa9f18700 1 --2- 172.21.15.32:0/1163906092 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7aa41024e0 0x7f7aa4074530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:26.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.036+0000 7f7aa9717700 1 -- 172.21.15.32:0/1163906092 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7aa41024e0 msgr2=0x7f7aa4074530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:26.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.036+0000 7f7aa9717700 1 --2- 172.21.15.32:0/1163906092 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7aa41024e0 0x7f7aa4074530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:26.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.036+0000 7f7aa9717700 1 -- 172.21.15.32:0/1163906092 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7aa4107570 msgr2=0x7f7aa4074fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:26.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.036+0000 7f7aa9717700 1 --2- 172.21.15.32:0/1163906092 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7aa4107570 0x7f7aa4074fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:26.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.036+0000 7f7aa9717700 1 -- 172.21.15.32:0/1163906092 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7aa41a6ee0 con 0x7f7aa4102e90 2024-04-25T00:32:26.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.036+0000 7f7aa9f18700 1 --2- 172.21.15.32:0/1163906092 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7aa41024e0 0x7f7aa4074530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:32:26.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.036+0000 7f7aa9717700 1 --2- 172.21.15.32:0/1163906092 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7aa4102e90 0x7f7aa4074a70 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x7f7a9400d980 tx=0x7f7a9400dc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:26.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.036+0000 7f7a9affd700 1 -- 172.21.15.32:0/1163906092 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7a94017070 con 0x7f7aa4102e90 2024-04-25T00:32:26.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.036+0000 7f7a9affd700 1 -- 172.21.15.32:0/1163906092 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7a94009e10 con 0x7f7aa4102e90 2024-04-25T00:32:26.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.036+0000 7f7aac17c700 1 -- 172.21.15.32:0/1163906092 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7aa41a71d0 con 0x7f7aa4102e90 2024-04-25T00:32:26.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.036+0000 7f7a9affd700 1 -- 172.21.15.32:0/1163906092 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7a9400f620 con 0x7f7aa4102e90 2024-04-25T00:32:26.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.036+0000 7f7aac17c700 1 -- 172.21.15.32:0/1163906092 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7aa41a7710 con 0x7f7aa4102e90 2024-04-25T00:32:26.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.037+0000 7f7aac17c700 1 -- 172.21.15.32:0/1163906092 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7aa4068f20 con 0x7f7aa4102e90 2024-04-25T00:32:26.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.038+0000 7f7a9affd700 1 -- 172.21.15.32:0/1163906092 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f7a94010460 con 0x7f7aa4102e90 2024-04-25T00:32:26.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.039+0000 7f7a9affd700 1 --2- 172.21.15.32:0/1163906092 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f7a900a2cb0 0x7f7a900a5170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:26.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.039+0000 7f7aa9f18700 1 --2- 172.21.15.32:0/1163906092 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f7a900a2cb0 0x7f7a900a5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:26.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.039+0000 7f7a9affd700 1 -- 172.21.15.32:0/1163906092 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f7a940cf130 con 0x7f7aa4102e90 2024-04-25T00:32:26.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.040+0000 7f7aa9f18700 1 --2- 172.21.15.32:0/1163906092 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f7a900a2cb0 0x7f7a900a5170 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f7aa000b5c0 tx=0x7f7aa0005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:26.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.044+0000 7f7a9affd700 1 -- 172.21.15.32:0/1163906092 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f7a9409bb70 con 0x7f7aa4102e90 2024-04-25T00:32:26.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.255+0000 7f7aac17c700 1 -- 172.21.15.32:0/1163906092 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 4} v 0) -- 0x7f7aa4075a10 con 0x7f7aa4102e90 2024-04-25T00:32:26.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.255+0000 7f7a9affd700 1 -- 172.21.15.32:0/1163906092 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 4}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f7a9409b470 con 0x7f7aa4102e90 2024-04-25T00:32:26.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.258+0000 7f7aac17c700 1 -- 172.21.15.32:0/1163906092 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f7a900a2cb0 msgr2=0x7f7a900a5170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:26.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.258+0000 7f7aac17c700 1 --2- 172.21.15.32:0/1163906092 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f7a900a2cb0 0x7f7a900a5170 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f7aa000b5c0 tx=0x7f7aa0005fb0 comp rx=0 tx=0).stop 2024-04-25T00:32:26.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.258+0000 7f7aac17c700 1 -- 172.21.15.32:0/1163906092 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7aa4102e90 msgr2=0x7f7aa4074a70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:26.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.258+0000 7f7aac17c700 1 --2- 172.21.15.32:0/1163906092 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7aa4102e90 0x7f7aa4074a70 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x7f7a9400d980 tx=0x7f7a9400dc90 comp rx=0 tx=0).stop 2024-04-25T00:32:26.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.258+0000 7f7aac17c700 1 -- 172.21.15.32:0/1163906092 shutdown_connections 2024-04-25T00:32:26.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.258+0000 7f7aac17c700 1 --2- 172.21.15.32:0/1163906092 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f7a900a2cb0 0x7f7a900a5170 unknown :-1 s=CLOSED pgs=217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:26.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.258+0000 7f7aac17c700 1 --2- 172.21.15.32:0/1163906092 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7aa4107570 0x7f7aa4074fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:26.362 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.258+0000 7f7aac17c700 1 --2- 172.21.15.32:0/1163906092 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7aa41024e0 0x7f7aa4074530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:26.362 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.258+0000 7f7aac17c700 1 --2- 172.21.15.32:0/1163906092 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7aa4102e90 0x7f7aa4074a70 unknown :-1 s=CLOSED pgs=337 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:26.362 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.258+0000 7f7aac17c700 1 -- 172.21.15.32:0/1163906092 >> 172.21.15.32:0/1163906092 conn(0x7f7aa4079390 msgr2=0x7f7aa4101a00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:26.362 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.258+0000 7f7aac17c700 1 -- 172.21.15.32:0/1163906092 shutdown_connections 2024-04-25T00:32:26.362 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:26.258+0000 7f7aac17c700 1 -- 172.21.15.32:0/1163906092 wait complete. 2024-04-25T00:32:26.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:25 smithi130 ceph-mon[32525]: pgmap v342: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-25T00:32:26.982 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:26 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1163906092' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-25T00:32:26.982 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:26 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:32:26.982 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:26 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:32:27.198 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.194+0000 7f11958c2700 1 -- 172.21.15.32:0/449002562 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f119006b5f0 msgr2=0x7f1190106eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:27.198 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.194+0000 7f11958c2700 1 --2- 172.21.15.32:0/449002562 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f119006b5f0 0x7f1190106eb0 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 crypto rx=0x7f11840099c0 tx=0x7f1184009cd0 comp rx=0 tx=0).stop 2024-04-25T00:32:27.198 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.194+0000 7f11958c2700 1 -- 172.21.15.32:0/449002562 shutdown_connections 2024-04-25T00:32:27.198 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.194+0000 7f11958c2700 1 --2- 172.21.15.32:0/449002562 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1190107590 0x7f119010f160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:27.198 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.194+0000 7f11958c2700 1 --2- 172.21.15.32:0/449002562 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f119006ac40 0x7f119006b020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:27.198 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.194+0000 7f11958c2700 1 --2- 172.21.15.32:0/449002562 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f119006b5f0 0x7f1190106eb0 unknown :-1 s=CLOSED pgs=338 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:27.198 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.194+0000 7f11958c2700 1 -- 172.21.15.32:0/449002562 >> 172.21.15.32:0/449002562 conn(0x7f11900781d0 msgr2=0x7f11900785e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:27.198 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.195+0000 7f11958c2700 1 -- 172.21.15.32:0/449002562 shutdown_connections 2024-04-25T00:32:27.198 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.195+0000 7f11958c2700 1 -- 172.21.15.32:0/449002562 wait complete. 2024-04-25T00:32:27.198 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.195+0000 7f11958c2700 1 Processor -- start 2024-04-25T00:32:27.198 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.195+0000 7f11958c2700 1 -- start start 2024-04-25T00:32:27.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.195+0000 7f11958c2700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f119006ac40 0x7f11901011a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:27.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.196+0000 7f11958c2700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f119006b5f0 0x7f11901016e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:27.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.196+0000 7f11958c2700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1190107590 0x7f1190101c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:27.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.196+0000 7f11958c2700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f1190112570 con 0x7f119006b5f0 2024-04-25T00:32:27.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.196+0000 7f11958c2700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f11901123f0 con 0x7f1190107590 2024-04-25T00:32:27.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.196+0000 7f11958c2700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f11901126f0 con 0x7f119006ac40 2024-04-25T00:32:27.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.196+0000 7f118e7fc700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f119006b5f0 0x7f11901016e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:27.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.196+0000 7f118e7fc700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f119006b5f0 0x7f11901016e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:41964/0 (socket says 172.21.15.32:41964) 2024-04-25T00:32:27.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.196+0000 7f118e7fc700 1 -- 172.21.15.32:0/4183149708 learned_addr learned my addr 172.21.15.32:0/4183149708 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:27.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.196+0000 7f118effd700 1 --2- 172.21.15.32:0/4183149708 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f119006ac40 0x7f11901011a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:27.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.196+0000 7f118f7fe700 1 --2- 172.21.15.32:0/4183149708 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1190107590 0x7f1190101c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:27.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.196+0000 7f118e7fc700 1 -- 172.21.15.32:0/4183149708 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f119006ac40 msgr2=0x7f11901011a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:27.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.196+0000 7f118e7fc700 1 --2- 172.21.15.32:0/4183149708 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f119006ac40 0x7f11901011a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:27.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.196+0000 7f118e7fc700 1 -- 172.21.15.32:0/4183149708 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1190107590 msgr2=0x7f1190101c20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:27.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.196+0000 7f118e7fc700 1 --2- 172.21.15.32:0/4183149708 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1190107590 0x7f1190101c20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:27.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.196+0000 7f118e7fc700 1 -- 172.21.15.32:0/4183149708 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f11901a6f30 con 0x7f119006b5f0 2024-04-25T00:32:27.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.196+0000 7f118e7fc700 1 --2- 172.21.15.32:0/4183149708 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f119006b5f0 0x7f11901016e0 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 crypto rx=0x7f118400c010 tx=0x7f11840047b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:27.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.196+0000 7f11948c0700 1 -- 172.21.15.32:0/4183149708 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1184004830 con 0x7f119006b5f0 2024-04-25T00:32:27.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.197+0000 7f11948c0700 1 -- 172.21.15.32:0/4183149708 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1184005020 con 0x7f119006b5f0 2024-04-25T00:32:27.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.197+0000 7f11958c2700 1 -- 172.21.15.32:0/4183149708 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f11901a71c0 con 0x7f119006b5f0 2024-04-25T00:32:27.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.197+0000 7f11948c0700 1 -- 172.21.15.32:0/4183149708 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1184003cb0 con 0x7f119006b5f0 2024-04-25T00:32:27.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.197+0000 7f11958c2700 1 -- 172.21.15.32:0/4183149708 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f11901a75b0 con 0x7f119006b5f0 2024-04-25T00:32:27.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.198+0000 7f11958c2700 1 -- 172.21.15.32:0/4183149708 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1190068f20 con 0x7f119006b5f0 2024-04-25T00:32:27.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.199+0000 7f11948c0700 1 -- 172.21.15.32:0/4183149708 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f11840049d0 con 0x7f119006b5f0 2024-04-25T00:32:27.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.199+0000 7f11948c0700 1 --2- 172.21.15.32:0/4183149708 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f11780a2cb0 0x7f11780a5170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:27.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.200+0000 7f118effd700 1 --2- 172.21.15.32:0/4183149708 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f11780a2cb0 0x7f11780a5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:27.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.200+0000 7f11948c0700 1 -- 172.21.15.32:0/4183149708 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f11840d0430 con 0x7f119006b5f0 2024-04-25T00:32:27.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.200+0000 7f118effd700 1 --2- 172.21.15.32:0/4183149708 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f11780a2cb0 0x7f11780a5170 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f1180004630 tx=0x7f118000b3c0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:27.209 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.208+0000 7f11948c0700 1 -- 172.21.15.32:0/4183149708 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f1184013070 con 0x7f119006b5f0 2024-04-25T00:32:27.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:26 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1163906092' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-25T00:32:27.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:26 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:32:27.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:26 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:32:27.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:26 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1163906092' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-25T00:32:27.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:26 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:32:27.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:26 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:32:27.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.421+0000 7f11958c2700 1 -- 172.21.15.32:0/4183149708 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 9} v 0) -- 0x7f119002dcf0 con 0x7f119006b5f0 2024-04-25T00:32:27.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.421+0000 7f11948c0700 1 -- 172.21.15.32:0/4183149708 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 9}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f118409cf60 con 0x7f119006b5f0 2024-04-25T00:32:27.422 INFO:teuthology.orchestra.run.smithi032.stdout:287762808876 2024-04-25T00:32:27.424 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.423+0000 7f11958c2700 1 -- 172.21.15.32:0/4183149708 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f11780a2cb0 msgr2=0x7f11780a5170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:27.424 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.423+0000 7f11958c2700 1 --2- 172.21.15.32:0/4183149708 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f11780a2cb0 0x7f11780a5170 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f1180004630 tx=0x7f118000b3c0 comp rx=0 tx=0).stop 2024-04-25T00:32:27.424 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.423+0000 7f11958c2700 1 -- 172.21.15.32:0/4183149708 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f119006b5f0 msgr2=0x7f11901016e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:27.424 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.423+0000 7f11958c2700 1 --2- 172.21.15.32:0/4183149708 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f119006b5f0 0x7f11901016e0 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 crypto rx=0x7f118400c010 tx=0x7f11840047b0 comp rx=0 tx=0).stop 2024-04-25T00:32:27.425 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.423+0000 7f11958c2700 1 -- 172.21.15.32:0/4183149708 shutdown_connections 2024-04-25T00:32:27.425 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.423+0000 7f11958c2700 1 --2- 172.21.15.32:0/4183149708 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f11780a2cb0 0x7f11780a5170 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:27.425 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.423+0000 7f11958c2700 1 --2- 172.21.15.32:0/4183149708 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1190107590 0x7f1190101c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:27.425 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.423+0000 7f11958c2700 1 --2- 172.21.15.32:0/4183149708 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f119006ac40 0x7f11901011a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:27.425 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.423+0000 7f11958c2700 1 --2- 172.21.15.32:0/4183149708 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f119006b5f0 0x7f11901016e0 unknown :-1 s=CLOSED pgs=339 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:27.425 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.423+0000 7f11958c2700 1 -- 172.21.15.32:0/4183149708 >> 172.21.15.32:0/4183149708 conn(0x7f11900781d0 msgr2=0x7f11900ff0e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:27.425 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.424+0000 7f11958c2700 1 -- 172.21.15.32:0/4183149708 shutdown_connections 2024-04-25T00:32:27.425 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:27.424+0000 7f11958c2700 1 -- 172.21.15.32:0/4183149708 wait complete. 2024-04-25T00:32:27.579 INFO:tasks.cephadm.ceph_manager.ceph:need seq 219043332143 got 219043332148 for osd.7 2024-04-25T00:32:27.579 DEBUG:teuthology.parallel:result is None 2024-04-25T00:32:28.214 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:27 smithi032 ceph-mon[29357]: pgmap v343: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:32:28.214 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:27 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/4183149708' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 9}]: dispatch 2024-04-25T00:32:28.248 INFO:tasks.cephadm.ceph_manager.ceph:need seq 287762808869 got 287762808876 for osd.9 2024-04-25T00:32:28.248 DEBUG:teuthology.parallel:result is None 2024-04-25T00:32:28.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:27 smithi121 ceph-mon[32231]: pgmap v343: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:32:28.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:27 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/4183149708' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 9}]: dispatch 2024-04-25T00:32:28.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:27 smithi130 ceph-mon[32525]: pgmap v343: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:32:28.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:27 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/4183149708' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 9}]: dispatch 2024-04-25T00:32:28.718 INFO:tasks.cephadm.ceph_manager.ceph:need seq 141733920830 got 141733920837 for osd.4 2024-04-25T00:32:28.718 DEBUG:teuthology.parallel:result is None 2024-04-25T00:32:28.719 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-25T00:32:28.719 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-25T00:32:28.972 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:32:29.239 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:28 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:32:29.239 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:28 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:32:29.239 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:28 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:32:29.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:28 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:32:29.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:28 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:32:29.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:28 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:32:29.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:32:29.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:32:29.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:28 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' 2024-04-25T00:32:30.666 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:30 smithi032 ceph-mon[29357]: pgmap v344: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 7 op/s 2024-04-25T00:32:30.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.665+0000 7f020288c700 1 -- 172.21.15.32:0/1171988094 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01fc107590 msgr2=0x7f01fc10f160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:30.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.665+0000 7f020288c700 1 --2- 172.21.15.32:0/1171988094 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01fc107590 0x7f01fc10f160 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 crypto rx=0x7f01f400b0d0 tx=0x7f01f400b3e0 comp rx=0 tx=0).stop 2024-04-25T00:32:30.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.665+0000 7f020288c700 1 -- 172.21.15.32:0/1171988094 shutdown_connections 2024-04-25T00:32:30.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.665+0000 7f020288c700 1 --2- 172.21.15.32:0/1171988094 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01fc107590 0x7f01fc10f160 unknown :-1 s=CLOSED pgs=340 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:30.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.665+0000 7f020288c700 1 --2- 172.21.15.32:0/1171988094 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f01fc06ac40 0x7f01fc06b020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:30.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.665+0000 7f020288c700 1 --2- 172.21.15.32:0/1171988094 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f01fc06b5f0 0x7f01fc106eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:30.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.665+0000 7f020288c700 1 -- 172.21.15.32:0/1171988094 >> 172.21.15.32:0/1171988094 conn(0x7f01fc0781d0 msgr2=0x7f01fc0785e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:30.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.665+0000 7f020288c700 1 -- 172.21.15.32:0/1171988094 shutdown_connections 2024-04-25T00:32:30.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.665+0000 7f020288c700 1 -- 172.21.15.32:0/1171988094 wait complete. 2024-04-25T00:32:30.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.666+0000 7f020288c700 1 Processor -- start 2024-04-25T00:32:30.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.666+0000 7f020288c700 1 -- start start 2024-04-25T00:32:30.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.666+0000 7f020288c700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f01fc06ac40 0x7f01fc19e650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:30.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.666+0000 7f020288c700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01fc06b5f0 0x7f01fc19eb90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:30.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.666+0000 7f020288c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f01fc107590 0x7f01fc1987d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:30.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.666+0000 7f020288c700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f01fc112590 con 0x7f01fc06b5f0 2024-04-25T00:32:30.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.666+0000 7f020288c700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f01fc112410 con 0x7f01fc06ac40 2024-04-25T00:32:30.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.666+0000 7f020288c700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f01fc112710 con 0x7f01fc107590 2024-04-25T00:32:30.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.666+0000 7f01fb7fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01fc06b5f0 0x7f01fc19eb90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:30.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.666+0000 7f01fb7fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01fc06b5f0 0x7f01fc19eb90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:41988/0 (socket says 172.21.15.32:41988) 2024-04-25T00:32:30.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.666+0000 7f01fb7fe700 1 -- 172.21.15.32:0/1808140333 learned_addr learned my addr 172.21.15.32:0/1808140333 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:30.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.666+0000 7f0200e29700 1 --2- 172.21.15.32:0/1808140333 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f01fc107590 0x7f01fc1987d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:30.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.667+0000 7f01fbfff700 1 --2- 172.21.15.32:0/1808140333 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f01fc06ac40 0x7f01fc19e650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:30.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.667+0000 7f01fb7fe700 1 -- 172.21.15.32:0/1808140333 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f01fc107590 msgr2=0x7f01fc1987d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:30.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.667+0000 7f01fb7fe700 1 --2- 172.21.15.32:0/1808140333 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f01fc107590 0x7f01fc1987d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:30.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.667+0000 7f01fb7fe700 1 -- 172.21.15.32:0/1808140333 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f01fc06ac40 msgr2=0x7f01fc19e650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:30.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.667+0000 7f01fb7fe700 1 --2- 172.21.15.32:0/1808140333 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f01fc06ac40 0x7f01fc19e650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:30.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.667+0000 7f01fb7fe700 1 -- 172.21.15.32:0/1808140333 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f01fc198fb0 con 0x7f01fc06b5f0 2024-04-25T00:32:30.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.667+0000 7f01fb7fe700 1 --2- 172.21.15.32:0/1808140333 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01fc06b5f0 0x7f01fc19eb90 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7f01e400d950 tx=0x7f01e400dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:30.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.667+0000 7f01f97fa700 1 -- 172.21.15.32:0/1808140333 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f01e4017070 con 0x7f01fc06b5f0 2024-04-25T00:32:30.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.667+0000 7f020288c700 1 -- 172.21.15.32:0/1808140333 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f01fc1992a0 con 0x7f01fc06b5f0 2024-04-25T00:32:30.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.667+0000 7f01f97fa700 1 -- 172.21.15.32:0/1808140333 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f01e4005020 con 0x7f01fc06b5f0 2024-04-25T00:32:30.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.667+0000 7f01f97fa700 1 -- 172.21.15.32:0/1808140333 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f01e4009b20 con 0x7f01fc06b5f0 2024-04-25T00:32:30.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.667+0000 7f020288c700 1 -- 172.21.15.32:0/1808140333 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f01fc1a54b0 con 0x7f01fc06b5f0 2024-04-25T00:32:30.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.668+0000 7f020288c700 1 -- 172.21.15.32:0/1808140333 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f01fc103990 con 0x7f01fc06b5f0 2024-04-25T00:32:30.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.669+0000 7f01f97fa700 1 -- 172.21.15.32:0/1808140333 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f01e40051d0 con 0x7f01fc06b5f0 2024-04-25T00:32:30.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.670+0000 7f01f97fa700 1 --2- 172.21.15.32:0/1808140333 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f01e80a2d30 0x7f01e80a51f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:30.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.670+0000 7f01fbfff700 1 --2- 172.21.15.32:0/1808140333 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f01e80a2d30 0x7f01e80a51f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:30.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.670+0000 7f01f97fa700 1 -- 172.21.15.32:0/1808140333 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f01e40ced70 con 0x7f01fc06b5f0 2024-04-25T00:32:30.726 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.670+0000 7f01fbfff700 1 --2- 172.21.15.32:0/1808140333 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f01e80a2d30 0x7f01e80a51f0 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f01ec005fd0 tx=0x7f01ec00a560 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:30.726 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.674+0000 7f01f97fa700 1 -- 172.21.15.32:0/1808140333 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f01e409b810 con 0x7f01fc06b5f0 2024-04-25T00:32:30.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:30 smithi121 ceph-mon[32231]: pgmap v344: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 7 op/s 2024-04-25T00:32:30.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.876+0000 7f020288c700 1 -- 172.21.15.32:0/1808140333 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f01fc062670 con 0x7f01e80a2d30 2024-04-25T00:32:30.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.886+0000 7f01f97fa700 1 -- 172.21.15.32:0/1808140333 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+340355 (secure 0 0 0) 0x7f01fc062670 con 0x7f01e80a2d30 2024-04-25T00:32:30.891 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:32:30.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.889+0000 7f020288c700 1 -- 172.21.15.32:0/1808140333 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f01e80a2d30 msgr2=0x7f01e80a51f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:30.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.889+0000 7f020288c700 1 --2- 172.21.15.32:0/1808140333 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f01e80a2d30 0x7f01e80a51f0 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f01ec005fd0 tx=0x7f01ec00a560 comp rx=0 tx=0).stop 2024-04-25T00:32:30.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.889+0000 7f020288c700 1 -- 172.21.15.32:0/1808140333 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01fc06b5f0 msgr2=0x7f01fc19eb90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:30.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.889+0000 7f020288c700 1 --2- 172.21.15.32:0/1808140333 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01fc06b5f0 0x7f01fc19eb90 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7f01e400d950 tx=0x7f01e400dc60 comp rx=0 tx=0).stop 2024-04-25T00:32:30.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.889+0000 7f020288c700 1 -- 172.21.15.32:0/1808140333 shutdown_connections 2024-04-25T00:32:30.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.889+0000 7f020288c700 1 --2- 172.21.15.32:0/1808140333 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f01e80a2d30 0x7f01e80a51f0 unknown :-1 s=CLOSED pgs=219 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:30.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.889+0000 7f020288c700 1 --2- 172.21.15.32:0/1808140333 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f01fc107590 0x7f01fc1987d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:30.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.889+0000 7f020288c700 1 --2- 172.21.15.32:0/1808140333 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f01fc06ac40 0x7f01fc19e650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:30.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.889+0000 7f020288c700 1 --2- 172.21.15.32:0/1808140333 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01fc06b5f0 0x7f01fc19eb90 unknown :-1 s=CLOSED pgs=341 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:30.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.889+0000 7f020288c700 1 -- 172.21.15.32:0/1808140333 >> 172.21.15.32:0/1808140333 conn(0x7f01fc0781d0 msgr2=0x7f01fc0ff080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:30.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.890+0000 7f020288c700 1 -- 172.21.15.32:0/1808140333 shutdown_connections 2024-04-25T00:32:30.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:30.890+0000 7f020288c700 1 -- 172.21.15.32:0/1808140333 wait complete. 2024-04-25T00:32:30.895 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-04-25T00:32:30.896 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:30 smithi130 ceph-mon[32525]: pgmap v344: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 7 op/s 2024-04-25T00:32:31.821 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":344,"stamp":"2024-04-25T00:32:28.886896+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":656378,"num_objects":64,"num_object_clones":0,"num_object_copies":194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":64,"num_whiteouts":0,"num_read":598,"num_read_kb":420,"num_write":342,"num_write_kb":3053,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1180853,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":277,"ondisk_log_size":277,"up":419,"acting":419,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":419,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":375776,"kb_used_data":14084,"kb_used_omap":18,"kb_used_meta":361517,"kb_avail":1124566048,"statfs":{"total":1151940427776,"available":1151555633152,"internally_reserved":0,"allocated":14422016,"data_stored":7954844,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19142,"internal_metadata":370193722},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":90,"num_read_kb":45,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003449"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.711419+0000","last_change":"2024-04-25T00:30:48.538719+0000","last_active":"2024-04-25T00:31:44.711419+0000","last_peered":"2024-04-25T00:31:44.711419+0000","last_clean":"2024-04-25T00:31:44.711419+0000","last_became_active":"2024-04-25T00:30:48.538356+0000","last_became_peered":"2024-04-25T00:30:48.538356+0000","last_unstale":"2024-04-25T00:31:44.711419+0000","last_undegraded":"2024-04-25T00:31:44.711419+0000","last_fullsized":"2024-04-25T00:31:44.711419+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:39:08.974838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707296+0000","last_change":"2024-04-25T00:30:48.532262+0000","last_active":"2024-04-25T00:31:44.707296+0000","last_peered":"2024-04-25T00:31:44.707296+0000","last_clean":"2024-04-25T00:31:44.707296+0000","last_became_active":"2024-04-25T00:30:48.531926+0000","last_became_peered":"2024-04-25T00:30:48.531926+0000","last_unstale":"2024-04-25T00:31:44.707296+0000","last_undegraded":"2024-04-25T00:31:44.707296+0000","last_fullsized":"2024-04-25T00:31:44.707296+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:12:06.527649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707060+0000","last_change":"2024-04-25T00:30:48.536383+0000","last_active":"2024-04-25T00:31:44.707060+0000","last_peered":"2024-04-25T00:31:44.707060+0000","last_clean":"2024-04-25T00:31:44.707060+0000","last_became_active":"2024-04-25T00:30:48.536048+0000","last_became_peered":"2024-04-25T00:30:48.536048+0000","last_unstale":"2024-04-25T00:31:44.707060+0000","last_undegraded":"2024-04-25T00:31:44.707060+0000","last_fullsized":"2024-04-25T00:31:44.707060+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:32:26.728441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707986+0000","last_change":"2024-04-25T00:30:48.532195+0000","last_active":"2024-04-25T00:31:44.707986+0000","last_peered":"2024-04-25T00:31:44.707986+0000","last_clean":"2024-04-25T00:31:44.707986+0000","last_became_active":"2024-04-25T00:30:48.531851+0000","last_became_peered":"2024-04-25T00:30:48.531851+0000","last_unstale":"2024-04-25T00:31:44.707986+0000","last_undegraded":"2024-04-25T00:31:44.707986+0000","last_fullsized":"2024-04-25T00:31:44.707986+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:15:34.704958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710960+0000","last_change":"2024-04-25T00:30:48.534024+0000","last_active":"2024-04-25T00:31:44.710960+0000","last_peered":"2024-04-25T00:31:44.710960+0000","last_clean":"2024-04-25T00:31:44.710960+0000","last_became_active":"2024-04-25T00:30:48.533681+0000","last_became_peered":"2024-04-25T00:30:48.533681+0000","last_unstale":"2024-04-25T00:31:44.710960+0000","last_undegraded":"2024-04-25T00:31:44.710960+0000","last_fullsized":"2024-04-25T00:31:44.710960+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:07:35.816816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707570+0000","last_change":"2024-04-25T00:30:48.541263+0000","last_active":"2024-04-25T00:31:44.707570+0000","last_peered":"2024-04-25T00:31:44.707570+0000","last_clean":"2024-04-25T00:31:44.707570+0000","last_became_active":"2024-04-25T00:30:48.540889+0000","last_became_peered":"2024-04-25T00:30:48.540889+0000","last_unstale":"2024-04-25T00:31:44.707570+0000","last_undegraded":"2024-04-25T00:31:44.707570+0000","last_fullsized":"2024-04-25T00:31:44.707570+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:31:42.067004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,4],"acting":[10,9,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710287+0000","last_change":"2024-04-25T00:30:48.533194+0000","last_active":"2024-04-25T00:31:44.710287+0000","last_peered":"2024-04-25T00:31:44.710287+0000","last_clean":"2024-04-25T00:31:44.710287+0000","last_became_active":"2024-04-25T00:30:48.532424+0000","last_became_peered":"2024-04-25T00:30:48.532424+0000","last_unstale":"2024-04-25T00:31:44.710287+0000","last_undegraded":"2024-04-25T00:31:44.710287+0000","last_fullsized":"2024-04-25T00:31:44.710287+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:06:50.187796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.169414+0000","last_change":"2024-04-25T00:30:48.542829+0000","last_active":"2024-04-25T00:31:45.169414+0000","last_peered":"2024-04-25T00:31:45.169414+0000","last_clean":"2024-04-25T00:31:45.169414+0000","last_became_active":"2024-04-25T00:30:48.542204+0000","last_became_peered":"2024-04-25T00:30:48.542204+0000","last_unstale":"2024-04-25T00:31:45.169414+0000","last_undegraded":"2024-04-25T00:31:45.169414+0000","last_fullsized":"2024-04-25T00:31:45.169414+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:56:47.869075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705841+0000","last_change":"2024-04-25T00:30:48.537924+0000","last_active":"2024-04-25T00:31:44.705841+0000","last_peered":"2024-04-25T00:31:44.705841+0000","last_clean":"2024-04-25T00:31:44.705841+0000","last_became_active":"2024-04-25T00:30:48.537547+0000","last_became_peered":"2024-04-25T00:30:48.537547+0000","last_unstale":"2024-04-25T00:31:44.705841+0000","last_undegraded":"2024-04-25T00:31:44.705841+0000","last_fullsized":"2024-04-25T00:31:44.705841+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:26:45.590923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705846+0000","last_change":"2024-04-25T00:30:48.539717+0000","last_active":"2024-04-25T00:31:44.705846+0000","last_peered":"2024-04-25T00:31:44.705846+0000","last_clean":"2024-04-25T00:31:44.705846+0000","last_became_active":"2024-04-25T00:30:48.539242+0000","last_became_peered":"2024-04-25T00:30:48.539242+0000","last_unstale":"2024-04-25T00:31:44.705846+0000","last_undegraded":"2024-04-25T00:31:44.705846+0000","last_fullsized":"2024-04-25T00:31:44.705846+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:58:15.353959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710657+0000","last_change":"2024-04-25T00:30:48.542767+0000","last_active":"2024-04-25T00:31:44.710657+0000","last_peered":"2024-04-25T00:31:44.710657+0000","last_clean":"2024-04-25T00:31:44.710657+0000","last_became_active":"2024-04-25T00:30:48.542252+0000","last_became_peered":"2024-04-25T00:30:48.542252+0000","last_unstale":"2024-04-25T00:31:44.710657+0000","last_undegraded":"2024-04-25T00:31:44.710657+0000","last_fullsized":"2024-04-25T00:31:44.710657+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:07:49.414660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707985+0000","last_change":"2024-04-25T00:30:48.532248+0000","last_active":"2024-04-25T00:31:44.707985+0000","last_peered":"2024-04-25T00:31:44.707985+0000","last_clean":"2024-04-25T00:31:44.707985+0000","last_became_active":"2024-04-25T00:30:48.531876+0000","last_became_peered":"2024-04-25T00:30:48.531876+0000","last_unstale":"2024-04-25T00:31:44.707985+0000","last_undegraded":"2024-04-25T00:31:44.707985+0000","last_fullsized":"2024-04-25T00:31:44.707985+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:00:35.711693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.027601+0000","last_change":"2024-04-25T00:30:48.526871+0000","last_active":"2024-04-25T00:31:45.027601+0000","last_peered":"2024-04-25T00:31:45.027601+0000","last_clean":"2024-04-25T00:31:45.027601+0000","last_became_active":"2024-04-25T00:30:48.526477+0000","last_became_peered":"2024-04-25T00:30:48.526477+0000","last_unstale":"2024-04-25T00:31:45.027601+0000","last_undegraded":"2024-04-25T00:31:45.027601+0000","last_fullsized":"2024-04-25T00:31:45.027601+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:17:59.447932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707068+0000","last_change":"2024-04-25T00:30:48.530297+0000","last_active":"2024-04-25T00:31:44.707068+0000","last_peered":"2024-04-25T00:31:44.707068+0000","last_clean":"2024-04-25T00:31:44.707068+0000","last_became_active":"2024-04-25T00:30:48.529091+0000","last_became_peered":"2024-04-25T00:30:48.529091+0000","last_unstale":"2024-04-25T00:31:44.707068+0000","last_undegraded":"2024-04-25T00:31:44.707068+0000","last_fullsized":"2024-04-25T00:31:44.707068+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:54:52.206881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.706839+0000","last_change":"2024-04-25T00:30:48.535629+0000","last_active":"2024-04-25T00:31:44.706839+0000","last_peered":"2024-04-25T00:31:44.706839+0000","last_clean":"2024-04-25T00:31:44.706839+0000","last_became_active":"2024-04-25T00:30:48.535190+0000","last_became_peered":"2024-04-25T00:30:48.535190+0000","last_unstale":"2024-04-25T00:31:44.706839+0000","last_undegraded":"2024-04-25T00:31:44.706839+0000","last_fullsized":"2024-04-25T00:31:44.706839+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:11:27.127964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.704421+0000","last_change":"2024-04-25T00:30:48.532988+0000","last_active":"2024-04-25T00:31:44.704421+0000","last_peered":"2024-04-25T00:31:44.704421+0000","last_clean":"2024-04-25T00:31:44.704421+0000","last_became_active":"2024-04-25T00:30:48.532670+0000","last_became_peered":"2024-04-25T00:30:48.532670+0000","last_unstale":"2024-04-25T00:31:44.704421+0000","last_undegraded":"2024-04-25T00:31:44.704421+0000","last_fullsized":"2024-04-25T00:31:44.704421+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:03:35.336175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.711247+0000","last_change":"2024-04-25T00:30:48.539499+0000","last_active":"2024-04-25T00:31:44.711247+0000","last_peered":"2024-04-25T00:31:44.711247+0000","last_clean":"2024-04-25T00:31:44.711247+0000","last_became_active":"2024-04-25T00:30:48.538516+0000","last_became_peered":"2024-04-25T00:30:48.538516+0000","last_unstale":"2024-04-25T00:31:44.711247+0000","last_undegraded":"2024-04-25T00:31:44.711247+0000","last_fullsized":"2024-04-25T00:31:44.711247+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:03:16.781091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.703779+0000","last_change":"2024-04-25T00:30:48.534204+0000","last_active":"2024-04-25T00:31:44.703779+0000","last_peered":"2024-04-25T00:31:44.703779+0000","last_clean":"2024-04-25T00:31:44.703779+0000","last_became_active":"2024-04-25T00:30:48.533818+0000","last_became_peered":"2024-04-25T00:30:48.533818+0000","last_unstale":"2024-04-25T00:31:44.703779+0000","last_undegraded":"2024-04-25T00:31:44.703779+0000","last_fullsized":"2024-04-25T00:31:44.703779+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:08:42.964056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707214+0000","last_change":"2024-04-25T00:30:48.542756+0000","last_active":"2024-04-25T00:31:44.707214+0000","last_peered":"2024-04-25T00:31:44.707214+0000","last_clean":"2024-04-25T00:31:44.707214+0000","last_became_active":"2024-04-25T00:30:48.541824+0000","last_became_peered":"2024-04-25T00:30:48.541824+0000","last_unstale":"2024-04-25T00:31:44.707214+0000","last_undegraded":"2024-04-25T00:31:44.707214+0000","last_fullsized":"2024-04-25T00:31:44.707214+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:31:40.996474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.706047+0000","last_change":"2024-04-25T00:30:52.788477+0000","last_active":"2024-04-25T00:31:44.706047+0000","last_peered":"2024-04-25T00:31:44.706047+0000","last_clean":"2024-04-25T00:31:44.706047+0000","last_became_active":"2024-04-25T00:30:52.788073+0000","last_became_peered":"2024-04-25T00:30:52.788073+0000","last_unstale":"2024-04-25T00:31:44.706047+0000","last_undegraded":"2024-04-25T00:31:44.706047+0000","last_fullsized":"2024-04-25T00:31:44.706047+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:29:34.781119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,9,0],"acting":[2,3,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"89'9","reported_seq":37,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.703897+0000","last_change":"2024-04-25T00:30:47.524580+0000","last_active":"2024-04-25T00:31:44.703897+0000","last_peered":"2024-04-25T00:31:44.703897+0000","last_clean":"2024-04-25T00:31:44.703897+0000","last_became_active":"2024-04-25T00:30:47.524170+0000","last_became_peered":"2024-04-25T00:30:47.524170+0000","last_unstale":"2024-04-25T00:31:44.703897+0000","last_undegraded":"2024-04-25T00:31:44.703897+0000","last_fullsized":"2024-04-25T00:31:44.703897+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:53:08.734842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.169997+0000","last_change":"2024-04-25T00:30:48.542077+0000","last_active":"2024-04-25T00:31:45.169997+0000","last_peered":"2024-04-25T00:31:45.169997+0000","last_clean":"2024-04-25T00:31:45.169997+0000","last_became_active":"2024-04-25T00:30:48.539043+0000","last_became_peered":"2024-04-25T00:30:48.539043+0000","last_unstale":"2024-04-25T00:31:45.169997+0000","last_undegraded":"2024-04-25T00:31:45.169997+0000","last_fullsized":"2024-04-25T00:31:45.169997+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:48:52.254800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705852+0000","last_change":"2024-04-25T00:30:52.786091+0000","last_active":"2024-04-25T00:31:44.705852+0000","last_peered":"2024-04-25T00:31:44.705852+0000","last_clean":"2024-04-25T00:31:44.705852+0000","last_became_active":"2024-04-25T00:30:52.785511+0000","last_became_peered":"2024-04-25T00:30:52.785511+0000","last_unstale":"2024-04-25T00:31:44.705852+0000","last_undegraded":"2024-04-25T00:31:44.705852+0000","last_fullsized":"2024-04-25T00:31:44.705852+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:19:17.992033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,9,3],"acting":[4,6,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1e","version":"89'2","reported_seq":29,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.170167+0000","last_change":"2024-04-25T00:30:47.529293+0000","last_active":"2024-04-25T00:31:45.170167+0000","last_peered":"2024-04-25T00:31:45.170167+0000","last_clean":"2024-04-25T00:31:45.170167+0000","last_became_active":"2024-04-25T00:30:47.527975+0000","last_became_peered":"2024-04-25T00:30:47.527975+0000","last_unstale":"2024-04-25T00:31:45.170167+0000","last_undegraded":"2024-04-25T00:31:45.170167+0000","last_fullsized":"2024-04-25T00:31:45.170167+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:17:40.057962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707444+0000","last_change":"2024-04-25T00:30:48.530993+0000","last_active":"2024-04-25T00:31:44.707444+0000","last_peered":"2024-04-25T00:31:44.707444+0000","last_clean":"2024-04-25T00:31:44.707444+0000","last_became_active":"2024-04-25T00:30:48.530550+0000","last_became_peered":"2024-04-25T00:30:48.530550+0000","last_unstale":"2024-04-25T00:31:44.707444+0000","last_undegraded":"2024-04-25T00:31:44.707444+0000","last_fullsized":"2024-04-25T00:31:44.707444+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:11:35.308456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.706981+0000","last_change":"2024-04-25T00:30:52.787774+0000","last_active":"2024-04-25T00:31:44.706981+0000","last_peered":"2024-04-25T00:31:44.706981+0000","last_clean":"2024-04-25T00:31:44.706981+0000","last_became_active":"2024-04-25T00:30:52.786988+0000","last_became_peered":"2024-04-25T00:30:52.786988+0000","last_unstale":"2024-04-25T00:31:44.706981+0000","last_undegraded":"2024-04-25T00:31:44.706981+0000","last_fullsized":"2024-04-25T00:31:44.706981+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:58:28.081767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,9,5],"acting":[4,3,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1d","version":"89'4","reported_seq":31,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.711899+0000","last_change":"2024-04-25T00:30:47.523477+0000","last_active":"2024-04-25T00:31:44.711899+0000","last_peered":"2024-04-25T00:31:44.711899+0000","last_clean":"2024-04-25T00:31:44.711899+0000","last_became_active":"2024-04-25T00:30:47.522961+0000","last_became_peered":"2024-04-25T00:30:47.522961+0000","last_unstale":"2024-04-25T00:31:44.711899+0000","last_undegraded":"2024-04-25T00:31:44.711899+0000","last_fullsized":"2024-04-25T00:31:44.711899+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:39:39.793477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705019+0000","last_change":"2024-04-25T00:30:48.527746+0000","last_active":"2024-04-25T00:31:44.705019+0000","last_peered":"2024-04-25T00:31:44.705019+0000","last_clean":"2024-04-25T00:31:44.705019+0000","last_became_active":"2024-04-25T00:30:48.527374+0000","last_became_peered":"2024-04-25T00:30:48.527374+0000","last_unstale":"2024-04-25T00:31:44.705019+0000","last_undegraded":"2024-04-25T00:31:44.705019+0000","last_fullsized":"2024-04-25T00:31:44.705019+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:28:36.822220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705693+0000","last_change":"2024-04-25T00:30:52.782368+0000","last_active":"2024-04-25T00:31:44.705693+0000","last_peered":"2024-04-25T00:31:44.705693+0000","last_clean":"2024-04-25T00:31:44.705693+0000","last_became_active":"2024-04-25T00:30:52.781875+0000","last_became_peered":"2024-04-25T00:30:52.781875+0000","last_unstale":"2024-04-25T00:31:44.705693+0000","last_undegraded":"2024-04-25T00:31:44.705693+0000","last_fullsized":"2024-04-25T00:31:44.705693+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:07:55.509131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5,3],"acting":[4,6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1c","version":"84'2","reported_seq":29,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.711538+0000","last_change":"2024-04-25T00:30:47.523707+0000","last_active":"2024-04-25T00:31:44.711538+0000","last_peered":"2024-04-25T00:31:44.711538+0000","last_clean":"2024-04-25T00:31:44.711538+0000","last_became_active":"2024-04-25T00:30:47.523216+0000","last_became_peered":"2024-04-25T00:30:47.523216+0000","last_unstale":"2024-04-25T00:31:44.711538+0000","last_undegraded":"2024-04-25T00:31:44.711538+0000","last_fullsized":"2024-04-25T00:31:44.711538+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:21:21.006368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.029744+0000","last_change":"2024-04-25T00:30:48.529235+0000","last_active":"2024-04-25T00:31:45.029744+0000","last_peered":"2024-04-25T00:31:45.029744+0000","last_clean":"2024-04-25T00:31:45.029744+0000","last_became_active":"2024-04-25T00:30:48.528887+0000","last_became_peered":"2024-04-25T00:30:48.528887+0000","last_unstale":"2024-04-25T00:31:45.029744+0000","last_undegraded":"2024-04-25T00:31:45.029744+0000","last_fullsized":"2024-04-25T00:31:45.029744+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:18:05.290578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.714433+0000","last_change":"2024-04-25T00:31:44.714433+0000","last_active":"2024-04-25T00:31:44.714433+0000","last_peered":"2024-04-25T00:31:44.714433+0000","last_clean":"2024-04-25T00:31:44.714433+0000","last_became_active":"2024-04-25T00:31:44.713810+0000","last_became_peered":"2024-04-25T00:31:44.713810+0000","last_unstale":"2024-04-25T00:31:44.714433+0000","last_undegraded":"2024-04-25T00:31:44.714433+0000","last_fullsized":"2024-04-25T00:31:44.714433+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:00:49.815134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,7,11],"acting":[2,8,7,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1b","version":"89'5","reported_seq":107,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:32:26.088688+0000","last_change":"2024-04-25T00:30:47.518472+0000","last_active":"2024-04-25T00:32:26.088688+0000","last_peered":"2024-04-25T00:32:26.088688+0000","last_clean":"2024-04-25T00:32:26.088688+0000","last_became_active":"2024-04-25T00:30:47.518140+0000","last_became_peered":"2024-04-25T00:30:47.518140+0000","last_unstale":"2024-04-25T00:32:26.088688+0000","last_undegraded":"2024-04-25T00:32:26.088688+0000","last_fullsized":"2024-04-25T00:32:26.088688+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:19:07.780949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":622,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":75,"num_read_kb":2,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710517+0000","last_change":"2024-04-25T00:30:48.533757+0000","last_active":"2024-04-25T00:31:44.710517+0000","last_peered":"2024-04-25T00:31:44.710517+0000","last_clean":"2024-04-25T00:31:44.710517+0000","last_became_active":"2024-04-25T00:30:48.533421+0000","last_became_peered":"2024-04-25T00:30:48.533421+0000","last_unstale":"2024-04-25T00:31:44.710517+0000","last_undegraded":"2024-04-25T00:31:44.710517+0000","last_fullsized":"2024-04-25T00:31:44.710517+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:11:38.190783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.031412+0000","last_change":"2024-04-25T00:31:45.031412+0000","last_active":"2024-04-25T00:31:45.031412+0000","last_peered":"2024-04-25T00:31:45.031412+0000","last_clean":"2024-04-25T00:31:45.031412+0000","last_became_active":"2024-04-25T00:31:45.030822+0000","last_became_peered":"2024-04-25T00:31:45.030822+0000","last_unstale":"2024-04-25T00:31:45.031412+0000","last_undegraded":"2024-04-25T00:31:45.031412+0000","last_fullsized":"2024-04-25T00:31:45.031412+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:34:26.758966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,8,5],"acting":[2,6,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1a","version":"89'3","reported_seq":30,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707538+0000","last_change":"2024-04-25T00:30:47.527306+0000","last_active":"2024-04-25T00:31:44.707538+0000","last_peered":"2024-04-25T00:31:44.707538+0000","last_clean":"2024-04-25T00:31:44.707538+0000","last_became_active":"2024-04-25T00:30:47.526935+0000","last_became_peered":"2024-04-25T00:30:47.526935+0000","last_unstale":"2024-04-25T00:31:44.707538+0000","last_undegraded":"2024-04-25T00:31:44.707538+0000","last_fullsized":"2024-04-25T00:31:44.707538+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:41:37.951143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":11,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.171065+0000","last_change":"2024-04-25T00:31:45.170610+0000","last_active":"2024-04-25T00:31:45.171065+0000","last_peered":"2024-04-25T00:31:45.171065+0000","last_clean":"2024-04-25T00:31:45.171065+0000","last_became_active":"2024-04-25T00:31:45.170183+0000","last_became_peered":"2024-04-25T00:31:45.170183+0000","last_unstale":"2024-04-25T00:31:45.171065+0000","last_undegraded":"2024-04-25T00:31:45.171065+0000","last_fullsized":"2024-04-25T00:31:45.171065+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:39:39.781997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,4,3],"acting":[9,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710518+0000","last_change":"2024-04-25T00:30:52.784986+0000","last_active":"2024-04-25T00:31:44.710518+0000","last_peered":"2024-04-25T00:31:44.710518+0000","last_clean":"2024-04-25T00:31:44.710518+0000","last_became_active":"2024-04-25T00:30:52.784593+0000","last_became_peered":"2024-04-25T00:30:52.784593+0000","last_unstale":"2024-04-25T00:31:44.710518+0000","last_undegraded":"2024-04-25T00:31:44.710518+0000","last_fullsized":"2024-04-25T00:31:44.710518+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:04:05.833730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,9,0],"acting":[11,4,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.19","version":"89'2","reported_seq":29,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.027126+0000","last_change":"2024-04-25T00:30:47.523663+0000","last_active":"2024-04-25T00:31:45.027126+0000","last_peered":"2024-04-25T00:31:45.027126+0000","last_clean":"2024-04-25T00:31:45.027126+0000","last_became_active":"2024-04-25T00:30:47.523324+0000","last_became_peered":"2024-04-25T00:30:47.523324+0000","last_unstale":"2024-04-25T00:31:45.027126+0000","last_undegraded":"2024-04-25T00:31:45.027126+0000","last_fullsized":"2024-04-25T00:31:45.027126+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:51:23.593579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.169180+0000","last_change":"2024-04-25T00:30:48.529738+0000","last_active":"2024-04-25T00:31:45.169180+0000","last_peered":"2024-04-25T00:31:45.169180+0000","last_clean":"2024-04-25T00:31:45.169180+0000","last_became_active":"2024-04-25T00:30:48.529350+0000","last_became_peered":"2024-04-25T00:30:48.529350+0000","last_unstale":"2024-04-25T00:31:45.169180+0000","last_undegraded":"2024-04-25T00:31:45.169180+0000","last_fullsized":"2024-04-25T00:31:45.169180+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:51:29.238293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707182+0000","last_change":"2024-04-25T00:30:52.788480+0000","last_active":"2024-04-25T00:31:44.707182+0000","last_peered":"2024-04-25T00:31:44.707182+0000","last_clean":"2024-04-25T00:31:44.707182+0000","last_became_active":"2024-04-25T00:30:52.788065+0000","last_became_peered":"2024-04-25T00:30:52.788065+0000","last_unstale":"2024-04-25T00:31:44.707182+0000","last_undegraded":"2024-04-25T00:31:44.707182+0000","last_fullsized":"2024-04-25T00:31:44.707182+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:16:50.179177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6,1],"acting":[0,7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.18","version":"89'1","reported_seq":28,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.029206+0000","last_change":"2024-04-25T00:30:47.524706+0000","last_active":"2024-04-25T00:31:45.029206+0000","last_peered":"2024-04-25T00:31:45.029206+0000","last_clean":"2024-04-25T00:31:45.029206+0000","last_became_active":"2024-04-25T00:30:47.524355+0000","last_became_peered":"2024-04-25T00:30:47.524355+0000","last_unstale":"2024-04-25T00:31:45.029206+0000","last_undegraded":"2024-04-25T00:31:45.029206+0000","last_fullsized":"2024-04-25T00:31:45.029206+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:22:38.093362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.703331+0000","last_change":"2024-04-25T00:30:48.533266+0000","last_active":"2024-04-25T00:31:44.703331+0000","last_peered":"2024-04-25T00:31:44.703331+0000","last_clean":"2024-04-25T00:31:44.703331+0000","last_became_active":"2024-04-25T00:30:48.532932+0000","last_became_peered":"2024-04-25T00:30:48.532932+0000","last_unstale":"2024-04-25T00:31:44.703331+0000","last_undegraded":"2024-04-25T00:31:44.703331+0000","last_fullsized":"2024-04-25T00:31:44.703331+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:00:54.147725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.169493+0000","last_change":"2024-04-25T00:30:52.780317+0000","last_active":"2024-04-25T00:31:45.169493+0000","last_peered":"2024-04-25T00:31:45.169493+0000","last_clean":"2024-04-25T00:31:45.169493+0000","last_became_active":"2024-04-25T00:30:52.778539+0000","last_became_peered":"2024-04-25T00:30:52.778539+0000","last_unstale":"2024-04-25T00:31:45.169493+0000","last_undegraded":"2024-04-25T00:31:45.169493+0000","last_fullsized":"2024-04-25T00:31:45.169493+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:34:49.527307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1,5],"acting":[3,7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.17","version":"89'3","reported_seq":102,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:32:25.086677+0000","last_change":"2024-04-25T00:30:47.525647+0000","last_active":"2024-04-25T00:32:25.086677+0000","last_peered":"2024-04-25T00:32:25.086677+0000","last_clean":"2024-04-25T00:32:25.086677+0000","last_became_active":"2024-04-25T00:30:47.525280+0000","last_became_peered":"2024-04-25T00:30:47.525280+0000","last_unstale":"2024-04-25T00:32:25.086677+0000","last_undegraded":"2024-04-25T00:32:25.086677+0000","last_fullsized":"2024-04-25T00:32:25.086677+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:56:15.761173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":72,"num_read_kb":72,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705528+0000","last_change":"2024-04-25T00:30:48.527913+0000","last_active":"2024-04-25T00:31:44.705528+0000","last_peered":"2024-04-25T00:31:44.705528+0000","last_clean":"2024-04-25T00:31:44.705528+0000","last_became_active":"2024-04-25T00:30:48.527538+0000","last_became_peered":"2024-04-25T00:30:48.527538+0000","last_unstale":"2024-04-25T00:31:44.705528+0000","last_undegraded":"2024-04-25T00:31:44.705528+0000","last_fullsized":"2024-04-25T00:31:44.705528+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:48:38.646867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.703108+0000","last_change":"2024-04-25T00:30:52.788267+0000","last_active":"2024-04-25T00:31:44.703108+0000","last_peered":"2024-04-25T00:31:44.703108+0000","last_clean":"2024-04-25T00:31:44.703108+0000","last_became_active":"2024-04-25T00:30:52.785188+0000","last_became_peered":"2024-04-25T00:30:52.785188+0000","last_unstale":"2024-04-25T00:31:44.703108+0000","last_undegraded":"2024-04-25T00:31:44.703108+0000","last_fullsized":"2024-04-25T00:31:44.703108+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:38:25.194673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,0,2,4],"acting":[9,0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.16","version":"89'2","reported_seq":29,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710588+0000","last_change":"2024-04-25T00:30:47.522213+0000","last_active":"2024-04-25T00:31:44.710588+0000","last_peered":"2024-04-25T00:31:44.710588+0000","last_clean":"2024-04-25T00:31:44.710588+0000","last_became_active":"2024-04-25T00:30:47.521519+0000","last_became_peered":"2024-04-25T00:30:47.521519+0000","last_unstale":"2024-04-25T00:31:44.710588+0000","last_undegraded":"2024-04-25T00:31:44.710588+0000","last_fullsized":"2024-04-25T00:31:44.710588+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:16:10.001222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":31,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.029866+0000","last_change":"2024-04-25T00:31:45.029866+0000","last_active":"2024-04-25T00:31:45.029866+0000","last_peered":"2024-04-25T00:31:45.029866+0000","last_clean":"2024-04-25T00:31:45.029866+0000","last_became_active":"2024-04-25T00:31:45.029587+0000","last_became_peered":"2024-04-25T00:31:45.029587+0000","last_unstale":"2024-04-25T00:31:45.029866+0000","last_undegraded":"2024-04-25T00:31:45.029866+0000","last_fullsized":"2024-04-25T00:31:45.029866+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:09:09.434492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707134+0000","last_change":"2024-04-25T00:30:52.786818+0000","last_active":"2024-04-25T00:31:44.707134+0000","last_peered":"2024-04-25T00:31:44.707134+0000","last_clean":"2024-04-25T00:31:44.707134+0000","last_became_active":"2024-04-25T00:30:52.786293+0000","last_became_peered":"2024-04-25T00:30:52.786293+0000","last_unstale":"2024-04-25T00:31:44.707134+0000","last_undegraded":"2024-04-25T00:31:44.707134+0000","last_fullsized":"2024-04-25T00:31:44.707134+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:27:02.751296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2,9],"acting":[4,1,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.15","version":"89'4","reported_seq":32,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.029587+0000","last_change":"2024-04-25T00:30:47.520882+0000","last_active":"2024-04-25T00:31:45.029587+0000","last_peered":"2024-04-25T00:31:45.029587+0000","last_clean":"2024-04-25T00:31:45.029587+0000","last_became_active":"2024-04-25T00:30:47.520545+0000","last_became_peered":"2024-04-25T00:30:47.520545+0000","last_unstale":"2024-04-25T00:31:45.029587+0000","last_undegraded":"2024-04-25T00:31:45.029587+0000","last_fullsized":"2024-04-25T00:31:45.029587+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:46:04.715580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705195+0000","last_change":"2024-04-25T00:30:48.536814+0000","last_active":"2024-04-25T00:31:44.705195+0000","last_peered":"2024-04-25T00:31:44.705195+0000","last_clean":"2024-04-25T00:31:44.705195+0000","last_became_active":"2024-04-25T00:30:48.536486+0000","last_became_peered":"2024-04-25T00:30:48.536486+0000","last_unstale":"2024-04-25T00:31:44.705195+0000","last_undegraded":"2024-04-25T00:31:44.705195+0000","last_fullsized":"2024-04-25T00:31:44.705195+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:58:23.861959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.031626+0000","last_change":"2024-04-25T00:31:45.031208+0000","last_active":"2024-04-25T00:31:45.031626+0000","last_peered":"2024-04-25T00:31:45.031626+0000","last_clean":"2024-04-25T00:31:45.031626+0000","last_became_active":"2024-04-25T00:31:45.030831+0000","last_became_peered":"2024-04-25T00:31:45.030831+0000","last_unstale":"2024-04-25T00:31:45.031626+0000","last_undegraded":"2024-04-25T00:31:45.031626+0000","last_fullsized":"2024-04-25T00:31:45.031626+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:42:01.869146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,11,5],"acting":[10,8,11,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.14","version":"89'31","reported_seq":209,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:32:26.088101+0000","last_change":"2024-04-25T00:30:47.526899+0000","last_active":"2024-04-25T00:32:26.088101+0000","last_peered":"2024-04-25T00:32:26.088101+0000","last_clean":"2024-04-25T00:32:26.088101+0000","last_became_active":"2024-04-25T00:30:47.526528+0000","last_became_peered":"2024-04-25T00:30:47.526528+0000","last_unstale":"2024-04-25T00:32:26.088101+0000","last_undegraded":"2024-04-25T00:32:26.088101+0000","last_fullsized":"2024-04-25T00:32:26.088101+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":31,"log_dups_size":0,"ondisk_log_size":31,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:13:34.004584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61775,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":151,"num_read_kb":136,"num_write":32,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.712063+0000","last_change":"2024-04-25T00:30:48.538225+0000","last_active":"2024-04-25T00:31:44.712063+0000","last_peered":"2024-04-25T00:31:44.712063+0000","last_clean":"2024-04-25T00:31:44.712063+0000","last_became_active":"2024-04-25T00:30:48.537807+0000","last_became_peered":"2024-04-25T00:30:48.537807+0000","last_unstale":"2024-04-25T00:31:44.712063+0000","last_undegraded":"2024-04-25T00:31:44.712063+0000","last_fullsized":"2024-04-25T00:31:44.712063+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:00:46.275584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.029972+0000","last_change":"2024-04-25T00:30:52.782266+0000","last_active":"2024-04-25T00:31:45.029972+0000","last_peered":"2024-04-25T00:31:45.029972+0000","last_clean":"2024-04-25T00:31:45.029972+0000","last_became_active":"2024-04-25T00:30:52.781133+0000","last_became_peered":"2024-04-25T00:30:52.781133+0000","last_unstale":"2024-04-25T00:31:45.029972+0000","last_undegraded":"2024-04-25T00:31:45.029972+0000","last_fullsized":"2024-04-25T00:31:45.029972+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:19:03.818560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,8,3,6],"acting":[5,8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.13","version":"89'6","reported_seq":34,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.706421+0000","last_change":"2024-04-25T00:30:47.520909+0000","last_active":"2024-04-25T00:31:44.706421+0000","last_peered":"2024-04-25T00:31:44.706421+0000","last_clean":"2024-04-25T00:31:44.706421+0000","last_became_active":"2024-04-25T00:30:47.520287+0000","last_became_peered":"2024-04-25T00:30:47.520287+0000","last_unstale":"2024-04-25T00:31:44.706421+0000","last_undegraded":"2024-04-25T00:31:44.706421+0000","last_fullsized":"2024-04-25T00:31:44.706421+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:49:43.795115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.027156+0000","last_change":"2024-04-25T00:30:48.532270+0000","last_active":"2024-04-25T00:31:45.027156+0000","last_peered":"2024-04-25T00:31:45.027156+0000","last_clean":"2024-04-25T00:31:45.027156+0000","last_became_active":"2024-04-25T00:30:48.531882+0000","last_became_peered":"2024-04-25T00:30:48.531882+0000","last_unstale":"2024-04-25T00:31:45.027156+0000","last_undegraded":"2024-04-25T00:31:45.027156+0000","last_fullsized":"2024-04-25T00:31:45.027156+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:58:17.865635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.711177+0000","last_change":"2024-04-25T00:30:52.777708+0000","last_active":"2024-04-25T00:31:44.711177+0000","last_peered":"2024-04-25T00:31:44.711177+0000","last_clean":"2024-04-25T00:31:44.711177+0000","last_became_active":"2024-04-25T00:30:52.777254+0000","last_became_peered":"2024-04-25T00:30:52.777254+0000","last_unstale":"2024-04-25T00:31:44.711177+0000","last_undegraded":"2024-04-25T00:31:44.711177+0000","last_fullsized":"2024-04-25T00:31:44.711177+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:29:47.427190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,9,7,6],"acting":[11,9,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.12","version":"89'3","reported_seq":30,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.029488+0000","last_change":"2024-04-25T00:30:47.524667+0000","last_active":"2024-04-25T00:31:45.029488+0000","last_peered":"2024-04-25T00:31:45.029488+0000","last_clean":"2024-04-25T00:31:45.029488+0000","last_became_active":"2024-04-25T00:30:47.524361+0000","last_became_peered":"2024-04-25T00:30:47.524361+0000","last_unstale":"2024-04-25T00:31:45.029488+0000","last_undegraded":"2024-04-25T00:31:45.029488+0000","last_fullsized":"2024-04-25T00:31:45.029488+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:16:08.805720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710541+0000","last_change":"2024-04-25T00:30:48.538658+0000","last_active":"2024-04-25T00:31:44.710541+0000","last_peered":"2024-04-25T00:31:44.710541+0000","last_clean":"2024-04-25T00:31:44.710541+0000","last_became_active":"2024-04-25T00:30:48.538316+0000","last_became_peered":"2024-04-25T00:30:48.538316+0000","last_unstale":"2024-04-25T00:31:44.710541+0000","last_undegraded":"2024-04-25T00:31:44.710541+0000","last_fullsized":"2024-04-25T00:31:44.710541+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:24:47.303559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.703610+0000","last_change":"2024-04-25T00:30:52.789512+0000","last_active":"2024-04-25T00:31:44.703610+0000","last_peered":"2024-04-25T00:31:44.703610+0000","last_clean":"2024-04-25T00:31:44.703610+0000","last_became_active":"2024-04-25T00:30:52.788929+0000","last_became_peered":"2024-04-25T00:30:52.788929+0000","last_unstale":"2024-04-25T00:31:44.703610+0000","last_undegraded":"2024-04-25T00:31:44.703610+0000","last_fullsized":"2024-04-25T00:31:44.703610+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:21:05.195296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,0,3],"acting":[9,7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.11","version":"89'4","reported_seq":107,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:32:26.088063+0000","last_change":"2024-04-25T00:30:47.513627+0000","last_active":"2024-04-25T00:32:26.088063+0000","last_peered":"2024-04-25T00:32:26.088063+0000","last_clean":"2024-04-25T00:32:26.088063+0000","last_became_active":"2024-04-25T00:30:47.513215+0000","last_became_peered":"2024-04-25T00:30:47.513215+0000","last_unstale":"2024-04-25T00:32:26.088063+0000","last_undegraded":"2024-04-25T00:32:26.088063+0000","last_fullsized":"2024-04-25T00:32:26.088063+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:32:16.111590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":979,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":77,"num_read_kb":4,"num_write":3,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710171+0000","last_change":"2024-04-25T00:30:48.533056+0000","last_active":"2024-04-25T00:31:44.710171+0000","last_peered":"2024-04-25T00:31:44.710171+0000","last_clean":"2024-04-25T00:31:44.710171+0000","last_became_active":"2024-04-25T00:30:48.532698+0000","last_became_peered":"2024-04-25T00:30:48.532698+0000","last_unstale":"2024-04-25T00:31:44.710171+0000","last_undegraded":"2024-04-25T00:31:44.710171+0000","last_fullsized":"2024-04-25T00:31:44.710171+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:36:24.987784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.027530+0000","last_change":"2024-04-25T00:30:52.780669+0000","last_active":"2024-04-25T00:31:45.027530+0000","last_peered":"2024-04-25T00:31:45.027530+0000","last_clean":"2024-04-25T00:31:45.027530+0000","last_became_active":"2024-04-25T00:30:52.780222+0000","last_became_peered":"2024-04-25T00:30:52.780222+0000","last_unstale":"2024-04-25T00:31:45.027530+0000","last_undegraded":"2024-04-25T00:31:45.027530+0000","last_fullsized":"2024-04-25T00:31:45.027530+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:50:27.415077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,9,11],"acting":[6,3,9,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"89'15","reported_seq":43,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.709662+0000","last_change":"2024-04-25T00:30:47.522547+0000","last_active":"2024-04-25T00:31:44.709662+0000","last_peered":"2024-04-25T00:31:44.709662+0000","last_clean":"2024-04-25T00:31:44.709662+0000","last_became_active":"2024-04-25T00:30:47.522193+0000","last_became_peered":"2024-04-25T00:30:47.522193+0000","last_unstale":"2024-04-25T00:31:44.709662+0000","last_undegraded":"2024-04-25T00:31:44.709662+0000","last_fullsized":"2024-04-25T00:31:44.709662+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:10:10.417552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,2],"acting":[8,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.029017+0000","last_change":"2024-04-25T00:30:48.534363+0000","last_active":"2024-04-25T00:31:45.029017+0000","last_peered":"2024-04-25T00:31:45.029017+0000","last_clean":"2024-04-25T00:31:45.029017+0000","last_became_active":"2024-04-25T00:30:48.533855+0000","last_became_peered":"2024-04-25T00:30:48.533855+0000","last_unstale":"2024-04-25T00:31:45.029017+0000","last_undegraded":"2024-04-25T00:31:45.029017+0000","last_fullsized":"2024-04-25T00:31:45.029017+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:28:29.374501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.028927+0000","last_change":"2024-04-25T00:30:52.790323+0000","last_active":"2024-04-25T00:31:45.028927+0000","last_peered":"2024-04-25T00:31:45.028927+0000","last_clean":"2024-04-25T00:31:45.028927+0000","last_became_active":"2024-04-25T00:30:52.789905+0000","last_became_peered":"2024-04-25T00:30:52.789905+0000","last_unstale":"2024-04-25T00:31:45.028927+0000","last_undegraded":"2024-04-25T00:31:45.028927+0000","last_fullsized":"2024-04-25T00:31:45.028927+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:19:03.687318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3,1],"acting":[5,0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"89'2","reported_seq":30,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.711542+0000","last_change":"2024-04-25T00:30:47.516468+0000","last_active":"2024-04-25T00:31:44.711542+0000","last_peered":"2024-04-25T00:31:44.711542+0000","last_clean":"2024-04-25T00:31:44.711542+0000","last_became_active":"2024-04-25T00:30:47.515672+0000","last_became_peered":"2024-04-25T00:30:47.515672+0000","last_unstale":"2024-04-25T00:31:44.711542+0000","last_undegraded":"2024-04-25T00:31:44.711542+0000","last_fullsized":"2024-04-25T00:31:44.711542+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:43:38.525018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":2,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710310+0000","last_change":"2024-04-25T00:30:48.539679+0000","last_active":"2024-04-25T00:31:44.710310+0000","last_peered":"2024-04-25T00:31:44.710310+0000","last_clean":"2024-04-25T00:31:44.710310+0000","last_became_active":"2024-04-25T00:30:48.538065+0000","last_became_peered":"2024-04-25T00:30:48.538065+0000","last_unstale":"2024-04-25T00:31:44.710310+0000","last_undegraded":"2024-04-25T00:31:44.710310+0000","last_fullsized":"2024-04-25T00:31:44.710310+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:05:16.435010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710478+0000","last_change":"2024-04-25T00:30:52.804824+0000","last_active":"2024-04-25T00:31:44.710478+0000","last_peered":"2024-04-25T00:31:44.710478+0000","last_clean":"2024-04-25T00:31:44.710478+0000","last_became_active":"2024-04-25T00:30:52.804322+0000","last_became_peered":"2024-04-25T00:30:52.804322+0000","last_unstale":"2024-04-25T00:31:44.710478+0000","last_undegraded":"2024-04-25T00:31:44.710478+0000","last_fullsized":"2024-04-25T00:31:44.710478+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:38:08.385170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,10,11,9],"acting":[8,10,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":27,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.703679+0000","last_change":"2024-04-25T00:30:47.516916+0000","last_active":"2024-04-25T00:31:44.703679+0000","last_peered":"2024-04-25T00:31:44.703679+0000","last_clean":"2024-04-25T00:31:44.703679+0000","last_became_active":"2024-04-25T00:30:47.516520+0000","last_became_peered":"2024-04-25T00:30:47.516520+0000","last_unstale":"2024-04-25T00:31:44.703679+0000","last_undegraded":"2024-04-25T00:31:44.703679+0000","last_fullsized":"2024-04-25T00:31:44.703679+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:29:19.099764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705654+0000","last_change":"2024-04-25T00:30:48.537584+0000","last_active":"2024-04-25T00:31:44.705654+0000","last_peered":"2024-04-25T00:31:44.705654+0000","last_clean":"2024-04-25T00:31:44.705654+0000","last_became_active":"2024-04-25T00:30:48.537193+0000","last_became_peered":"2024-04-25T00:30:48.537193+0000","last_unstale":"2024-04-25T00:31:44.705654+0000","last_undegraded":"2024-04-25T00:31:44.705654+0000","last_fullsized":"2024-04-25T00:31:44.705654+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:15:22.075323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.027171+0000","last_change":"2024-04-25T00:30:52.784802+0000","last_active":"2024-04-25T00:31:45.027171+0000","last_peered":"2024-04-25T00:31:45.027171+0000","last_clean":"2024-04-25T00:31:45.027171+0000","last_became_active":"2024-04-25T00:30:52.784359+0000","last_became_peered":"2024-04-25T00:30:52.784359+0000","last_unstale":"2024-04-25T00:31:45.027171+0000","last_undegraded":"2024-04-25T00:31:45.027171+0000","last_fullsized":"2024-04-25T00:31:45.027171+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:19:57.723542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1,8],"acting":[6,0,1,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"89'1","reported_seq":28,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710701+0000","last_change":"2024-04-25T00:30:47.525494+0000","last_active":"2024-04-25T00:31:44.710701+0000","last_peered":"2024-04-25T00:31:44.710701+0000","last_clean":"2024-04-25T00:31:44.710701+0000","last_became_active":"2024-04-25T00:30:47.525158+0000","last_became_peered":"2024-04-25T00:30:47.525158+0000","last_unstale":"2024-04-25T00:31:44.710701+0000","last_undegraded":"2024-04-25T00:31:44.710701+0000","last_fullsized":"2024-04-25T00:31:44.710701+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:51:18.543352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705133+0000","last_change":"2024-04-25T00:30:48.540926+0000","last_active":"2024-04-25T00:31:44.705133+0000","last_peered":"2024-04-25T00:31:44.705133+0000","last_clean":"2024-04-25T00:31:44.705133+0000","last_became_active":"2024-04-25T00:30:48.540204+0000","last_became_peered":"2024-04-25T00:30:48.540204+0000","last_unstale":"2024-04-25T00:31:44.705133+0000","last_undegraded":"2024-04-25T00:31:44.705133+0000","last_fullsized":"2024-04-25T00:31:44.705133+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:02:45.530266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.027310+0000","last_change":"2024-04-25T00:30:52.805780+0000","last_active":"2024-04-25T00:31:45.027310+0000","last_peered":"2024-04-25T00:31:45.027310+0000","last_clean":"2024-04-25T00:31:45.027310+0000","last_became_active":"2024-04-25T00:30:52.805222+0000","last_became_peered":"2024-04-25T00:30:52.805222+0000","last_unstale":"2024-04-25T00:31:45.027310+0000","last_undegraded":"2024-04-25T00:31:45.027310+0000","last_fullsized":"2024-04-25T00:31:45.027310+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:32:17.034630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,10,0],"acting":[6,5,10,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.029706+0000","last_change":"2024-04-25T00:30:48.534335+0000","last_active":"2024-04-25T00:31:45.029706+0000","last_peered":"2024-04-25T00:31:45.029706+0000","last_clean":"2024-04-25T00:31:45.029706+0000","last_became_active":"2024-04-25T00:30:48.532670+0000","last_became_peered":"2024-04-25T00:30:48.532670+0000","last_unstale":"2024-04-25T00:31:45.029706+0000","last_undegraded":"2024-04-25T00:31:45.029706+0000","last_fullsized":"2024-04-25T00:31:45.029706+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:26:23.145918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"89'4","reported_seq":105,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:32:27.088840+0000","last_change":"2024-04-25T00:30:47.527695+0000","last_active":"2024-04-25T00:32:27.088840+0000","last_peered":"2024-04-25T00:32:27.088840+0000","last_clean":"2024-04-25T00:32:27.088840+0000","last_became_active":"2024-04-25T00:30:47.527239+0000","last_became_peered":"2024-04-25T00:30:47.527239+0000","last_unstale":"2024-04-25T00:32:27.088840+0000","last_undegraded":"2024-04-25T00:32:27.088840+0000","last_fullsized":"2024-04-25T00:32:27.088840+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:54:30.911284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":74,"num_read_kb":74,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"89'1","reported_seq":19,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710447+0000","last_change":"2024-04-25T00:30:52.781091+0000","last_active":"2024-04-25T00:31:44.710447+0000","last_peered":"2024-04-25T00:31:44.710447+0000","last_clean":"2024-04-25T00:31:44.710447+0000","last_became_active":"2024-04-25T00:30:52.780538+0000","last_became_peered":"2024-04-25T00:30:52.780538+0000","last_unstale":"2024-04-25T00:31:44.710447+0000","last_undegraded":"2024-04-25T00:31:44.710447+0000","last_fullsized":"2024-04-25T00:31:44.710447+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:49:18.141802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,11,9],"acting":[7,5,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.169331+0000","last_change":"2024-04-25T00:30:48.538730+0000","last_active":"2024-04-25T00:31:45.169331+0000","last_peered":"2024-04-25T00:31:45.169331+0000","last_clean":"2024-04-25T00:31:45.169331+0000","last_became_active":"2024-04-25T00:30:48.538336+0000","last_became_peered":"2024-04-25T00:30:48.538336+0000","last_unstale":"2024-04-25T00:31:45.169331+0000","last_undegraded":"2024-04-25T00:31:45.169331+0000","last_fullsized":"2024-04-25T00:31:45.169331+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:44:22.370448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"89'3","reported_seq":32,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707062+0000","last_change":"2024-04-25T00:30:47.520450+0000","last_active":"2024-04-25T00:31:44.707062+0000","last_peered":"2024-04-25T00:31:44.707062+0000","last_clean":"2024-04-25T00:31:44.707062+0000","last_became_active":"2024-04-25T00:30:47.520073+0000","last_became_peered":"2024-04-25T00:30:47.520073+0000","last_unstale":"2024-04-25T00:31:44.707062+0000","last_undegraded":"2024-04-25T00:31:44.707062+0000","last_fullsized":"2024-04-25T00:31:44.707062+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:49:53.570421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":4,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"22'137","reported_seq":361,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.709973+0000","last_change":"2024-04-25T00:29:34.720552+0000","last_active":"2024-04-25T00:31:44.709973+0000","last_peered":"2024-04-25T00:31:44.709973+0000","last_clean":"2024-04-25T00:31:44.709973+0000","last_became_active":"2024-04-25T00:29:34.720041+0000","last_became_peered":"2024-04-25T00:29:34.720041+0000","last_unstale":"2024-04-25T00:31:44.709973+0000","last_undegraded":"2024-04-25T00:31:44.709973+0000","last_fullsized":"2024-04-25T00:31:44.709973+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:26:07.691582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:26:07.691582+0000","last_clean_scrub_stamp":"2024-04-25T00:26:07.691582+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:15:27.995162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.029957+0000","last_change":"2024-04-25T00:30:52.805489+0000","last_active":"2024-04-25T00:31:45.029957+0000","last_peered":"2024-04-25T00:31:45.029957+0000","last_clean":"2024-04-25T00:31:45.029957+0000","last_became_active":"2024-04-25T00:30:52.804758+0000","last_became_peered":"2024-04-25T00:30:52.804758+0000","last_unstale":"2024-04-25T00:31:45.029957+0000","last_undegraded":"2024-04-25T00:31:45.029957+0000","last_fullsized":"2024-04-25T00:31:45.029957+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:17:31.325548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,6,10],"acting":[5,9,6,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.026977+0000","last_change":"2024-04-25T00:30:48.537699+0000","last_active":"2024-04-25T00:31:45.026977+0000","last_peered":"2024-04-25T00:31:45.026977+0000","last_clean":"2024-04-25T00:31:45.026977+0000","last_became_active":"2024-04-25T00:30:48.537291+0000","last_became_peered":"2024-04-25T00:30:48.537291+0000","last_unstale":"2024-04-25T00:31:45.026977+0000","last_undegraded":"2024-04-25T00:31:45.026977+0000","last_fullsized":"2024-04-25T00:31:45.026977+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:25:36.816499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"89'3","reported_seq":30,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710041+0000","last_change":"2024-04-25T00:30:47.524694+0000","last_active":"2024-04-25T00:31:44.710041+0000","last_peered":"2024-04-25T00:31:44.710041+0000","last_clean":"2024-04-25T00:31:44.710041+0000","last_became_active":"2024-04-25T00:30:47.524359+0000","last_became_peered":"2024-04-25T00:30:47.524359+0000","last_unstale":"2024-04-25T00:31:44.710041+0000","last_undegraded":"2024-04-25T00:31:44.710041+0000","last_fullsized":"2024-04-25T00:31:44.710041+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:55:16.677234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.708832+0000","last_change":"2024-04-25T00:30:52.780364+0000","last_active":"2024-04-25T00:31:44.708832+0000","last_peered":"2024-04-25T00:31:44.708832+0000","last_clean":"2024-04-25T00:31:44.708832+0000","last_became_active":"2024-04-25T00:30:52.779904+0000","last_became_peered":"2024-04-25T00:30:52.779904+0000","last_unstale":"2024-04-25T00:31:44.708832+0000","last_undegraded":"2024-04-25T00:31:44.708832+0000","last_fullsized":"2024-04-25T00:31:44.708832+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:11:26.738526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,7,6],"acting":[11,5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.5","version":"89'1","reported_seq":29,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.712031+0000","last_change":"2024-04-25T00:30:47.525205+0000","last_active":"2024-04-25T00:31:44.712031+0000","last_peered":"2024-04-25T00:31:44.712031+0000","last_clean":"2024-04-25T00:31:44.712031+0000","last_became_active":"2024-04-25T00:30:47.517729+0000","last_became_peered":"2024-04-25T00:30:47.517729+0000","last_unstale":"2024-04-25T00:31:44.712031+0000","last_undegraded":"2024-04-25T00:31:44.712031+0000","last_fullsized":"2024-04-25T00:31:44.712031+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:21:14.810119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.027310+0000","last_change":"2024-04-25T00:30:48.533822+0000","last_active":"2024-04-25T00:31:45.027310+0000","last_peered":"2024-04-25T00:31:45.027310+0000","last_clean":"2024-04-25T00:31:45.027310+0000","last_became_active":"2024-04-25T00:30:48.533479+0000","last_became_peered":"2024-04-25T00:30:48.533479+0000","last_unstale":"2024-04-25T00:31:45.027310+0000","last_undegraded":"2024-04-25T00:31:45.027310+0000","last_fullsized":"2024-04-25T00:31:45.027310+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:49:01.028124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.169164+0000","last_change":"2024-04-25T00:30:52.804770+0000","last_active":"2024-04-25T00:31:45.169164+0000","last_peered":"2024-04-25T00:31:45.169164+0000","last_clean":"2024-04-25T00:31:45.169164+0000","last_became_active":"2024-04-25T00:30:52.804242+0000","last_became_peered":"2024-04-25T00:30:52.804242+0000","last_unstale":"2024-04-25T00:31:45.169164+0000","last_undegraded":"2024-04-25T00:31:45.169164+0000","last_fullsized":"2024-04-25T00:31:45.169164+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:34:11.761478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,10,0,7],"acting":[3,10,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"89'8","reported_seq":35,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.703669+0000","last_change":"2024-04-25T00:30:47.521364+0000","last_active":"2024-04-25T00:31:44.703669+0000","last_peered":"2024-04-25T00:31:44.703669+0000","last_clean":"2024-04-25T00:31:44.703669+0000","last_became_active":"2024-04-25T00:30:47.521032+0000","last_became_peered":"2024-04-25T00:30:47.521032+0000","last_unstale":"2024-04-25T00:31:44.703669+0000","last_undegraded":"2024-04-25T00:31:44.703669+0000","last_fullsized":"2024-04-25T00:31:44.703669+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:33:26.156568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,8,3],"acting":[9,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.169845+0000","last_change":"2024-04-25T00:30:48.532942+0000","last_active":"2024-04-25T00:31:45.169845+0000","last_peered":"2024-04-25T00:31:45.169845+0000","last_clean":"2024-04-25T00:31:45.169845+0000","last_became_active":"2024-04-25T00:30:48.532580+0000","last_became_peered":"2024-04-25T00:30:48.532580+0000","last_unstale":"2024-04-25T00:31:45.169845+0000","last_undegraded":"2024-04-25T00:31:45.169845+0000","last_fullsized":"2024-04-25T00:31:45.169845+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:42:22.887920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705899+0000","last_change":"2024-04-25T00:30:52.779757+0000","last_active":"2024-04-25T00:31:44.705899+0000","last_peered":"2024-04-25T00:31:44.705899+0000","last_clean":"2024-04-25T00:31:44.705899+0000","last_became_active":"2024-04-25T00:30:52.779324+0000","last_became_peered":"2024-04-25T00:30:52.779324+0000","last_unstale":"2024-04-25T00:31:44.705899+0000","last_undegraded":"2024-04-25T00:31:44.705899+0000","last_fullsized":"2024-04-25T00:31:44.705899+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:22:39.062000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4,7],"acting":[2,1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"89'4","reported_seq":31,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.706470+0000","last_change":"2024-04-25T00:30:47.518212+0000","last_active":"2024-04-25T00:31:44.706470+0000","last_peered":"2024-04-25T00:31:44.706470+0000","last_clean":"2024-04-25T00:31:44.706470+0000","last_became_active":"2024-04-25T00:30:47.517610+0000","last_became_peered":"2024-04-25T00:30:47.517610+0000","last_unstale":"2024-04-25T00:31:44.706470+0000","last_undegraded":"2024-04-25T00:31:44.706470+0000","last_fullsized":"2024-04-25T00:31:44.706470+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:13:58.904020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":31,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.028411+0000","last_change":"2024-04-25T00:31:45.028411+0000","last_active":"2024-04-25T00:31:45.028411+0000","last_peered":"2024-04-25T00:31:45.028411+0000","last_clean":"2024-04-25T00:31:45.028411+0000","last_became_active":"2024-04-25T00:31:45.028066+0000","last_became_peered":"2024-04-25T00:31:45.028066+0000","last_unstale":"2024-04-25T00:31:45.028411+0000","last_undegraded":"2024-04-25T00:31:45.028411+0000","last_fullsized":"2024-04-25T00:31:45.028411+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:33:46.556862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705862+0000","last_change":"2024-04-25T00:30:52.804915+0000","last_active":"2024-04-25T00:31:44.705862+0000","last_peered":"2024-04-25T00:31:44.705862+0000","last_clean":"2024-04-25T00:31:44.705862+0000","last_became_active":"2024-04-25T00:30:52.804491+0000","last_became_peered":"2024-04-25T00:30:52.804491+0000","last_unstale":"2024-04-25T00:31:44.705862+0000","last_undegraded":"2024-04-25T00:31:44.705862+0000","last_fullsized":"2024-04-25T00:31:44.705862+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:51:50.219556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,10,9],"acting":[4,5,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.8","version":"89'2","reported_seq":29,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.704458+0000","last_change":"2024-04-25T00:30:47.521909+0000","last_active":"2024-04-25T00:31:44.704458+0000","last_peered":"2024-04-25T00:31:44.704458+0000","last_clean":"2024-04-25T00:31:44.704458+0000","last_became_active":"2024-04-25T00:30:47.521551+0000","last_became_peered":"2024-04-25T00:30:47.521551+0000","last_unstale":"2024-04-25T00:31:44.704458+0000","last_undegraded":"2024-04-25T00:31:44.704458+0000","last_fullsized":"2024-04-25T00:31:44.704458+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:51:16.368609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.709641+0000","last_change":"2024-04-25T00:30:48.530514+0000","last_active":"2024-04-25T00:31:44.709641+0000","last_peered":"2024-04-25T00:31:44.709641+0000","last_clean":"2024-04-25T00:31:44.709641+0000","last_became_active":"2024-04-25T00:30:48.530129+0000","last_became_peered":"2024-04-25T00:30:48.530129+0000","last_unstale":"2024-04-25T00:31:44.709641+0000","last_undegraded":"2024-04-25T00:31:44.709641+0000","last_fullsized":"2024-04-25T00:31:44.709641+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:51:25.623186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.703839+0000","last_change":"2024-04-25T00:30:52.785767+0000","last_active":"2024-04-25T00:31:44.703839+0000","last_peered":"2024-04-25T00:31:44.703839+0000","last_clean":"2024-04-25T00:31:44.703839+0000","last_became_active":"2024-04-25T00:30:52.785301+0000","last_became_peered":"2024-04-25T00:30:52.785301+0000","last_unstale":"2024-04-25T00:31:44.703839+0000","last_undegraded":"2024-04-25T00:31:44.703839+0000","last_fullsized":"2024-04-25T00:31:44.703839+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:10:14.071733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,2,5],"acting":[9,3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.9","version":"89'3","reported_seq":30,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.169284+0000","last_change":"2024-04-25T00:30:47.526364+0000","last_active":"2024-04-25T00:31:45.169284+0000","last_peered":"2024-04-25T00:31:45.169284+0000","last_clean":"2024-04-25T00:31:45.169284+0000","last_became_active":"2024-04-25T00:30:47.526009+0000","last_became_peered":"2024-04-25T00:30:47.526009+0000","last_unstale":"2024-04-25T00:31:45.169284+0000","last_undegraded":"2024-04-25T00:31:45.169284+0000","last_fullsized":"2024-04-25T00:31:45.169284+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:17:38.471878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.709530+0000","last_change":"2024-04-25T00:30:48.532283+0000","last_active":"2024-04-25T00:31:44.709530+0000","last_peered":"2024-04-25T00:31:44.709530+0000","last_clean":"2024-04-25T00:31:44.709530+0000","last_became_active":"2024-04-25T00:30:48.531935+0000","last_became_peered":"2024-04-25T00:30:48.531935+0000","last_unstale":"2024-04-25T00:31:44.709530+0000","last_undegraded":"2024-04-25T00:31:44.709530+0000","last_fullsized":"2024-04-25T00:31:44.709530+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:01:15.965611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.c","version":"89'1","reported_seq":30,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.717218+0000","last_change":"2024-04-25T00:31:44.717218+0000","last_active":"2024-04-25T00:31:44.717218+0000","last_peered":"2024-04-25T00:31:44.717218+0000","last_clean":"2024-04-25T00:31:44.717218+0000","last_became_active":"2024-04-25T00:31:44.712770+0000","last_became_peered":"2024-04-25T00:31:44.712770+0000","last_unstale":"2024-04-25T00:31:44.717218+0000","last_undegraded":"2024-04-25T00:31:44.717218+0000","last_fullsized":"2024-04-25T00:31:44.717218+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:06:58.496133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":117,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,7,8],"acting":[4,10,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.a","version":"89'1","reported_seq":28,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.706038+0000","last_change":"2024-04-25T00:30:47.520608+0000","last_active":"2024-04-25T00:31:44.706038+0000","last_peered":"2024-04-25T00:31:44.706038+0000","last_clean":"2024-04-25T00:31:44.706038+0000","last_became_active":"2024-04-25T00:30:47.520063+0000","last_became_peered":"2024-04-25T00:30:47.520063+0000","last_unstale":"2024-04-25T00:31:44.706038+0000","last_undegraded":"2024-04-25T00:31:44.706038+0000","last_fullsized":"2024-04-25T00:31:44.706038+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:58:47.823712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.169717+0000","last_change":"2024-04-25T00:30:48.535073+0000","last_active":"2024-04-25T00:31:45.169717+0000","last_peered":"2024-04-25T00:31:45.169717+0000","last_clean":"2024-04-25T00:31:45.169717+0000","last_became_active":"2024-04-25T00:30:48.534247+0000","last_became_peered":"2024-04-25T00:30:48.534247+0000","last_unstale":"2024-04-25T00:31:45.169717+0000","last_undegraded":"2024-04-25T00:31:45.169717+0000","last_fullsized":"2024-04-25T00:31:45.169717+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:46:18.613705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705968+0000","last_change":"2024-04-25T00:30:52.804519+0000","last_active":"2024-04-25T00:31:44.705968+0000","last_peered":"2024-04-25T00:31:44.705968+0000","last_clean":"2024-04-25T00:31:44.705968+0000","last_became_active":"2024-04-25T00:30:52.804074+0000","last_became_peered":"2024-04-25T00:30:52.804074+0000","last_unstale":"2024-04-25T00:31:44.705968+0000","last_undegraded":"2024-04-25T00:31:44.705968+0000","last_fullsized":"2024-04-25T00:31:44.705968+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:48:45.058873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,10,3],"acting":[4,2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.b","version":"89'4","reported_seq":32,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710833+0000","last_change":"2024-04-25T00:30:47.521054+0000","last_active":"2024-04-25T00:31:44.710833+0000","last_peered":"2024-04-25T00:31:44.710833+0000","last_clean":"2024-04-25T00:31:44.710833+0000","last_became_active":"2024-04-25T00:30:47.520716+0000","last_became_peered":"2024-04-25T00:30:47.520716+0000","last_unstale":"2024-04-25T00:31:44.710833+0000","last_undegraded":"2024-04-25T00:31:44.710833+0000","last_fullsized":"2024-04-25T00:31:44.710833+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:05:16.471589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710832+0000","last_change":"2024-04-25T00:30:48.537152+0000","last_active":"2024-04-25T00:31:44.710832+0000","last_peered":"2024-04-25T00:31:44.710832+0000","last_clean":"2024-04-25T00:31:44.710832+0000","last_became_active":"2024-04-25T00:30:48.536256+0000","last_became_peered":"2024-04-25T00:30:48.536256+0000","last_unstale":"2024-04-25T00:31:44.710832+0000","last_undegraded":"2024-04-25T00:31:44.710832+0000","last_fullsized":"2024-04-25T00:31:44.710832+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:57:53.870212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.704501+0000","last_change":"2024-04-25T00:30:52.781022+0000","last_active":"2024-04-25T00:31:44.704501+0000","last_peered":"2024-04-25T00:31:44.704501+0000","last_clean":"2024-04-25T00:31:44.704501+0000","last_became_active":"2024-04-25T00:30:52.780595+0000","last_became_peered":"2024-04-25T00:30:52.780595+0000","last_unstale":"2024-04-25T00:31:44.704501+0000","last_undegraded":"2024-04-25T00:31:44.704501+0000","last_fullsized":"2024-04-25T00:31:44.704501+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:27:02.580870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6,7],"acting":[1,5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.c","version":"89'3","reported_seq":31,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.706842+0000","last_change":"2024-04-25T00:30:47.519118+0000","last_active":"2024-04-25T00:31:44.706842+0000","last_peered":"2024-04-25T00:31:44.706842+0000","last_clean":"2024-04-25T00:31:44.706842+0000","last_became_active":"2024-04-25T00:30:47.518639+0000","last_became_peered":"2024-04-25T00:30:47.518639+0000","last_unstale":"2024-04-25T00:31:44.706842+0000","last_undegraded":"2024-04-25T00:31:44.706842+0000","last_fullsized":"2024-04-25T00:31:44.706842+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:32:29.747903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710945+0000","last_change":"2024-04-25T00:30:48.542073+0000","last_active":"2024-04-25T00:31:44.710945+0000","last_peered":"2024-04-25T00:31:44.710945+0000","last_clean":"2024-04-25T00:31:44.710945+0000","last_became_active":"2024-04-25T00:30:48.541722+0000","last_became_peered":"2024-04-25T00:30:48.541722+0000","last_unstale":"2024-04-25T00:31:44.710945+0000","last_undegraded":"2024-04-25T00:31:44.710945+0000","last_fullsized":"2024-04-25T00:31:44.710945+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:34:03.397539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707011+0000","last_change":"2024-04-25T00:30:52.790234+0000","last_active":"2024-04-25T00:31:44.707011+0000","last_peered":"2024-04-25T00:31:44.707011+0000","last_clean":"2024-04-25T00:31:44.707011+0000","last_became_active":"2024-04-25T00:30:52.789788+0000","last_became_peered":"2024-04-25T00:30:52.789788+0000","last_unstale":"2024-04-25T00:31:44.707011+0000","last_undegraded":"2024-04-25T00:31:44.707011+0000","last_fullsized":"2024-04-25T00:31:44.707011+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:44:43.399771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,9,1,4],"acting":[0,9,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.d","version":"84'1","reported_seq":29,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.029778+0000","last_change":"2024-04-25T00:30:47.523597+0000","last_active":"2024-04-25T00:31:45.029778+0000","last_peered":"2024-04-25T00:31:45.029778+0000","last_clean":"2024-04-25T00:31:45.029778+0000","last_became_active":"2024-04-25T00:30:47.523225+0000","last_became_peered":"2024-04-25T00:30:47.523225+0000","last_unstale":"2024-04-25T00:31:45.029778+0000","last_undegraded":"2024-04-25T00:31:45.029778+0000","last_fullsized":"2024-04-25T00:31:45.029778+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:51:12.341015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710527+0000","last_change":"2024-04-25T00:30:48.541417+0000","last_active":"2024-04-25T00:31:44.710527+0000","last_peered":"2024-04-25T00:31:44.710527+0000","last_clean":"2024-04-25T00:31:44.710527+0000","last_became_active":"2024-04-25T00:30:48.541015+0000","last_became_peered":"2024-04-25T00:30:48.541015+0000","last_unstale":"2024-04-25T00:31:44.710527+0000","last_undegraded":"2024-04-25T00:31:44.710527+0000","last_fullsized":"2024-04-25T00:31:44.710527+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:21:35.299823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.029015+0000","last_change":"2024-04-25T00:30:52.782230+0000","last_active":"2024-04-25T00:31:45.029015+0000","last_peered":"2024-04-25T00:31:45.029015+0000","last_clean":"2024-04-25T00:31:45.029015+0000","last_became_active":"2024-04-25T00:30:52.781708+0000","last_became_peered":"2024-04-25T00:30:52.781708+0000","last_unstale":"2024-04-25T00:31:45.029015+0000","last_undegraded":"2024-04-25T00:31:45.029015+0000","last_fullsized":"2024-04-25T00:31:45.029015+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:58:53.260985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7,1],"acting":[5,2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.e","version":"84'2","reported_seq":29,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.027519+0000","last_change":"2024-04-25T00:30:47.522400+0000","last_active":"2024-04-25T00:31:45.027519+0000","last_peered":"2024-04-25T00:31:45.027519+0000","last_clean":"2024-04-25T00:31:45.027519+0000","last_became_active":"2024-04-25T00:30:47.522032+0000","last_became_peered":"2024-04-25T00:30:47.522032+0000","last_unstale":"2024-04-25T00:31:45.027519+0000","last_undegraded":"2024-04-25T00:31:45.027519+0000","last_fullsized":"2024-04-25T00:31:45.027519+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:24:17.749437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.711092+0000","last_change":"2024-04-25T00:30:48.532730+0000","last_active":"2024-04-25T00:31:44.711092+0000","last_peered":"2024-04-25T00:31:44.711092+0000","last_clean":"2024-04-25T00:31:44.711092+0000","last_became_active":"2024-04-25T00:30:48.532129+0000","last_became_peered":"2024-04-25T00:30:48.532129+0000","last_unstale":"2024-04-25T00:31:44.711092+0000","last_undegraded":"2024-04-25T00:31:44.711092+0000","last_fullsized":"2024-04-25T00:31:44.711092+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:36:30.115125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":31,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.712173+0000","last_change":"2024-04-25T00:31:44.712173+0000","last_active":"2024-04-25T00:31:44.712173+0000","last_peered":"2024-04-25T00:31:44.712173+0000","last_clean":"2024-04-25T00:31:44.712173+0000","last_became_active":"2024-04-25T00:31:44.711738+0000","last_became_peered":"2024-04-25T00:31:44.711738+0000","last_unstale":"2024-04-25T00:31:44.712173+0000","last_undegraded":"2024-04-25T00:31:44.712173+0000","last_fullsized":"2024-04-25T00:31:44.712173+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:06:59.466566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,9],"acting":[11,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.706223+0000","last_change":"2024-04-25T00:30:48.540676+0000","last_active":"2024-04-25T00:31:44.706223+0000","last_peered":"2024-04-25T00:31:44.706223+0000","last_clean":"2024-04-25T00:31:44.706223+0000","last_became_active":"2024-04-25T00:30:48.540258+0000","last_became_peered":"2024-04-25T00:30:48.540258+0000","last_unstale":"2024-04-25T00:31:44.706223+0000","last_undegraded":"2024-04-25T00:31:44.706223+0000","last_fullsized":"2024-04-25T00:31:44.706223+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:10:29.930897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705989+0000","last_change":"2024-04-25T00:30:48.542191+0000","last_active":"2024-04-25T00:31:44.705989+0000","last_peered":"2024-04-25T00:31:44.705989+0000","last_clean":"2024-04-25T00:31:44.705989+0000","last_became_active":"2024-04-25T00:30:48.540433+0000","last_became_peered":"2024-04-25T00:30:48.540433+0000","last_unstale":"2024-04-25T00:31:44.705989+0000","last_undegraded":"2024-04-25T00:31:44.705989+0000","last_fullsized":"2024-04-25T00:31:44.705989+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:21:55.694555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":31,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.028445+0000","last_change":"2024-04-25T00:31:45.028445+0000","last_active":"2024-04-25T00:31:45.028445+0000","last_peered":"2024-04-25T00:31:45.028445+0000","last_clean":"2024-04-25T00:31:45.028445+0000","last_became_active":"2024-04-25T00:31:45.028048+0000","last_became_peered":"2024-04-25T00:31:45.028048+0000","last_unstale":"2024-04-25T00:31:45.028445+0000","last_undegraded":"2024-04-25T00:31:45.028445+0000","last_fullsized":"2024-04-25T00:31:45.028445+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:04:02.714111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,8],"acting":[0,6,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.027430+0000","last_change":"2024-04-25T00:30:48.527892+0000","last_active":"2024-04-25T00:31:45.027430+0000","last_peered":"2024-04-25T00:31:45.027430+0000","last_clean":"2024-04-25T00:31:45.027430+0000","last_became_active":"2024-04-25T00:30:48.527522+0000","last_became_peered":"2024-04-25T00:30:48.527522+0000","last_unstale":"2024-04-25T00:31:45.027430+0000","last_undegraded":"2024-04-25T00:31:45.027430+0000","last_fullsized":"2024-04-25T00:31:45.027430+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:21:22.779470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.711690+0000","last_change":"2024-04-25T00:30:48.535066+0000","last_active":"2024-04-25T00:31:44.711690+0000","last_peered":"2024-04-25T00:31:44.711690+0000","last_clean":"2024-04-25T00:31:44.711690+0000","last_became_active":"2024-04-25T00:30:48.534710+0000","last_became_peered":"2024-04-25T00:30:48.534710+0000","last_unstale":"2024-04-25T00:31:44.711690+0000","last_undegraded":"2024-04-25T00:31:44.711690+0000","last_fullsized":"2024-04-25T00:31:44.711690+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:57:18.582569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707210+0000","last_change":"2024-04-25T00:30:48.541921+0000","last_active":"2024-04-25T00:31:44.707210+0000","last_peered":"2024-04-25T00:31:44.707210+0000","last_clean":"2024-04-25T00:31:44.707210+0000","last_became_active":"2024-04-25T00:30:48.541348+0000","last_became_peered":"2024-04-25T00:30:48.541348+0000","last_unstale":"2024-04-25T00:31:44.707210+0000","last_undegraded":"2024-04-25T00:31:44.707210+0000","last_fullsized":"2024-04-25T00:31:44.707210+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:07:00.001504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.169960+0000","last_change":"2024-04-25T00:30:48.536902+0000","last_active":"2024-04-25T00:31:45.169960+0000","last_peered":"2024-04-25T00:31:45.169960+0000","last_clean":"2024-04-25T00:31:45.169960+0000","last_became_active":"2024-04-25T00:30:48.536008+0000","last_became_peered":"2024-04-25T00:30:48.536008+0000","last_unstale":"2024-04-25T00:31:45.169960+0000","last_undegraded":"2024-04-25T00:31:45.169960+0000","last_fullsized":"2024-04-25T00:31:45.169960+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:19:15.050142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.029352+0000","last_change":"2024-04-25T00:30:48.533104+0000","last_active":"2024-04-25T00:31:45.029352+0000","last_peered":"2024-04-25T00:31:45.029352+0000","last_clean":"2024-04-25T00:31:45.029352+0000","last_became_active":"2024-04-25T00:30:48.532124+0000","last_became_peered":"2024-04-25T00:30:48.532124+0000","last_unstale":"2024-04-25T00:31:45.029352+0000","last_undegraded":"2024-04-25T00:31:45.029352+0000","last_fullsized":"2024-04-25T00:31:45.029352+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:54:57.391433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.169213+0000","last_change":"2024-04-25T00:30:48.542074+0000","last_active":"2024-04-25T00:31:45.169213+0000","last_peered":"2024-04-25T00:31:45.169213+0000","last_clean":"2024-04-25T00:31:45.169213+0000","last_became_active":"2024-04-25T00:30:48.538752+0000","last_became_peered":"2024-04-25T00:30:48.538752+0000","last_unstale":"2024-04-25T00:31:45.169213+0000","last_undegraded":"2024-04-25T00:31:45.169213+0000","last_fullsized":"2024-04-25T00:31:45.169213+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:55:30.613616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.704798+0000","last_change":"2024-04-25T00:30:48.534158+0000","last_active":"2024-04-25T00:31:44.704798+0000","last_peered":"2024-04-25T00:31:44.704798+0000","last_clean":"2024-04-25T00:31:44.704798+0000","last_became_active":"2024-04-25T00:30:48.533821+0000","last_became_peered":"2024-04-25T00:30:48.533821+0000","last_unstale":"2024-04-25T00:31:44.704798+0000","last_undegraded":"2024-04-25T00:31:44.704798+0000","last_fullsized":"2024-04-25T00:31:44.704798+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:05:19.327608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.027512+0000","last_change":"2024-04-25T00:30:48.531718+0000","last_active":"2024-04-25T00:31:45.027512+0000","last_peered":"2024-04-25T00:31:45.027512+0000","last_clean":"2024-04-25T00:31:45.027512+0000","last_became_active":"2024-04-25T00:30:48.531291+0000","last_became_peered":"2024-04-25T00:30:48.531291+0000","last_unstale":"2024-04-25T00:31:45.027512+0000","last_undegraded":"2024-04-25T00:31:45.027512+0000","last_fullsized":"2024-04-25T00:31:45.027512+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:56:49.032937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":31,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.713036+0000","last_change":"2024-04-25T00:31:44.713036+0000","last_active":"2024-04-25T00:31:44.713036+0000","last_peered":"2024-04-25T00:31:44.713036+0000","last_clean":"2024-04-25T00:31:44.713036+0000","last_became_active":"2024-04-25T00:31:44.712651+0000","last_became_peered":"2024-04-25T00:31:44.712651+0000","last_unstale":"2024-04-25T00:31:44.713036+0000","last_undegraded":"2024-04-25T00:31:44.713036+0000","last_fullsized":"2024-04-25T00:31:44.713036+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:05:04.811510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,9,8],"acting":[11,9,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":234,"num_objects":2,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":117,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":32768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2,"ondisk_log_size":2,"up":128,"acting":128,"num_store_stats":12},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":65776,"num_objects":60,"num_object_clones":0,"num_object_copies":180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":60,"num_whiteouts":0,"num_read":462,"num_read_kb":301,"num_write":159,"num_write_kb":113,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":417792,"data_stored":197328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138,"ondisk_log_size":138,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":80,"seq":343597383714,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32176,"kb_used_data":1000,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712976,"statfs":{"total":95995035648,"available":95962087424,"internally_reserved":0,"allocated":1024000,"data_stored":500815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1596,"internal_metadata":31914436},"hb_peers":[0,1,2,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":10,"up_from":72,"seq":309237645351,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28696,"kb_used_data":1676,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716456,"statfs":{"total":95995035648,"available":95965650944,"internally_reserved":0,"allocated":1716224,"data_stored":1154345,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1600,"internal_metadata":27654592},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":9,"up_from":67,"seq":287762808876,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36356,"kb_used_data":1016,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708796,"statfs":{"total":95995035648,"available":95957807104,"internally_reserved":0,"allocated":1040384,"data_stored":501051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1600,"internal_metadata":36174272},"hb_peers":[0,1,2,3,4,5,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":8,"up_from":59,"seq":253403070512,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32192,"kb_used_data":1016,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712960,"statfs":{"total":95995035648,"available":95962071040,"internally_reserved":0,"allocated":1040384,"data_stored":500971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":7,"up_from":51,"seq":219043332150,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32780,"kb_used_data":1596,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712372,"statfs":{"total":95995035648,"available":95961468928,"internally_reserved":0,"allocated":1634304,"data_stored":1096880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495676,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36336,"kb_used_data":996,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708816,"statfs":{"total":95995035648,"available":95957827584,"internally_reserved":0,"allocated":1019904,"data_stored":496227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":36174278},"hb_peers":[0,1,2,3,4,5,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574931,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28080,"kb_used_data":1060,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717072,"statfs":{"total":95995035648,"available":95966281728,"internally_reserved":0,"allocated":1085440,"data_stored":557912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738457,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28044,"kb_used_data":1020,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717108,"statfs":{"total":95995035648,"available":95966318592,"internally_reserved":0,"allocated":1044480,"data_stored":497615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":18,"seq":77309411407,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28108,"kb_used_data":1084,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717044,"statfs":{"total":95995035648,"available":95966253056,"internally_reserved":0,"allocated":1110016,"data_stored":559262,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":25,"seq":107374182474,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36348,"kb_used_data":1008,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708804,"statfs":{"total":95995035648,"available":95957815296,"internally_reserved":0,"allocated":1032192,"data_stored":496421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":36174279},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":33,"seq":141733920838,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28032,"kb_used_data":1008,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717120,"statfs":{"total":95995035648,"available":95966330880,"internally_reserved":0,"allocated":1032192,"data_stored":501268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757313,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28628,"kb_used_data":1604,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716524,"statfs":{"total":95995035648,"available":95965720576,"internally_reserved":0,"allocated":1642496,"data_stored":1092077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":69632,"data_stored":61775,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":94208,"data_stored":63125,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2183,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":63744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":8192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-25T00:32:31.827 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-25T00:32:32.083 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:32:32.464 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:32 smithi032 ceph-mon[29357]: from='client.15141 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:32:32.464 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:32 smithi032 ceph-mon[29357]: pgmap v345: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:32:32.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:32 smithi121 ceph-mon[32231]: from='client.15141 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:32:32.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:32 smithi121 ceph-mon[32231]: pgmap v345: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:32:32.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:32 smithi130 ceph-mon[32525]: from='client.15141 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:32:32.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:32 smithi130 ceph-mon[32525]: pgmap v345: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:32:33.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.930+0000 7f0add6cb700 1 -- 172.21.15.32:0/4063489454 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0ad8101420 msgr2=0x7f0ad8101800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:33.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.930+0000 7f0add6cb700 1 --2- 172.21.15.32:0/4063489454 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0ad8101420 0x7f0ad8101800 secure :-1 s=READY pgs=342 cs=0 l=1 rev1=1 crypto rx=0x7f0ac80099c0 tx=0x7f0ac8009cd0 comp rx=0 tx=0).stop 2024-04-25T00:32:33.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.930+0000 7f0add6cb700 1 -- 172.21.15.32:0/4063489454 shutdown_connections 2024-04-25T00:32:33.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.930+0000 7f0add6cb700 1 --2- 172.21.15.32:0/4063489454 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0ad8106470 0x7f0ad810e040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:33.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.930+0000 7f0add6cb700 1 --2- 172.21.15.32:0/4063489454 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0ad8101420 0x7f0ad8101800 unknown :-1 s=CLOSED pgs=342 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:33.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.930+0000 7f0add6cb700 1 --2- 172.21.15.32:0/4063489454 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0ad8101dd0 0x7f0ad8105d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:33.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.930+0000 7f0add6cb700 1 -- 172.21.15.32:0/4063489454 >> 172.21.15.32:0/4063489454 conn(0x7f0ad8077180 msgr2=0x7f0ad8077590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:33.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.930+0000 7f0add6cb700 1 -- 172.21.15.32:0/4063489454 shutdown_connections 2024-04-25T00:32:33.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.930+0000 7f0add6cb700 1 -- 172.21.15.32:0/4063489454 wait complete. 2024-04-25T00:32:33.935 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.931+0000 7f0add6cb700 1 Processor -- start 2024-04-25T00:32:33.935 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.931+0000 7f0add6cb700 1 -- start start 2024-04-25T00:32:33.935 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.931+0000 7f0add6cb700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0ad8101420 0x7f0ad819e750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:33.935 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.931+0000 7f0add6cb700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0ad8101dd0 0x7f0ad819ec90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:33.935 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.931+0000 7f0add6cb700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0ad8106470 0x7f0ad8198910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:33.935 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.931+0000 7f0add6cb700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f0ad81114d0 con 0x7f0ad8101dd0 2024-04-25T00:32:33.935 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.931+0000 7f0add6cb700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f0ad8111350 con 0x7f0ad8101420 2024-04-25T00:32:33.935 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.931+0000 7f0add6cb700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f0ad8111650 con 0x7f0ad8106470 2024-04-25T00:32:33.935 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.931+0000 7f0ad67fc700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0ad8101dd0 0x7f0ad819ec90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:33.935 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.932+0000 7f0ad67fc700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0ad8101dd0 0x7f0ad819ec90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40738/0 (socket says 172.21.15.32:40738) 2024-04-25T00:32:33.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.932+0000 7f0ad67fc700 1 -- 172.21.15.32:0/221981867 learned_addr learned my addr 172.21.15.32:0/221981867 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:33.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.932+0000 7f0ad6ffd700 1 --2- 172.21.15.32:0/221981867 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0ad8101420 0x7f0ad819e750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:33.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.932+0000 7f0ad77fe700 1 --2- 172.21.15.32:0/221981867 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0ad8106470 0x7f0ad8198910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:33.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.932+0000 7f0ad67fc700 1 -- 172.21.15.32:0/221981867 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0ad8106470 msgr2=0x7f0ad8198910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:33.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.932+0000 7f0ad67fc700 1 --2- 172.21.15.32:0/221981867 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0ad8106470 0x7f0ad8198910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:33.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.932+0000 7f0ad67fc700 1 -- 172.21.15.32:0/221981867 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0ad8101420 msgr2=0x7f0ad819e750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:33.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.932+0000 7f0ad67fc700 1 --2- 172.21.15.32:0/221981867 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0ad8101420 0x7f0ad819e750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:33.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.932+0000 7f0ad67fc700 1 -- 172.21.15.32:0/221981867 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0ad8199070 con 0x7f0ad8101dd0 2024-04-25T00:32:33.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.932+0000 7f0ad67fc700 1 --2- 172.21.15.32:0/221981867 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0ad8101dd0 0x7f0ad819ec90 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x7f0acc00d980 tx=0x7f0acc00dc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:33.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.932+0000 7f0abffff700 1 -- 172.21.15.32:0/221981867 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0acc017070 con 0x7f0ad8101dd0 2024-04-25T00:32:33.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.932+0000 7f0add6cb700 1 -- 172.21.15.32:0/221981867 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0ad8199360 con 0x7f0ad8101dd0 2024-04-25T00:32:33.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.932+0000 7f0abffff700 1 -- 172.21.15.32:0/221981867 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0acc009e10 con 0x7f0ad8101dd0 2024-04-25T00:32:33.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.932+0000 7f0add6cb700 1 -- 172.21.15.32:0/221981867 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0ad81a5620 con 0x7f0ad8101dd0 2024-04-25T00:32:33.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.932+0000 7f0abffff700 1 -- 172.21.15.32:0/221981867 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0acc00f620 con 0x7f0ad8101dd0 2024-04-25T00:32:33.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.933+0000 7f0add6cb700 1 -- 172.21.15.32:0/221981867 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0ad8068f20 con 0x7f0ad8101dd0 2024-04-25T00:32:33.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.935+0000 7f0abffff700 1 -- 172.21.15.32:0/221981867 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f0acc010460 con 0x7f0ad8101dd0 2024-04-25T00:32:33.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.935+0000 7f0abffff700 1 --2- 172.21.15.32:0/221981867 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f0ac00a2cb0 0x7f0ac00a5170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:33.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.935+0000 7f0abffff700 1 -- 172.21.15.32:0/221981867 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f0acc0ce990 con 0x7f0ad8101dd0 2024-04-25T00:32:33.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.935+0000 7f0ad6ffd700 1 --2- 172.21.15.32:0/221981867 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f0ac00a2cb0 0x7f0ac00a5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:33.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.936+0000 7f0ad6ffd700 1 --2- 172.21.15.32:0/221981867 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f0ac00a2cb0 0x7f0ac00a5170 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f0ac8000c00 tx=0x7f0ac8005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:33.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:33.940+0000 7f0abffff700 1 -- 172.21.15.32:0/221981867 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f0acc09b430 con 0x7f0ad8101dd0 2024-04-25T00:32:34.144 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:34.142+0000 7f0add6cb700 1 -- 172.21.15.32:0/221981867 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f0ad8199e40 con 0x7f0ac00a2cb0 2024-04-25T00:32:34.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:34.152+0000 7f0abffff700 1 -- 172.21.15.32:0/221981867 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+340355 (secure 0 0 0) 0x7f0ad8199e40 con 0x7f0ac00a2cb0 2024-04-25T00:32:34.157 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:32:34.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:34.155+0000 7f0add6cb700 1 -- 172.21.15.32:0/221981867 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f0ac00a2cb0 msgr2=0x7f0ac00a5170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:34.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:34.155+0000 7f0add6cb700 1 --2- 172.21.15.32:0/221981867 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f0ac00a2cb0 0x7f0ac00a5170 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f0ac8000c00 tx=0x7f0ac8005fb0 comp rx=0 tx=0).stop 2024-04-25T00:32:34.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:34.155+0000 7f0add6cb700 1 -- 172.21.15.32:0/221981867 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0ad8101dd0 msgr2=0x7f0ad819ec90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:34.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:34.155+0000 7f0add6cb700 1 --2- 172.21.15.32:0/221981867 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0ad8101dd0 0x7f0ad819ec90 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x7f0acc00d980 tx=0x7f0acc00dc90 comp rx=0 tx=0).stop 2024-04-25T00:32:34.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:34.155+0000 7f0add6cb700 1 -- 172.21.15.32:0/221981867 shutdown_connections 2024-04-25T00:32:34.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:34.155+0000 7f0add6cb700 1 --2- 172.21.15.32:0/221981867 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f0ac00a2cb0 0x7f0ac00a5170 unknown :-1 s=CLOSED pgs=220 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:34.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:34.155+0000 7f0add6cb700 1 --2- 172.21.15.32:0/221981867 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0ad8106470 0x7f0ad8198910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:34.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:34.155+0000 7f0add6cb700 1 --2- 172.21.15.32:0/221981867 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0ad8101420 0x7f0ad819e750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:34.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:34.155+0000 7f0add6cb700 1 --2- 172.21.15.32:0/221981867 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0ad8101dd0 0x7f0ad819ec90 unknown :-1 s=CLOSED pgs=343 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:34.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:34.155+0000 7f0add6cb700 1 -- 172.21.15.32:0/221981867 >> 172.21.15.32:0/221981867 conn(0x7f0ad8077180 msgr2=0x7f0ad806ad10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:34.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:34.155+0000 7f0add6cb700 1 -- 172.21.15.32:0/221981867 shutdown_connections 2024-04-25T00:32:34.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:34.155+0000 7f0add6cb700 1 -- 172.21.15.32:0/221981867 wait complete. 2024-04-25T00:32:34.161 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-04-25T00:32:34.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:34 smithi032 ceph-mon[29357]: pgmap v346: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 7 op/s 2024-04-25T00:32:34.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:34 smithi121 ceph-mon[32231]: pgmap v346: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 7 op/s 2024-04-25T00:32:34.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:34 smithi130 ceph-mon[32525]: pgmap v346: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 7 op/s 2024-04-25T00:32:35.060 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":346,"stamp":"2024-04-25T00:32:32.887928+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":656378,"num_objects":64,"num_object_clones":0,"num_object_copies":194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":64,"num_whiteouts":0,"num_read":623,"num_read_kb":430,"num_write":342,"num_write_kb":3053,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1180853,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":277,"ondisk_log_size":277,"up":419,"acting":419,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":419,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":375776,"kb_used_data":14084,"kb_used_omap":18,"kb_used_meta":361517,"kb_avail":1124566048,"statfs":{"total":1151940427776,"available":1151555633152,"internally_reserved":0,"allocated":14422016,"data_stored":7954844,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19142,"internal_metadata":370193722},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":85,"num_read_kb":40,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003062"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.711419+0000","last_change":"2024-04-25T00:30:48.538719+0000","last_active":"2024-04-25T00:31:44.711419+0000","last_peered":"2024-04-25T00:31:44.711419+0000","last_clean":"2024-04-25T00:31:44.711419+0000","last_became_active":"2024-04-25T00:30:48.538356+0000","last_became_peered":"2024-04-25T00:30:48.538356+0000","last_unstale":"2024-04-25T00:31:44.711419+0000","last_undegraded":"2024-04-25T00:31:44.711419+0000","last_fullsized":"2024-04-25T00:31:44.711419+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:39:08.974838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707296+0000","last_change":"2024-04-25T00:30:48.532262+0000","last_active":"2024-04-25T00:31:44.707296+0000","last_peered":"2024-04-25T00:31:44.707296+0000","last_clean":"2024-04-25T00:31:44.707296+0000","last_became_active":"2024-04-25T00:30:48.531926+0000","last_became_peered":"2024-04-25T00:30:48.531926+0000","last_unstale":"2024-04-25T00:31:44.707296+0000","last_undegraded":"2024-04-25T00:31:44.707296+0000","last_fullsized":"2024-04-25T00:31:44.707296+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:12:06.527649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707060+0000","last_change":"2024-04-25T00:30:48.536383+0000","last_active":"2024-04-25T00:31:44.707060+0000","last_peered":"2024-04-25T00:31:44.707060+0000","last_clean":"2024-04-25T00:31:44.707060+0000","last_became_active":"2024-04-25T00:30:48.536048+0000","last_became_peered":"2024-04-25T00:30:48.536048+0000","last_unstale":"2024-04-25T00:31:44.707060+0000","last_undegraded":"2024-04-25T00:31:44.707060+0000","last_fullsized":"2024-04-25T00:31:44.707060+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:32:26.728441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707986+0000","last_change":"2024-04-25T00:30:48.532195+0000","last_active":"2024-04-25T00:31:44.707986+0000","last_peered":"2024-04-25T00:31:44.707986+0000","last_clean":"2024-04-25T00:31:44.707986+0000","last_became_active":"2024-04-25T00:30:48.531851+0000","last_became_peered":"2024-04-25T00:30:48.531851+0000","last_unstale":"2024-04-25T00:31:44.707986+0000","last_undegraded":"2024-04-25T00:31:44.707986+0000","last_fullsized":"2024-04-25T00:31:44.707986+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:15:34.704958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710960+0000","last_change":"2024-04-25T00:30:48.534024+0000","last_active":"2024-04-25T00:31:44.710960+0000","last_peered":"2024-04-25T00:31:44.710960+0000","last_clean":"2024-04-25T00:31:44.710960+0000","last_became_active":"2024-04-25T00:30:48.533681+0000","last_became_peered":"2024-04-25T00:30:48.533681+0000","last_unstale":"2024-04-25T00:31:44.710960+0000","last_undegraded":"2024-04-25T00:31:44.710960+0000","last_fullsized":"2024-04-25T00:31:44.710960+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:07:35.816816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707570+0000","last_change":"2024-04-25T00:30:48.541263+0000","last_active":"2024-04-25T00:31:44.707570+0000","last_peered":"2024-04-25T00:31:44.707570+0000","last_clean":"2024-04-25T00:31:44.707570+0000","last_became_active":"2024-04-25T00:30:48.540889+0000","last_became_peered":"2024-04-25T00:30:48.540889+0000","last_unstale":"2024-04-25T00:31:44.707570+0000","last_undegraded":"2024-04-25T00:31:44.707570+0000","last_fullsized":"2024-04-25T00:31:44.707570+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:31:42.067004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,4],"acting":[10,9,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710287+0000","last_change":"2024-04-25T00:30:48.533194+0000","last_active":"2024-04-25T00:31:44.710287+0000","last_peered":"2024-04-25T00:31:44.710287+0000","last_clean":"2024-04-25T00:31:44.710287+0000","last_became_active":"2024-04-25T00:30:48.532424+0000","last_became_peered":"2024-04-25T00:30:48.532424+0000","last_unstale":"2024-04-25T00:31:44.710287+0000","last_undegraded":"2024-04-25T00:31:44.710287+0000","last_fullsized":"2024-04-25T00:31:44.710287+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:06:50.187796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.169414+0000","last_change":"2024-04-25T00:30:48.542829+0000","last_active":"2024-04-25T00:31:45.169414+0000","last_peered":"2024-04-25T00:31:45.169414+0000","last_clean":"2024-04-25T00:31:45.169414+0000","last_became_active":"2024-04-25T00:30:48.542204+0000","last_became_peered":"2024-04-25T00:30:48.542204+0000","last_unstale":"2024-04-25T00:31:45.169414+0000","last_undegraded":"2024-04-25T00:31:45.169414+0000","last_fullsized":"2024-04-25T00:31:45.169414+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:56:47.869075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705841+0000","last_change":"2024-04-25T00:30:48.537924+0000","last_active":"2024-04-25T00:31:44.705841+0000","last_peered":"2024-04-25T00:31:44.705841+0000","last_clean":"2024-04-25T00:31:44.705841+0000","last_became_active":"2024-04-25T00:30:48.537547+0000","last_became_peered":"2024-04-25T00:30:48.537547+0000","last_unstale":"2024-04-25T00:31:44.705841+0000","last_undegraded":"2024-04-25T00:31:44.705841+0000","last_fullsized":"2024-04-25T00:31:44.705841+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:26:45.590923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705846+0000","last_change":"2024-04-25T00:30:48.539717+0000","last_active":"2024-04-25T00:31:44.705846+0000","last_peered":"2024-04-25T00:31:44.705846+0000","last_clean":"2024-04-25T00:31:44.705846+0000","last_became_active":"2024-04-25T00:30:48.539242+0000","last_became_peered":"2024-04-25T00:30:48.539242+0000","last_unstale":"2024-04-25T00:31:44.705846+0000","last_undegraded":"2024-04-25T00:31:44.705846+0000","last_fullsized":"2024-04-25T00:31:44.705846+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:58:15.353959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710657+0000","last_change":"2024-04-25T00:30:48.542767+0000","last_active":"2024-04-25T00:31:44.710657+0000","last_peered":"2024-04-25T00:31:44.710657+0000","last_clean":"2024-04-25T00:31:44.710657+0000","last_became_active":"2024-04-25T00:30:48.542252+0000","last_became_peered":"2024-04-25T00:30:48.542252+0000","last_unstale":"2024-04-25T00:31:44.710657+0000","last_undegraded":"2024-04-25T00:31:44.710657+0000","last_fullsized":"2024-04-25T00:31:44.710657+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:07:49.414660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707985+0000","last_change":"2024-04-25T00:30:48.532248+0000","last_active":"2024-04-25T00:31:44.707985+0000","last_peered":"2024-04-25T00:31:44.707985+0000","last_clean":"2024-04-25T00:31:44.707985+0000","last_became_active":"2024-04-25T00:30:48.531876+0000","last_became_peered":"2024-04-25T00:30:48.531876+0000","last_unstale":"2024-04-25T00:31:44.707985+0000","last_undegraded":"2024-04-25T00:31:44.707985+0000","last_fullsized":"2024-04-25T00:31:44.707985+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:00:35.711693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.027601+0000","last_change":"2024-04-25T00:30:48.526871+0000","last_active":"2024-04-25T00:31:45.027601+0000","last_peered":"2024-04-25T00:31:45.027601+0000","last_clean":"2024-04-25T00:31:45.027601+0000","last_became_active":"2024-04-25T00:30:48.526477+0000","last_became_peered":"2024-04-25T00:30:48.526477+0000","last_unstale":"2024-04-25T00:31:45.027601+0000","last_undegraded":"2024-04-25T00:31:45.027601+0000","last_fullsized":"2024-04-25T00:31:45.027601+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:17:59.447932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707068+0000","last_change":"2024-04-25T00:30:48.530297+0000","last_active":"2024-04-25T00:31:44.707068+0000","last_peered":"2024-04-25T00:31:44.707068+0000","last_clean":"2024-04-25T00:31:44.707068+0000","last_became_active":"2024-04-25T00:30:48.529091+0000","last_became_peered":"2024-04-25T00:30:48.529091+0000","last_unstale":"2024-04-25T00:31:44.707068+0000","last_undegraded":"2024-04-25T00:31:44.707068+0000","last_fullsized":"2024-04-25T00:31:44.707068+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:54:52.206881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.706839+0000","last_change":"2024-04-25T00:30:48.535629+0000","last_active":"2024-04-25T00:31:44.706839+0000","last_peered":"2024-04-25T00:31:44.706839+0000","last_clean":"2024-04-25T00:31:44.706839+0000","last_became_active":"2024-04-25T00:30:48.535190+0000","last_became_peered":"2024-04-25T00:30:48.535190+0000","last_unstale":"2024-04-25T00:31:44.706839+0000","last_undegraded":"2024-04-25T00:31:44.706839+0000","last_fullsized":"2024-04-25T00:31:44.706839+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:11:27.127964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.704421+0000","last_change":"2024-04-25T00:30:48.532988+0000","last_active":"2024-04-25T00:31:44.704421+0000","last_peered":"2024-04-25T00:31:44.704421+0000","last_clean":"2024-04-25T00:31:44.704421+0000","last_became_active":"2024-04-25T00:30:48.532670+0000","last_became_peered":"2024-04-25T00:30:48.532670+0000","last_unstale":"2024-04-25T00:31:44.704421+0000","last_undegraded":"2024-04-25T00:31:44.704421+0000","last_fullsized":"2024-04-25T00:31:44.704421+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:03:35.336175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.711247+0000","last_change":"2024-04-25T00:30:48.539499+0000","last_active":"2024-04-25T00:31:44.711247+0000","last_peered":"2024-04-25T00:31:44.711247+0000","last_clean":"2024-04-25T00:31:44.711247+0000","last_became_active":"2024-04-25T00:30:48.538516+0000","last_became_peered":"2024-04-25T00:30:48.538516+0000","last_unstale":"2024-04-25T00:31:44.711247+0000","last_undegraded":"2024-04-25T00:31:44.711247+0000","last_fullsized":"2024-04-25T00:31:44.711247+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:03:16.781091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.703779+0000","last_change":"2024-04-25T00:30:48.534204+0000","last_active":"2024-04-25T00:31:44.703779+0000","last_peered":"2024-04-25T00:31:44.703779+0000","last_clean":"2024-04-25T00:31:44.703779+0000","last_became_active":"2024-04-25T00:30:48.533818+0000","last_became_peered":"2024-04-25T00:30:48.533818+0000","last_unstale":"2024-04-25T00:31:44.703779+0000","last_undegraded":"2024-04-25T00:31:44.703779+0000","last_fullsized":"2024-04-25T00:31:44.703779+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:08:42.964056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707214+0000","last_change":"2024-04-25T00:30:48.542756+0000","last_active":"2024-04-25T00:31:44.707214+0000","last_peered":"2024-04-25T00:31:44.707214+0000","last_clean":"2024-04-25T00:31:44.707214+0000","last_became_active":"2024-04-25T00:30:48.541824+0000","last_became_peered":"2024-04-25T00:30:48.541824+0000","last_unstale":"2024-04-25T00:31:44.707214+0000","last_undegraded":"2024-04-25T00:31:44.707214+0000","last_fullsized":"2024-04-25T00:31:44.707214+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:31:40.996474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.706047+0000","last_change":"2024-04-25T00:30:52.788477+0000","last_active":"2024-04-25T00:31:44.706047+0000","last_peered":"2024-04-25T00:31:44.706047+0000","last_clean":"2024-04-25T00:31:44.706047+0000","last_became_active":"2024-04-25T00:30:52.788073+0000","last_became_peered":"2024-04-25T00:30:52.788073+0000","last_unstale":"2024-04-25T00:31:44.706047+0000","last_undegraded":"2024-04-25T00:31:44.706047+0000","last_fullsized":"2024-04-25T00:31:44.706047+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:29:34.781119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,9,0],"acting":[2,3,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"89'9","reported_seq":37,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.703897+0000","last_change":"2024-04-25T00:30:47.524580+0000","last_active":"2024-04-25T00:31:44.703897+0000","last_peered":"2024-04-25T00:31:44.703897+0000","last_clean":"2024-04-25T00:31:44.703897+0000","last_became_active":"2024-04-25T00:30:47.524170+0000","last_became_peered":"2024-04-25T00:30:47.524170+0000","last_unstale":"2024-04-25T00:31:44.703897+0000","last_undegraded":"2024-04-25T00:31:44.703897+0000","last_fullsized":"2024-04-25T00:31:44.703897+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:53:08.734842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.169997+0000","last_change":"2024-04-25T00:30:48.542077+0000","last_active":"2024-04-25T00:31:45.169997+0000","last_peered":"2024-04-25T00:31:45.169997+0000","last_clean":"2024-04-25T00:31:45.169997+0000","last_became_active":"2024-04-25T00:30:48.539043+0000","last_became_peered":"2024-04-25T00:30:48.539043+0000","last_unstale":"2024-04-25T00:31:45.169997+0000","last_undegraded":"2024-04-25T00:31:45.169997+0000","last_fullsized":"2024-04-25T00:31:45.169997+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:48:52.254800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705852+0000","last_change":"2024-04-25T00:30:52.786091+0000","last_active":"2024-04-25T00:31:44.705852+0000","last_peered":"2024-04-25T00:31:44.705852+0000","last_clean":"2024-04-25T00:31:44.705852+0000","last_became_active":"2024-04-25T00:30:52.785511+0000","last_became_peered":"2024-04-25T00:30:52.785511+0000","last_unstale":"2024-04-25T00:31:44.705852+0000","last_undegraded":"2024-04-25T00:31:44.705852+0000","last_fullsized":"2024-04-25T00:31:44.705852+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:19:17.992033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,9,3],"acting":[4,6,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1e","version":"89'2","reported_seq":29,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.170167+0000","last_change":"2024-04-25T00:30:47.529293+0000","last_active":"2024-04-25T00:31:45.170167+0000","last_peered":"2024-04-25T00:31:45.170167+0000","last_clean":"2024-04-25T00:31:45.170167+0000","last_became_active":"2024-04-25T00:30:47.527975+0000","last_became_peered":"2024-04-25T00:30:47.527975+0000","last_unstale":"2024-04-25T00:31:45.170167+0000","last_undegraded":"2024-04-25T00:31:45.170167+0000","last_fullsized":"2024-04-25T00:31:45.170167+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:17:40.057962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707444+0000","last_change":"2024-04-25T00:30:48.530993+0000","last_active":"2024-04-25T00:31:44.707444+0000","last_peered":"2024-04-25T00:31:44.707444+0000","last_clean":"2024-04-25T00:31:44.707444+0000","last_became_active":"2024-04-25T00:30:48.530550+0000","last_became_peered":"2024-04-25T00:30:48.530550+0000","last_unstale":"2024-04-25T00:31:44.707444+0000","last_undegraded":"2024-04-25T00:31:44.707444+0000","last_fullsized":"2024-04-25T00:31:44.707444+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:11:35.308456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.706981+0000","last_change":"2024-04-25T00:30:52.787774+0000","last_active":"2024-04-25T00:31:44.706981+0000","last_peered":"2024-04-25T00:31:44.706981+0000","last_clean":"2024-04-25T00:31:44.706981+0000","last_became_active":"2024-04-25T00:30:52.786988+0000","last_became_peered":"2024-04-25T00:30:52.786988+0000","last_unstale":"2024-04-25T00:31:44.706981+0000","last_undegraded":"2024-04-25T00:31:44.706981+0000","last_fullsized":"2024-04-25T00:31:44.706981+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:58:28.081767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,9,5],"acting":[4,3,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1d","version":"89'4","reported_seq":31,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.711899+0000","last_change":"2024-04-25T00:30:47.523477+0000","last_active":"2024-04-25T00:31:44.711899+0000","last_peered":"2024-04-25T00:31:44.711899+0000","last_clean":"2024-04-25T00:31:44.711899+0000","last_became_active":"2024-04-25T00:30:47.522961+0000","last_became_peered":"2024-04-25T00:30:47.522961+0000","last_unstale":"2024-04-25T00:31:44.711899+0000","last_undegraded":"2024-04-25T00:31:44.711899+0000","last_fullsized":"2024-04-25T00:31:44.711899+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:39:39.793477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705019+0000","last_change":"2024-04-25T00:30:48.527746+0000","last_active":"2024-04-25T00:31:44.705019+0000","last_peered":"2024-04-25T00:31:44.705019+0000","last_clean":"2024-04-25T00:31:44.705019+0000","last_became_active":"2024-04-25T00:30:48.527374+0000","last_became_peered":"2024-04-25T00:30:48.527374+0000","last_unstale":"2024-04-25T00:31:44.705019+0000","last_undegraded":"2024-04-25T00:31:44.705019+0000","last_fullsized":"2024-04-25T00:31:44.705019+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:28:36.822220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705693+0000","last_change":"2024-04-25T00:30:52.782368+0000","last_active":"2024-04-25T00:31:44.705693+0000","last_peered":"2024-04-25T00:31:44.705693+0000","last_clean":"2024-04-25T00:31:44.705693+0000","last_became_active":"2024-04-25T00:30:52.781875+0000","last_became_peered":"2024-04-25T00:30:52.781875+0000","last_unstale":"2024-04-25T00:31:44.705693+0000","last_undegraded":"2024-04-25T00:31:44.705693+0000","last_fullsized":"2024-04-25T00:31:44.705693+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:07:55.509131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5,3],"acting":[4,6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1c","version":"84'2","reported_seq":29,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.711538+0000","last_change":"2024-04-25T00:30:47.523707+0000","last_active":"2024-04-25T00:31:44.711538+0000","last_peered":"2024-04-25T00:31:44.711538+0000","last_clean":"2024-04-25T00:31:44.711538+0000","last_became_active":"2024-04-25T00:30:47.523216+0000","last_became_peered":"2024-04-25T00:30:47.523216+0000","last_unstale":"2024-04-25T00:31:44.711538+0000","last_undegraded":"2024-04-25T00:31:44.711538+0000","last_fullsized":"2024-04-25T00:31:44.711538+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:21:21.006368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.029744+0000","last_change":"2024-04-25T00:30:48.529235+0000","last_active":"2024-04-25T00:31:45.029744+0000","last_peered":"2024-04-25T00:31:45.029744+0000","last_clean":"2024-04-25T00:31:45.029744+0000","last_became_active":"2024-04-25T00:30:48.528887+0000","last_became_peered":"2024-04-25T00:30:48.528887+0000","last_unstale":"2024-04-25T00:31:45.029744+0000","last_undegraded":"2024-04-25T00:31:45.029744+0000","last_fullsized":"2024-04-25T00:31:45.029744+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:18:05.290578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.714433+0000","last_change":"2024-04-25T00:31:44.714433+0000","last_active":"2024-04-25T00:31:44.714433+0000","last_peered":"2024-04-25T00:31:44.714433+0000","last_clean":"2024-04-25T00:31:44.714433+0000","last_became_active":"2024-04-25T00:31:44.713810+0000","last_became_peered":"2024-04-25T00:31:44.713810+0000","last_unstale":"2024-04-25T00:31:44.714433+0000","last_undegraded":"2024-04-25T00:31:44.714433+0000","last_fullsized":"2024-04-25T00:31:44.714433+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:00:49.815134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,7,11],"acting":[2,8,7,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1b","version":"89'5","reported_seq":112,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:32:31.096275+0000","last_change":"2024-04-25T00:30:47.518472+0000","last_active":"2024-04-25T00:32:31.096275+0000","last_peered":"2024-04-25T00:32:31.096275+0000","last_clean":"2024-04-25T00:32:31.096275+0000","last_became_active":"2024-04-25T00:30:47.518140+0000","last_became_peered":"2024-04-25T00:30:47.518140+0000","last_unstale":"2024-04-25T00:32:31.096275+0000","last_undegraded":"2024-04-25T00:32:31.096275+0000","last_fullsized":"2024-04-25T00:32:31.096275+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:19:07.780949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":622,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":80,"num_read_kb":2,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710517+0000","last_change":"2024-04-25T00:30:48.533757+0000","last_active":"2024-04-25T00:31:44.710517+0000","last_peered":"2024-04-25T00:31:44.710517+0000","last_clean":"2024-04-25T00:31:44.710517+0000","last_became_active":"2024-04-25T00:30:48.533421+0000","last_became_peered":"2024-04-25T00:30:48.533421+0000","last_unstale":"2024-04-25T00:31:44.710517+0000","last_undegraded":"2024-04-25T00:31:44.710517+0000","last_fullsized":"2024-04-25T00:31:44.710517+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:11:38.190783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.031412+0000","last_change":"2024-04-25T00:31:45.031412+0000","last_active":"2024-04-25T00:31:45.031412+0000","last_peered":"2024-04-25T00:31:45.031412+0000","last_clean":"2024-04-25T00:31:45.031412+0000","last_became_active":"2024-04-25T00:31:45.030822+0000","last_became_peered":"2024-04-25T00:31:45.030822+0000","last_unstale":"2024-04-25T00:31:45.031412+0000","last_undegraded":"2024-04-25T00:31:45.031412+0000","last_fullsized":"2024-04-25T00:31:45.031412+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:34:26.758966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,8,5],"acting":[2,6,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1a","version":"89'3","reported_seq":30,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707538+0000","last_change":"2024-04-25T00:30:47.527306+0000","last_active":"2024-04-25T00:31:44.707538+0000","last_peered":"2024-04-25T00:31:44.707538+0000","last_clean":"2024-04-25T00:31:44.707538+0000","last_became_active":"2024-04-25T00:30:47.526935+0000","last_became_peered":"2024-04-25T00:30:47.526935+0000","last_unstale":"2024-04-25T00:31:44.707538+0000","last_undegraded":"2024-04-25T00:31:44.707538+0000","last_fullsized":"2024-04-25T00:31:44.707538+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:41:37.951143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":11,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.171065+0000","last_change":"2024-04-25T00:31:45.170610+0000","last_active":"2024-04-25T00:31:45.171065+0000","last_peered":"2024-04-25T00:31:45.171065+0000","last_clean":"2024-04-25T00:31:45.171065+0000","last_became_active":"2024-04-25T00:31:45.170183+0000","last_became_peered":"2024-04-25T00:31:45.170183+0000","last_unstale":"2024-04-25T00:31:45.171065+0000","last_undegraded":"2024-04-25T00:31:45.171065+0000","last_fullsized":"2024-04-25T00:31:45.171065+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:39:39.781997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,4,3],"acting":[9,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710518+0000","last_change":"2024-04-25T00:30:52.784986+0000","last_active":"2024-04-25T00:31:44.710518+0000","last_peered":"2024-04-25T00:31:44.710518+0000","last_clean":"2024-04-25T00:31:44.710518+0000","last_became_active":"2024-04-25T00:30:52.784593+0000","last_became_peered":"2024-04-25T00:30:52.784593+0000","last_unstale":"2024-04-25T00:31:44.710518+0000","last_undegraded":"2024-04-25T00:31:44.710518+0000","last_fullsized":"2024-04-25T00:31:44.710518+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:04:05.833730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,9,0],"acting":[11,4,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.19","version":"89'2","reported_seq":29,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.027126+0000","last_change":"2024-04-25T00:30:47.523663+0000","last_active":"2024-04-25T00:31:45.027126+0000","last_peered":"2024-04-25T00:31:45.027126+0000","last_clean":"2024-04-25T00:31:45.027126+0000","last_became_active":"2024-04-25T00:30:47.523324+0000","last_became_peered":"2024-04-25T00:30:47.523324+0000","last_unstale":"2024-04-25T00:31:45.027126+0000","last_undegraded":"2024-04-25T00:31:45.027126+0000","last_fullsized":"2024-04-25T00:31:45.027126+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:51:23.593579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.169180+0000","last_change":"2024-04-25T00:30:48.529738+0000","last_active":"2024-04-25T00:31:45.169180+0000","last_peered":"2024-04-25T00:31:45.169180+0000","last_clean":"2024-04-25T00:31:45.169180+0000","last_became_active":"2024-04-25T00:30:48.529350+0000","last_became_peered":"2024-04-25T00:30:48.529350+0000","last_unstale":"2024-04-25T00:31:45.169180+0000","last_undegraded":"2024-04-25T00:31:45.169180+0000","last_fullsized":"2024-04-25T00:31:45.169180+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:51:29.238293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707182+0000","last_change":"2024-04-25T00:30:52.788480+0000","last_active":"2024-04-25T00:31:44.707182+0000","last_peered":"2024-04-25T00:31:44.707182+0000","last_clean":"2024-04-25T00:31:44.707182+0000","last_became_active":"2024-04-25T00:30:52.788065+0000","last_became_peered":"2024-04-25T00:30:52.788065+0000","last_unstale":"2024-04-25T00:31:44.707182+0000","last_undegraded":"2024-04-25T00:31:44.707182+0000","last_fullsized":"2024-04-25T00:31:44.707182+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:16:50.179177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6,1],"acting":[0,7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.18","version":"89'1","reported_seq":28,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.029206+0000","last_change":"2024-04-25T00:30:47.524706+0000","last_active":"2024-04-25T00:31:45.029206+0000","last_peered":"2024-04-25T00:31:45.029206+0000","last_clean":"2024-04-25T00:31:45.029206+0000","last_became_active":"2024-04-25T00:30:47.524355+0000","last_became_peered":"2024-04-25T00:30:47.524355+0000","last_unstale":"2024-04-25T00:31:45.029206+0000","last_undegraded":"2024-04-25T00:31:45.029206+0000","last_fullsized":"2024-04-25T00:31:45.029206+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:22:38.093362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.703331+0000","last_change":"2024-04-25T00:30:48.533266+0000","last_active":"2024-04-25T00:31:44.703331+0000","last_peered":"2024-04-25T00:31:44.703331+0000","last_clean":"2024-04-25T00:31:44.703331+0000","last_became_active":"2024-04-25T00:30:48.532932+0000","last_became_peered":"2024-04-25T00:30:48.532932+0000","last_unstale":"2024-04-25T00:31:44.703331+0000","last_undegraded":"2024-04-25T00:31:44.703331+0000","last_fullsized":"2024-04-25T00:31:44.703331+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:00:54.147725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.169493+0000","last_change":"2024-04-25T00:30:52.780317+0000","last_active":"2024-04-25T00:31:45.169493+0000","last_peered":"2024-04-25T00:31:45.169493+0000","last_clean":"2024-04-25T00:31:45.169493+0000","last_became_active":"2024-04-25T00:30:52.778539+0000","last_became_peered":"2024-04-25T00:30:52.778539+0000","last_unstale":"2024-04-25T00:31:45.169493+0000","last_undegraded":"2024-04-25T00:31:45.169493+0000","last_fullsized":"2024-04-25T00:31:45.169493+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:34:49.527307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1,5],"acting":[3,7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.17","version":"89'3","reported_seq":107,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:32:30.094202+0000","last_change":"2024-04-25T00:30:47.525647+0000","last_active":"2024-04-25T00:32:30.094202+0000","last_peered":"2024-04-25T00:32:30.094202+0000","last_clean":"2024-04-25T00:32:30.094202+0000","last_became_active":"2024-04-25T00:30:47.525280+0000","last_became_peered":"2024-04-25T00:30:47.525280+0000","last_unstale":"2024-04-25T00:32:30.094202+0000","last_undegraded":"2024-04-25T00:32:30.094202+0000","last_fullsized":"2024-04-25T00:32:30.094202+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:56:15.761173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":77,"num_read_kb":77,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705528+0000","last_change":"2024-04-25T00:30:48.527913+0000","last_active":"2024-04-25T00:31:44.705528+0000","last_peered":"2024-04-25T00:31:44.705528+0000","last_clean":"2024-04-25T00:31:44.705528+0000","last_became_active":"2024-04-25T00:30:48.527538+0000","last_became_peered":"2024-04-25T00:30:48.527538+0000","last_unstale":"2024-04-25T00:31:44.705528+0000","last_undegraded":"2024-04-25T00:31:44.705528+0000","last_fullsized":"2024-04-25T00:31:44.705528+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:48:38.646867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.703108+0000","last_change":"2024-04-25T00:30:52.788267+0000","last_active":"2024-04-25T00:31:44.703108+0000","last_peered":"2024-04-25T00:31:44.703108+0000","last_clean":"2024-04-25T00:31:44.703108+0000","last_became_active":"2024-04-25T00:30:52.785188+0000","last_became_peered":"2024-04-25T00:30:52.785188+0000","last_unstale":"2024-04-25T00:31:44.703108+0000","last_undegraded":"2024-04-25T00:31:44.703108+0000","last_fullsized":"2024-04-25T00:31:44.703108+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:38:25.194673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,0,2,4],"acting":[9,0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.16","version":"89'2","reported_seq":29,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710588+0000","last_change":"2024-04-25T00:30:47.522213+0000","last_active":"2024-04-25T00:31:44.710588+0000","last_peered":"2024-04-25T00:31:44.710588+0000","last_clean":"2024-04-25T00:31:44.710588+0000","last_became_active":"2024-04-25T00:30:47.521519+0000","last_became_peered":"2024-04-25T00:30:47.521519+0000","last_unstale":"2024-04-25T00:31:44.710588+0000","last_undegraded":"2024-04-25T00:31:44.710588+0000","last_fullsized":"2024-04-25T00:31:44.710588+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:16:10.001222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":31,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.029866+0000","last_change":"2024-04-25T00:31:45.029866+0000","last_active":"2024-04-25T00:31:45.029866+0000","last_peered":"2024-04-25T00:31:45.029866+0000","last_clean":"2024-04-25T00:31:45.029866+0000","last_became_active":"2024-04-25T00:31:45.029587+0000","last_became_peered":"2024-04-25T00:31:45.029587+0000","last_unstale":"2024-04-25T00:31:45.029866+0000","last_undegraded":"2024-04-25T00:31:45.029866+0000","last_fullsized":"2024-04-25T00:31:45.029866+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:09:09.434492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707134+0000","last_change":"2024-04-25T00:30:52.786818+0000","last_active":"2024-04-25T00:31:44.707134+0000","last_peered":"2024-04-25T00:31:44.707134+0000","last_clean":"2024-04-25T00:31:44.707134+0000","last_became_active":"2024-04-25T00:30:52.786293+0000","last_became_peered":"2024-04-25T00:30:52.786293+0000","last_unstale":"2024-04-25T00:31:44.707134+0000","last_undegraded":"2024-04-25T00:31:44.707134+0000","last_fullsized":"2024-04-25T00:31:44.707134+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:27:02.751296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2,9],"acting":[4,1,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.15","version":"89'4","reported_seq":32,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.029587+0000","last_change":"2024-04-25T00:30:47.520882+0000","last_active":"2024-04-25T00:31:45.029587+0000","last_peered":"2024-04-25T00:31:45.029587+0000","last_clean":"2024-04-25T00:31:45.029587+0000","last_became_active":"2024-04-25T00:30:47.520545+0000","last_became_peered":"2024-04-25T00:30:47.520545+0000","last_unstale":"2024-04-25T00:31:45.029587+0000","last_undegraded":"2024-04-25T00:31:45.029587+0000","last_fullsized":"2024-04-25T00:31:45.029587+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:46:04.715580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705195+0000","last_change":"2024-04-25T00:30:48.536814+0000","last_active":"2024-04-25T00:31:44.705195+0000","last_peered":"2024-04-25T00:31:44.705195+0000","last_clean":"2024-04-25T00:31:44.705195+0000","last_became_active":"2024-04-25T00:30:48.536486+0000","last_became_peered":"2024-04-25T00:30:48.536486+0000","last_unstale":"2024-04-25T00:31:44.705195+0000","last_undegraded":"2024-04-25T00:31:44.705195+0000","last_fullsized":"2024-04-25T00:31:44.705195+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:58:23.861959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.031626+0000","last_change":"2024-04-25T00:31:45.031208+0000","last_active":"2024-04-25T00:31:45.031626+0000","last_peered":"2024-04-25T00:31:45.031626+0000","last_clean":"2024-04-25T00:31:45.031626+0000","last_became_active":"2024-04-25T00:31:45.030831+0000","last_became_peered":"2024-04-25T00:31:45.030831+0000","last_unstale":"2024-04-25T00:31:45.031626+0000","last_undegraded":"2024-04-25T00:31:45.031626+0000","last_fullsized":"2024-04-25T00:31:45.031626+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:42:01.869146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,11,5],"acting":[10,8,11,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.14","version":"89'31","reported_seq":219,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:32:31.095617+0000","last_change":"2024-04-25T00:30:47.526899+0000","last_active":"2024-04-25T00:32:31.095617+0000","last_peered":"2024-04-25T00:32:31.095617+0000","last_clean":"2024-04-25T00:32:31.095617+0000","last_became_active":"2024-04-25T00:30:47.526528+0000","last_became_peered":"2024-04-25T00:30:47.526528+0000","last_unstale":"2024-04-25T00:32:31.095617+0000","last_undegraded":"2024-04-25T00:32:31.095617+0000","last_fullsized":"2024-04-25T00:32:31.095617+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":31,"log_dups_size":0,"ondisk_log_size":31,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:13:34.004584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61775,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":161,"num_read_kb":141,"num_write":32,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.712063+0000","last_change":"2024-04-25T00:30:48.538225+0000","last_active":"2024-04-25T00:31:44.712063+0000","last_peered":"2024-04-25T00:31:44.712063+0000","last_clean":"2024-04-25T00:31:44.712063+0000","last_became_active":"2024-04-25T00:30:48.537807+0000","last_became_peered":"2024-04-25T00:30:48.537807+0000","last_unstale":"2024-04-25T00:31:44.712063+0000","last_undegraded":"2024-04-25T00:31:44.712063+0000","last_fullsized":"2024-04-25T00:31:44.712063+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:00:46.275584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.029972+0000","last_change":"2024-04-25T00:30:52.782266+0000","last_active":"2024-04-25T00:31:45.029972+0000","last_peered":"2024-04-25T00:31:45.029972+0000","last_clean":"2024-04-25T00:31:45.029972+0000","last_became_active":"2024-04-25T00:30:52.781133+0000","last_became_peered":"2024-04-25T00:30:52.781133+0000","last_unstale":"2024-04-25T00:31:45.029972+0000","last_undegraded":"2024-04-25T00:31:45.029972+0000","last_fullsized":"2024-04-25T00:31:45.029972+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:19:03.818560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,8,3,6],"acting":[5,8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.13","version":"89'6","reported_seq":34,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.706421+0000","last_change":"2024-04-25T00:30:47.520909+0000","last_active":"2024-04-25T00:31:44.706421+0000","last_peered":"2024-04-25T00:31:44.706421+0000","last_clean":"2024-04-25T00:31:44.706421+0000","last_became_active":"2024-04-25T00:30:47.520287+0000","last_became_peered":"2024-04-25T00:30:47.520287+0000","last_unstale":"2024-04-25T00:31:44.706421+0000","last_undegraded":"2024-04-25T00:31:44.706421+0000","last_fullsized":"2024-04-25T00:31:44.706421+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:49:43.795115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.027156+0000","last_change":"2024-04-25T00:30:48.532270+0000","last_active":"2024-04-25T00:31:45.027156+0000","last_peered":"2024-04-25T00:31:45.027156+0000","last_clean":"2024-04-25T00:31:45.027156+0000","last_became_active":"2024-04-25T00:30:48.531882+0000","last_became_peered":"2024-04-25T00:30:48.531882+0000","last_unstale":"2024-04-25T00:31:45.027156+0000","last_undegraded":"2024-04-25T00:31:45.027156+0000","last_fullsized":"2024-04-25T00:31:45.027156+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:58:17.865635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.711177+0000","last_change":"2024-04-25T00:30:52.777708+0000","last_active":"2024-04-25T00:31:44.711177+0000","last_peered":"2024-04-25T00:31:44.711177+0000","last_clean":"2024-04-25T00:31:44.711177+0000","last_became_active":"2024-04-25T00:30:52.777254+0000","last_became_peered":"2024-04-25T00:30:52.777254+0000","last_unstale":"2024-04-25T00:31:44.711177+0000","last_undegraded":"2024-04-25T00:31:44.711177+0000","last_fullsized":"2024-04-25T00:31:44.711177+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:29:47.427190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,9,7,6],"acting":[11,9,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.12","version":"89'3","reported_seq":30,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.029488+0000","last_change":"2024-04-25T00:30:47.524667+0000","last_active":"2024-04-25T00:31:45.029488+0000","last_peered":"2024-04-25T00:31:45.029488+0000","last_clean":"2024-04-25T00:31:45.029488+0000","last_became_active":"2024-04-25T00:30:47.524361+0000","last_became_peered":"2024-04-25T00:30:47.524361+0000","last_unstale":"2024-04-25T00:31:45.029488+0000","last_undegraded":"2024-04-25T00:31:45.029488+0000","last_fullsized":"2024-04-25T00:31:45.029488+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:16:08.805720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710541+0000","last_change":"2024-04-25T00:30:48.538658+0000","last_active":"2024-04-25T00:31:44.710541+0000","last_peered":"2024-04-25T00:31:44.710541+0000","last_clean":"2024-04-25T00:31:44.710541+0000","last_became_active":"2024-04-25T00:30:48.538316+0000","last_became_peered":"2024-04-25T00:30:48.538316+0000","last_unstale":"2024-04-25T00:31:44.710541+0000","last_undegraded":"2024-04-25T00:31:44.710541+0000","last_fullsized":"2024-04-25T00:31:44.710541+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:24:47.303559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.703610+0000","last_change":"2024-04-25T00:30:52.789512+0000","last_active":"2024-04-25T00:31:44.703610+0000","last_peered":"2024-04-25T00:31:44.703610+0000","last_clean":"2024-04-25T00:31:44.703610+0000","last_became_active":"2024-04-25T00:30:52.788929+0000","last_became_peered":"2024-04-25T00:30:52.788929+0000","last_unstale":"2024-04-25T00:31:44.703610+0000","last_undegraded":"2024-04-25T00:31:44.703610+0000","last_fullsized":"2024-04-25T00:31:44.703610+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:21:05.195296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,0,3],"acting":[9,7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.11","version":"89'4","reported_seq":112,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:32:31.096250+0000","last_change":"2024-04-25T00:30:47.513627+0000","last_active":"2024-04-25T00:32:31.096250+0000","last_peered":"2024-04-25T00:32:31.096250+0000","last_clean":"2024-04-25T00:32:31.096250+0000","last_became_active":"2024-04-25T00:30:47.513215+0000","last_became_peered":"2024-04-25T00:30:47.513215+0000","last_unstale":"2024-04-25T00:32:31.096250+0000","last_undegraded":"2024-04-25T00:32:31.096250+0000","last_fullsized":"2024-04-25T00:32:31.096250+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:32:16.111590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":979,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":82,"num_read_kb":4,"num_write":3,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710171+0000","last_change":"2024-04-25T00:30:48.533056+0000","last_active":"2024-04-25T00:31:44.710171+0000","last_peered":"2024-04-25T00:31:44.710171+0000","last_clean":"2024-04-25T00:31:44.710171+0000","last_became_active":"2024-04-25T00:30:48.532698+0000","last_became_peered":"2024-04-25T00:30:48.532698+0000","last_unstale":"2024-04-25T00:31:44.710171+0000","last_undegraded":"2024-04-25T00:31:44.710171+0000","last_fullsized":"2024-04-25T00:31:44.710171+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:36:24.987784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.027530+0000","last_change":"2024-04-25T00:30:52.780669+0000","last_active":"2024-04-25T00:31:45.027530+0000","last_peered":"2024-04-25T00:31:45.027530+0000","last_clean":"2024-04-25T00:31:45.027530+0000","last_became_active":"2024-04-25T00:30:52.780222+0000","last_became_peered":"2024-04-25T00:30:52.780222+0000","last_unstale":"2024-04-25T00:31:45.027530+0000","last_undegraded":"2024-04-25T00:31:45.027530+0000","last_fullsized":"2024-04-25T00:31:45.027530+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:50:27.415077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,9,11],"acting":[6,3,9,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"89'15","reported_seq":43,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.709662+0000","last_change":"2024-04-25T00:30:47.522547+0000","last_active":"2024-04-25T00:31:44.709662+0000","last_peered":"2024-04-25T00:31:44.709662+0000","last_clean":"2024-04-25T00:31:44.709662+0000","last_became_active":"2024-04-25T00:30:47.522193+0000","last_became_peered":"2024-04-25T00:30:47.522193+0000","last_unstale":"2024-04-25T00:31:44.709662+0000","last_undegraded":"2024-04-25T00:31:44.709662+0000","last_fullsized":"2024-04-25T00:31:44.709662+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:10:10.417552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,2],"acting":[8,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.029017+0000","last_change":"2024-04-25T00:30:48.534363+0000","last_active":"2024-04-25T00:31:45.029017+0000","last_peered":"2024-04-25T00:31:45.029017+0000","last_clean":"2024-04-25T00:31:45.029017+0000","last_became_active":"2024-04-25T00:30:48.533855+0000","last_became_peered":"2024-04-25T00:30:48.533855+0000","last_unstale":"2024-04-25T00:31:45.029017+0000","last_undegraded":"2024-04-25T00:31:45.029017+0000","last_fullsized":"2024-04-25T00:31:45.029017+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:28:29.374501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.028927+0000","last_change":"2024-04-25T00:30:52.790323+0000","last_active":"2024-04-25T00:31:45.028927+0000","last_peered":"2024-04-25T00:31:45.028927+0000","last_clean":"2024-04-25T00:31:45.028927+0000","last_became_active":"2024-04-25T00:30:52.789905+0000","last_became_peered":"2024-04-25T00:30:52.789905+0000","last_unstale":"2024-04-25T00:31:45.028927+0000","last_undegraded":"2024-04-25T00:31:45.028927+0000","last_fullsized":"2024-04-25T00:31:45.028927+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:19:03.687318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3,1],"acting":[5,0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"89'2","reported_seq":30,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.711542+0000","last_change":"2024-04-25T00:30:47.516468+0000","last_active":"2024-04-25T00:31:44.711542+0000","last_peered":"2024-04-25T00:31:44.711542+0000","last_clean":"2024-04-25T00:31:44.711542+0000","last_became_active":"2024-04-25T00:30:47.515672+0000","last_became_peered":"2024-04-25T00:30:47.515672+0000","last_unstale":"2024-04-25T00:31:44.711542+0000","last_undegraded":"2024-04-25T00:31:44.711542+0000","last_fullsized":"2024-04-25T00:31:44.711542+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:43:38.525018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":2,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710310+0000","last_change":"2024-04-25T00:30:48.539679+0000","last_active":"2024-04-25T00:31:44.710310+0000","last_peered":"2024-04-25T00:31:44.710310+0000","last_clean":"2024-04-25T00:31:44.710310+0000","last_became_active":"2024-04-25T00:30:48.538065+0000","last_became_peered":"2024-04-25T00:30:48.538065+0000","last_unstale":"2024-04-25T00:31:44.710310+0000","last_undegraded":"2024-04-25T00:31:44.710310+0000","last_fullsized":"2024-04-25T00:31:44.710310+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:05:16.435010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710478+0000","last_change":"2024-04-25T00:30:52.804824+0000","last_active":"2024-04-25T00:31:44.710478+0000","last_peered":"2024-04-25T00:31:44.710478+0000","last_clean":"2024-04-25T00:31:44.710478+0000","last_became_active":"2024-04-25T00:30:52.804322+0000","last_became_peered":"2024-04-25T00:30:52.804322+0000","last_unstale":"2024-04-25T00:31:44.710478+0000","last_undegraded":"2024-04-25T00:31:44.710478+0000","last_fullsized":"2024-04-25T00:31:44.710478+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:38:08.385170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,10,11,9],"acting":[8,10,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":27,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.703679+0000","last_change":"2024-04-25T00:30:47.516916+0000","last_active":"2024-04-25T00:31:44.703679+0000","last_peered":"2024-04-25T00:31:44.703679+0000","last_clean":"2024-04-25T00:31:44.703679+0000","last_became_active":"2024-04-25T00:30:47.516520+0000","last_became_peered":"2024-04-25T00:30:47.516520+0000","last_unstale":"2024-04-25T00:31:44.703679+0000","last_undegraded":"2024-04-25T00:31:44.703679+0000","last_fullsized":"2024-04-25T00:31:44.703679+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:29:19.099764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705654+0000","last_change":"2024-04-25T00:30:48.537584+0000","last_active":"2024-04-25T00:31:44.705654+0000","last_peered":"2024-04-25T00:31:44.705654+0000","last_clean":"2024-04-25T00:31:44.705654+0000","last_became_active":"2024-04-25T00:30:48.537193+0000","last_became_peered":"2024-04-25T00:30:48.537193+0000","last_unstale":"2024-04-25T00:31:44.705654+0000","last_undegraded":"2024-04-25T00:31:44.705654+0000","last_fullsized":"2024-04-25T00:31:44.705654+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:15:22.075323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.027171+0000","last_change":"2024-04-25T00:30:52.784802+0000","last_active":"2024-04-25T00:31:45.027171+0000","last_peered":"2024-04-25T00:31:45.027171+0000","last_clean":"2024-04-25T00:31:45.027171+0000","last_became_active":"2024-04-25T00:30:52.784359+0000","last_became_peered":"2024-04-25T00:30:52.784359+0000","last_unstale":"2024-04-25T00:31:45.027171+0000","last_undegraded":"2024-04-25T00:31:45.027171+0000","last_fullsized":"2024-04-25T00:31:45.027171+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:19:57.723542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1,8],"acting":[6,0,1,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"89'1","reported_seq":28,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710701+0000","last_change":"2024-04-25T00:30:47.525494+0000","last_active":"2024-04-25T00:31:44.710701+0000","last_peered":"2024-04-25T00:31:44.710701+0000","last_clean":"2024-04-25T00:31:44.710701+0000","last_became_active":"2024-04-25T00:30:47.525158+0000","last_became_peered":"2024-04-25T00:30:47.525158+0000","last_unstale":"2024-04-25T00:31:44.710701+0000","last_undegraded":"2024-04-25T00:31:44.710701+0000","last_fullsized":"2024-04-25T00:31:44.710701+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:51:18.543352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705133+0000","last_change":"2024-04-25T00:30:48.540926+0000","last_active":"2024-04-25T00:31:44.705133+0000","last_peered":"2024-04-25T00:31:44.705133+0000","last_clean":"2024-04-25T00:31:44.705133+0000","last_became_active":"2024-04-25T00:30:48.540204+0000","last_became_peered":"2024-04-25T00:30:48.540204+0000","last_unstale":"2024-04-25T00:31:44.705133+0000","last_undegraded":"2024-04-25T00:31:44.705133+0000","last_fullsized":"2024-04-25T00:31:44.705133+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:02:45.530266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.027310+0000","last_change":"2024-04-25T00:30:52.805780+0000","last_active":"2024-04-25T00:31:45.027310+0000","last_peered":"2024-04-25T00:31:45.027310+0000","last_clean":"2024-04-25T00:31:45.027310+0000","last_became_active":"2024-04-25T00:30:52.805222+0000","last_became_peered":"2024-04-25T00:30:52.805222+0000","last_unstale":"2024-04-25T00:31:45.027310+0000","last_undegraded":"2024-04-25T00:31:45.027310+0000","last_fullsized":"2024-04-25T00:31:45.027310+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:32:17.034630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,10,0],"acting":[6,5,10,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.029706+0000","last_change":"2024-04-25T00:30:48.534335+0000","last_active":"2024-04-25T00:31:45.029706+0000","last_peered":"2024-04-25T00:31:45.029706+0000","last_clean":"2024-04-25T00:31:45.029706+0000","last_became_active":"2024-04-25T00:30:48.532670+0000","last_became_peered":"2024-04-25T00:30:48.532670+0000","last_unstale":"2024-04-25T00:31:45.029706+0000","last_undegraded":"2024-04-25T00:31:45.029706+0000","last_fullsized":"2024-04-25T00:31:45.029706+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:26:23.145918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"89'4","reported_seq":105,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:32:27.088840+0000","last_change":"2024-04-25T00:30:47.527695+0000","last_active":"2024-04-25T00:32:27.088840+0000","last_peered":"2024-04-25T00:32:27.088840+0000","last_clean":"2024-04-25T00:32:27.088840+0000","last_became_active":"2024-04-25T00:30:47.527239+0000","last_became_peered":"2024-04-25T00:30:47.527239+0000","last_unstale":"2024-04-25T00:32:27.088840+0000","last_undegraded":"2024-04-25T00:32:27.088840+0000","last_fullsized":"2024-04-25T00:32:27.088840+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:54:30.911284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":74,"num_read_kb":74,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"89'1","reported_seq":19,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710447+0000","last_change":"2024-04-25T00:30:52.781091+0000","last_active":"2024-04-25T00:31:44.710447+0000","last_peered":"2024-04-25T00:31:44.710447+0000","last_clean":"2024-04-25T00:31:44.710447+0000","last_became_active":"2024-04-25T00:30:52.780538+0000","last_became_peered":"2024-04-25T00:30:52.780538+0000","last_unstale":"2024-04-25T00:31:44.710447+0000","last_undegraded":"2024-04-25T00:31:44.710447+0000","last_fullsized":"2024-04-25T00:31:44.710447+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:49:18.141802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,11,9],"acting":[7,5,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.169331+0000","last_change":"2024-04-25T00:30:48.538730+0000","last_active":"2024-04-25T00:31:45.169331+0000","last_peered":"2024-04-25T00:31:45.169331+0000","last_clean":"2024-04-25T00:31:45.169331+0000","last_became_active":"2024-04-25T00:30:48.538336+0000","last_became_peered":"2024-04-25T00:30:48.538336+0000","last_unstale":"2024-04-25T00:31:45.169331+0000","last_undegraded":"2024-04-25T00:31:45.169331+0000","last_fullsized":"2024-04-25T00:31:45.169331+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:44:22.370448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"89'3","reported_seq":32,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707062+0000","last_change":"2024-04-25T00:30:47.520450+0000","last_active":"2024-04-25T00:31:44.707062+0000","last_peered":"2024-04-25T00:31:44.707062+0000","last_clean":"2024-04-25T00:31:44.707062+0000","last_became_active":"2024-04-25T00:30:47.520073+0000","last_became_peered":"2024-04-25T00:30:47.520073+0000","last_unstale":"2024-04-25T00:31:44.707062+0000","last_undegraded":"2024-04-25T00:31:44.707062+0000","last_fullsized":"2024-04-25T00:31:44.707062+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:49:53.570421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":4,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"22'137","reported_seq":361,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.709973+0000","last_change":"2024-04-25T00:29:34.720552+0000","last_active":"2024-04-25T00:31:44.709973+0000","last_peered":"2024-04-25T00:31:44.709973+0000","last_clean":"2024-04-25T00:31:44.709973+0000","last_became_active":"2024-04-25T00:29:34.720041+0000","last_became_peered":"2024-04-25T00:29:34.720041+0000","last_unstale":"2024-04-25T00:31:44.709973+0000","last_undegraded":"2024-04-25T00:31:44.709973+0000","last_fullsized":"2024-04-25T00:31:44.709973+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:26:07.691582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:26:07.691582+0000","last_clean_scrub_stamp":"2024-04-25T00:26:07.691582+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:15:27.995162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.029957+0000","last_change":"2024-04-25T00:30:52.805489+0000","last_active":"2024-04-25T00:31:45.029957+0000","last_peered":"2024-04-25T00:31:45.029957+0000","last_clean":"2024-04-25T00:31:45.029957+0000","last_became_active":"2024-04-25T00:30:52.804758+0000","last_became_peered":"2024-04-25T00:30:52.804758+0000","last_unstale":"2024-04-25T00:31:45.029957+0000","last_undegraded":"2024-04-25T00:31:45.029957+0000","last_fullsized":"2024-04-25T00:31:45.029957+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:17:31.325548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,6,10],"acting":[5,9,6,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.026977+0000","last_change":"2024-04-25T00:30:48.537699+0000","last_active":"2024-04-25T00:31:45.026977+0000","last_peered":"2024-04-25T00:31:45.026977+0000","last_clean":"2024-04-25T00:31:45.026977+0000","last_became_active":"2024-04-25T00:30:48.537291+0000","last_became_peered":"2024-04-25T00:30:48.537291+0000","last_unstale":"2024-04-25T00:31:45.026977+0000","last_undegraded":"2024-04-25T00:31:45.026977+0000","last_fullsized":"2024-04-25T00:31:45.026977+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:25:36.816499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"89'3","reported_seq":30,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710041+0000","last_change":"2024-04-25T00:30:47.524694+0000","last_active":"2024-04-25T00:31:44.710041+0000","last_peered":"2024-04-25T00:31:44.710041+0000","last_clean":"2024-04-25T00:31:44.710041+0000","last_became_active":"2024-04-25T00:30:47.524359+0000","last_became_peered":"2024-04-25T00:30:47.524359+0000","last_unstale":"2024-04-25T00:31:44.710041+0000","last_undegraded":"2024-04-25T00:31:44.710041+0000","last_fullsized":"2024-04-25T00:31:44.710041+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:55:16.677234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.708832+0000","last_change":"2024-04-25T00:30:52.780364+0000","last_active":"2024-04-25T00:31:44.708832+0000","last_peered":"2024-04-25T00:31:44.708832+0000","last_clean":"2024-04-25T00:31:44.708832+0000","last_became_active":"2024-04-25T00:30:52.779904+0000","last_became_peered":"2024-04-25T00:30:52.779904+0000","last_unstale":"2024-04-25T00:31:44.708832+0000","last_undegraded":"2024-04-25T00:31:44.708832+0000","last_fullsized":"2024-04-25T00:31:44.708832+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:11:26.738526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,7,6],"acting":[11,5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.5","version":"89'1","reported_seq":29,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.712031+0000","last_change":"2024-04-25T00:30:47.525205+0000","last_active":"2024-04-25T00:31:44.712031+0000","last_peered":"2024-04-25T00:31:44.712031+0000","last_clean":"2024-04-25T00:31:44.712031+0000","last_became_active":"2024-04-25T00:30:47.517729+0000","last_became_peered":"2024-04-25T00:30:47.517729+0000","last_unstale":"2024-04-25T00:31:44.712031+0000","last_undegraded":"2024-04-25T00:31:44.712031+0000","last_fullsized":"2024-04-25T00:31:44.712031+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:21:14.810119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.027310+0000","last_change":"2024-04-25T00:30:48.533822+0000","last_active":"2024-04-25T00:31:45.027310+0000","last_peered":"2024-04-25T00:31:45.027310+0000","last_clean":"2024-04-25T00:31:45.027310+0000","last_became_active":"2024-04-25T00:30:48.533479+0000","last_became_peered":"2024-04-25T00:30:48.533479+0000","last_unstale":"2024-04-25T00:31:45.027310+0000","last_undegraded":"2024-04-25T00:31:45.027310+0000","last_fullsized":"2024-04-25T00:31:45.027310+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:49:01.028124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.169164+0000","last_change":"2024-04-25T00:30:52.804770+0000","last_active":"2024-04-25T00:31:45.169164+0000","last_peered":"2024-04-25T00:31:45.169164+0000","last_clean":"2024-04-25T00:31:45.169164+0000","last_became_active":"2024-04-25T00:30:52.804242+0000","last_became_peered":"2024-04-25T00:30:52.804242+0000","last_unstale":"2024-04-25T00:31:45.169164+0000","last_undegraded":"2024-04-25T00:31:45.169164+0000","last_fullsized":"2024-04-25T00:31:45.169164+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:34:11.761478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,10,0,7],"acting":[3,10,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"89'8","reported_seq":35,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.703669+0000","last_change":"2024-04-25T00:30:47.521364+0000","last_active":"2024-04-25T00:31:44.703669+0000","last_peered":"2024-04-25T00:31:44.703669+0000","last_clean":"2024-04-25T00:31:44.703669+0000","last_became_active":"2024-04-25T00:30:47.521032+0000","last_became_peered":"2024-04-25T00:30:47.521032+0000","last_unstale":"2024-04-25T00:31:44.703669+0000","last_undegraded":"2024-04-25T00:31:44.703669+0000","last_fullsized":"2024-04-25T00:31:44.703669+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:33:26.156568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,8,3],"acting":[9,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.169845+0000","last_change":"2024-04-25T00:30:48.532942+0000","last_active":"2024-04-25T00:31:45.169845+0000","last_peered":"2024-04-25T00:31:45.169845+0000","last_clean":"2024-04-25T00:31:45.169845+0000","last_became_active":"2024-04-25T00:30:48.532580+0000","last_became_peered":"2024-04-25T00:30:48.532580+0000","last_unstale":"2024-04-25T00:31:45.169845+0000","last_undegraded":"2024-04-25T00:31:45.169845+0000","last_fullsized":"2024-04-25T00:31:45.169845+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:42:22.887920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705899+0000","last_change":"2024-04-25T00:30:52.779757+0000","last_active":"2024-04-25T00:31:44.705899+0000","last_peered":"2024-04-25T00:31:44.705899+0000","last_clean":"2024-04-25T00:31:44.705899+0000","last_became_active":"2024-04-25T00:30:52.779324+0000","last_became_peered":"2024-04-25T00:30:52.779324+0000","last_unstale":"2024-04-25T00:31:44.705899+0000","last_undegraded":"2024-04-25T00:31:44.705899+0000","last_fullsized":"2024-04-25T00:31:44.705899+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:22:39.062000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4,7],"acting":[2,1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"89'4","reported_seq":31,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.706470+0000","last_change":"2024-04-25T00:30:47.518212+0000","last_active":"2024-04-25T00:31:44.706470+0000","last_peered":"2024-04-25T00:31:44.706470+0000","last_clean":"2024-04-25T00:31:44.706470+0000","last_became_active":"2024-04-25T00:30:47.517610+0000","last_became_peered":"2024-04-25T00:30:47.517610+0000","last_unstale":"2024-04-25T00:31:44.706470+0000","last_undegraded":"2024-04-25T00:31:44.706470+0000","last_fullsized":"2024-04-25T00:31:44.706470+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:13:58.904020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":31,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.028411+0000","last_change":"2024-04-25T00:31:45.028411+0000","last_active":"2024-04-25T00:31:45.028411+0000","last_peered":"2024-04-25T00:31:45.028411+0000","last_clean":"2024-04-25T00:31:45.028411+0000","last_became_active":"2024-04-25T00:31:45.028066+0000","last_became_peered":"2024-04-25T00:31:45.028066+0000","last_unstale":"2024-04-25T00:31:45.028411+0000","last_undegraded":"2024-04-25T00:31:45.028411+0000","last_fullsized":"2024-04-25T00:31:45.028411+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:33:46.556862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705862+0000","last_change":"2024-04-25T00:30:52.804915+0000","last_active":"2024-04-25T00:31:44.705862+0000","last_peered":"2024-04-25T00:31:44.705862+0000","last_clean":"2024-04-25T00:31:44.705862+0000","last_became_active":"2024-04-25T00:30:52.804491+0000","last_became_peered":"2024-04-25T00:30:52.804491+0000","last_unstale":"2024-04-25T00:31:44.705862+0000","last_undegraded":"2024-04-25T00:31:44.705862+0000","last_fullsized":"2024-04-25T00:31:44.705862+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:51:50.219556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,10,9],"acting":[4,5,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.8","version":"89'2","reported_seq":29,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.704458+0000","last_change":"2024-04-25T00:30:47.521909+0000","last_active":"2024-04-25T00:31:44.704458+0000","last_peered":"2024-04-25T00:31:44.704458+0000","last_clean":"2024-04-25T00:31:44.704458+0000","last_became_active":"2024-04-25T00:30:47.521551+0000","last_became_peered":"2024-04-25T00:30:47.521551+0000","last_unstale":"2024-04-25T00:31:44.704458+0000","last_undegraded":"2024-04-25T00:31:44.704458+0000","last_fullsized":"2024-04-25T00:31:44.704458+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:51:16.368609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.709641+0000","last_change":"2024-04-25T00:30:48.530514+0000","last_active":"2024-04-25T00:31:44.709641+0000","last_peered":"2024-04-25T00:31:44.709641+0000","last_clean":"2024-04-25T00:31:44.709641+0000","last_became_active":"2024-04-25T00:30:48.530129+0000","last_became_peered":"2024-04-25T00:30:48.530129+0000","last_unstale":"2024-04-25T00:31:44.709641+0000","last_undegraded":"2024-04-25T00:31:44.709641+0000","last_fullsized":"2024-04-25T00:31:44.709641+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:51:25.623186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.703839+0000","last_change":"2024-04-25T00:30:52.785767+0000","last_active":"2024-04-25T00:31:44.703839+0000","last_peered":"2024-04-25T00:31:44.703839+0000","last_clean":"2024-04-25T00:31:44.703839+0000","last_became_active":"2024-04-25T00:30:52.785301+0000","last_became_peered":"2024-04-25T00:30:52.785301+0000","last_unstale":"2024-04-25T00:31:44.703839+0000","last_undegraded":"2024-04-25T00:31:44.703839+0000","last_fullsized":"2024-04-25T00:31:44.703839+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:10:14.071733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,2,5],"acting":[9,3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.9","version":"89'3","reported_seq":30,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.169284+0000","last_change":"2024-04-25T00:30:47.526364+0000","last_active":"2024-04-25T00:31:45.169284+0000","last_peered":"2024-04-25T00:31:45.169284+0000","last_clean":"2024-04-25T00:31:45.169284+0000","last_became_active":"2024-04-25T00:30:47.526009+0000","last_became_peered":"2024-04-25T00:30:47.526009+0000","last_unstale":"2024-04-25T00:31:45.169284+0000","last_undegraded":"2024-04-25T00:31:45.169284+0000","last_fullsized":"2024-04-25T00:31:45.169284+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:17:38.471878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.709530+0000","last_change":"2024-04-25T00:30:48.532283+0000","last_active":"2024-04-25T00:31:44.709530+0000","last_peered":"2024-04-25T00:31:44.709530+0000","last_clean":"2024-04-25T00:31:44.709530+0000","last_became_active":"2024-04-25T00:30:48.531935+0000","last_became_peered":"2024-04-25T00:30:48.531935+0000","last_unstale":"2024-04-25T00:31:44.709530+0000","last_undegraded":"2024-04-25T00:31:44.709530+0000","last_fullsized":"2024-04-25T00:31:44.709530+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:01:15.965611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.c","version":"89'1","reported_seq":30,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.717218+0000","last_change":"2024-04-25T00:31:44.717218+0000","last_active":"2024-04-25T00:31:44.717218+0000","last_peered":"2024-04-25T00:31:44.717218+0000","last_clean":"2024-04-25T00:31:44.717218+0000","last_became_active":"2024-04-25T00:31:44.712770+0000","last_became_peered":"2024-04-25T00:31:44.712770+0000","last_unstale":"2024-04-25T00:31:44.717218+0000","last_undegraded":"2024-04-25T00:31:44.717218+0000","last_fullsized":"2024-04-25T00:31:44.717218+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:06:58.496133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":117,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,7,8],"acting":[4,10,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.a","version":"89'1","reported_seq":28,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.706038+0000","last_change":"2024-04-25T00:30:47.520608+0000","last_active":"2024-04-25T00:31:44.706038+0000","last_peered":"2024-04-25T00:31:44.706038+0000","last_clean":"2024-04-25T00:31:44.706038+0000","last_became_active":"2024-04-25T00:30:47.520063+0000","last_became_peered":"2024-04-25T00:30:47.520063+0000","last_unstale":"2024-04-25T00:31:44.706038+0000","last_undegraded":"2024-04-25T00:31:44.706038+0000","last_fullsized":"2024-04-25T00:31:44.706038+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:58:47.823712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.169717+0000","last_change":"2024-04-25T00:30:48.535073+0000","last_active":"2024-04-25T00:31:45.169717+0000","last_peered":"2024-04-25T00:31:45.169717+0000","last_clean":"2024-04-25T00:31:45.169717+0000","last_became_active":"2024-04-25T00:30:48.534247+0000","last_became_peered":"2024-04-25T00:30:48.534247+0000","last_unstale":"2024-04-25T00:31:45.169717+0000","last_undegraded":"2024-04-25T00:31:45.169717+0000","last_fullsized":"2024-04-25T00:31:45.169717+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:46:18.613705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705968+0000","last_change":"2024-04-25T00:30:52.804519+0000","last_active":"2024-04-25T00:31:44.705968+0000","last_peered":"2024-04-25T00:31:44.705968+0000","last_clean":"2024-04-25T00:31:44.705968+0000","last_became_active":"2024-04-25T00:30:52.804074+0000","last_became_peered":"2024-04-25T00:30:52.804074+0000","last_unstale":"2024-04-25T00:31:44.705968+0000","last_undegraded":"2024-04-25T00:31:44.705968+0000","last_fullsized":"2024-04-25T00:31:44.705968+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:48:45.058873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,10,3],"acting":[4,2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.b","version":"89'4","reported_seq":32,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710833+0000","last_change":"2024-04-25T00:30:47.521054+0000","last_active":"2024-04-25T00:31:44.710833+0000","last_peered":"2024-04-25T00:31:44.710833+0000","last_clean":"2024-04-25T00:31:44.710833+0000","last_became_active":"2024-04-25T00:30:47.520716+0000","last_became_peered":"2024-04-25T00:30:47.520716+0000","last_unstale":"2024-04-25T00:31:44.710833+0000","last_undegraded":"2024-04-25T00:31:44.710833+0000","last_fullsized":"2024-04-25T00:31:44.710833+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:05:16.471589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710832+0000","last_change":"2024-04-25T00:30:48.537152+0000","last_active":"2024-04-25T00:31:44.710832+0000","last_peered":"2024-04-25T00:31:44.710832+0000","last_clean":"2024-04-25T00:31:44.710832+0000","last_became_active":"2024-04-25T00:30:48.536256+0000","last_became_peered":"2024-04-25T00:30:48.536256+0000","last_unstale":"2024-04-25T00:31:44.710832+0000","last_undegraded":"2024-04-25T00:31:44.710832+0000","last_fullsized":"2024-04-25T00:31:44.710832+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:57:53.870212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.704501+0000","last_change":"2024-04-25T00:30:52.781022+0000","last_active":"2024-04-25T00:31:44.704501+0000","last_peered":"2024-04-25T00:31:44.704501+0000","last_clean":"2024-04-25T00:31:44.704501+0000","last_became_active":"2024-04-25T00:30:52.780595+0000","last_became_peered":"2024-04-25T00:30:52.780595+0000","last_unstale":"2024-04-25T00:31:44.704501+0000","last_undegraded":"2024-04-25T00:31:44.704501+0000","last_fullsized":"2024-04-25T00:31:44.704501+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:27:02.580870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6,7],"acting":[1,5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.c","version":"89'3","reported_seq":31,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.706842+0000","last_change":"2024-04-25T00:30:47.519118+0000","last_active":"2024-04-25T00:31:44.706842+0000","last_peered":"2024-04-25T00:31:44.706842+0000","last_clean":"2024-04-25T00:31:44.706842+0000","last_became_active":"2024-04-25T00:30:47.518639+0000","last_became_peered":"2024-04-25T00:30:47.518639+0000","last_unstale":"2024-04-25T00:31:44.706842+0000","last_undegraded":"2024-04-25T00:31:44.706842+0000","last_fullsized":"2024-04-25T00:31:44.706842+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:32:29.747903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710945+0000","last_change":"2024-04-25T00:30:48.542073+0000","last_active":"2024-04-25T00:31:44.710945+0000","last_peered":"2024-04-25T00:31:44.710945+0000","last_clean":"2024-04-25T00:31:44.710945+0000","last_became_active":"2024-04-25T00:30:48.541722+0000","last_became_peered":"2024-04-25T00:30:48.541722+0000","last_unstale":"2024-04-25T00:31:44.710945+0000","last_undegraded":"2024-04-25T00:31:44.710945+0000","last_fullsized":"2024-04-25T00:31:44.710945+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:34:03.397539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707011+0000","last_change":"2024-04-25T00:30:52.790234+0000","last_active":"2024-04-25T00:31:44.707011+0000","last_peered":"2024-04-25T00:31:44.707011+0000","last_clean":"2024-04-25T00:31:44.707011+0000","last_became_active":"2024-04-25T00:30:52.789788+0000","last_became_peered":"2024-04-25T00:30:52.789788+0000","last_unstale":"2024-04-25T00:31:44.707011+0000","last_undegraded":"2024-04-25T00:31:44.707011+0000","last_fullsized":"2024-04-25T00:31:44.707011+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:44:43.399771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,9,1,4],"acting":[0,9,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.d","version":"84'1","reported_seq":29,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.029778+0000","last_change":"2024-04-25T00:30:47.523597+0000","last_active":"2024-04-25T00:31:45.029778+0000","last_peered":"2024-04-25T00:31:45.029778+0000","last_clean":"2024-04-25T00:31:45.029778+0000","last_became_active":"2024-04-25T00:30:47.523225+0000","last_became_peered":"2024-04-25T00:30:47.523225+0000","last_unstale":"2024-04-25T00:31:45.029778+0000","last_undegraded":"2024-04-25T00:31:45.029778+0000","last_fullsized":"2024-04-25T00:31:45.029778+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:51:12.341015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.710527+0000","last_change":"2024-04-25T00:30:48.541417+0000","last_active":"2024-04-25T00:31:44.710527+0000","last_peered":"2024-04-25T00:31:44.710527+0000","last_clean":"2024-04-25T00:31:44.710527+0000","last_became_active":"2024-04-25T00:30:48.541015+0000","last_became_peered":"2024-04-25T00:30:48.541015+0000","last_unstale":"2024-04-25T00:31:44.710527+0000","last_undegraded":"2024-04-25T00:31:44.710527+0000","last_fullsized":"2024-04-25T00:31:44.710527+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:21:35.299823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":18,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.029015+0000","last_change":"2024-04-25T00:30:52.782230+0000","last_active":"2024-04-25T00:31:45.029015+0000","last_peered":"2024-04-25T00:31:45.029015+0000","last_clean":"2024-04-25T00:31:45.029015+0000","last_became_active":"2024-04-25T00:30:52.781708+0000","last_became_peered":"2024-04-25T00:30:52.781708+0000","last_unstale":"2024-04-25T00:31:45.029015+0000","last_undegraded":"2024-04-25T00:31:45.029015+0000","last_fullsized":"2024-04-25T00:31:45.029015+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:58:53.260985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7,1],"acting":[5,2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.e","version":"84'2","reported_seq":29,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.027519+0000","last_change":"2024-04-25T00:30:47.522400+0000","last_active":"2024-04-25T00:31:45.027519+0000","last_peered":"2024-04-25T00:31:45.027519+0000","last_clean":"2024-04-25T00:31:45.027519+0000","last_became_active":"2024-04-25T00:30:47.522032+0000","last_became_peered":"2024-04-25T00:30:47.522032+0000","last_unstale":"2024-04-25T00:31:45.027519+0000","last_undegraded":"2024-04-25T00:31:45.027519+0000","last_fullsized":"2024-04-25T00:31:45.027519+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:24:17.749437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.711092+0000","last_change":"2024-04-25T00:30:48.532730+0000","last_active":"2024-04-25T00:31:44.711092+0000","last_peered":"2024-04-25T00:31:44.711092+0000","last_clean":"2024-04-25T00:31:44.711092+0000","last_became_active":"2024-04-25T00:30:48.532129+0000","last_became_peered":"2024-04-25T00:30:48.532129+0000","last_unstale":"2024-04-25T00:31:44.711092+0000","last_undegraded":"2024-04-25T00:31:44.711092+0000","last_fullsized":"2024-04-25T00:31:44.711092+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:36:30.115125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":31,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.712173+0000","last_change":"2024-04-25T00:31:44.712173+0000","last_active":"2024-04-25T00:31:44.712173+0000","last_peered":"2024-04-25T00:31:44.712173+0000","last_clean":"2024-04-25T00:31:44.712173+0000","last_became_active":"2024-04-25T00:31:44.711738+0000","last_became_peered":"2024-04-25T00:31:44.711738+0000","last_unstale":"2024-04-25T00:31:44.712173+0000","last_undegraded":"2024-04-25T00:31:44.712173+0000","last_fullsized":"2024-04-25T00:31:44.712173+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:06:59.466566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,9],"acting":[11,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.706223+0000","last_change":"2024-04-25T00:30:48.540676+0000","last_active":"2024-04-25T00:31:44.706223+0000","last_peered":"2024-04-25T00:31:44.706223+0000","last_clean":"2024-04-25T00:31:44.706223+0000","last_became_active":"2024-04-25T00:30:48.540258+0000","last_became_peered":"2024-04-25T00:30:48.540258+0000","last_unstale":"2024-04-25T00:31:44.706223+0000","last_undegraded":"2024-04-25T00:31:44.706223+0000","last_fullsized":"2024-04-25T00:31:44.706223+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:10:29.930897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.705989+0000","last_change":"2024-04-25T00:30:48.542191+0000","last_active":"2024-04-25T00:31:44.705989+0000","last_peered":"2024-04-25T00:31:44.705989+0000","last_clean":"2024-04-25T00:31:44.705989+0000","last_became_active":"2024-04-25T00:30:48.540433+0000","last_became_peered":"2024-04-25T00:30:48.540433+0000","last_unstale":"2024-04-25T00:31:44.705989+0000","last_undegraded":"2024-04-25T00:31:44.705989+0000","last_fullsized":"2024-04-25T00:31:44.705989+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:21:55.694555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":31,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.028445+0000","last_change":"2024-04-25T00:31:45.028445+0000","last_active":"2024-04-25T00:31:45.028445+0000","last_peered":"2024-04-25T00:31:45.028445+0000","last_clean":"2024-04-25T00:31:45.028445+0000","last_became_active":"2024-04-25T00:31:45.028048+0000","last_became_peered":"2024-04-25T00:31:45.028048+0000","last_unstale":"2024-04-25T00:31:45.028445+0000","last_undegraded":"2024-04-25T00:31:45.028445+0000","last_fullsized":"2024-04-25T00:31:45.028445+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:04:02.714111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,8],"acting":[0,6,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.027430+0000","last_change":"2024-04-25T00:30:48.527892+0000","last_active":"2024-04-25T00:31:45.027430+0000","last_peered":"2024-04-25T00:31:45.027430+0000","last_clean":"2024-04-25T00:31:45.027430+0000","last_became_active":"2024-04-25T00:30:48.527522+0000","last_became_peered":"2024-04-25T00:30:48.527522+0000","last_unstale":"2024-04-25T00:31:45.027430+0000","last_undegraded":"2024-04-25T00:31:45.027430+0000","last_fullsized":"2024-04-25T00:31:45.027430+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:21:22.779470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.711690+0000","last_change":"2024-04-25T00:30:48.535066+0000","last_active":"2024-04-25T00:31:44.711690+0000","last_peered":"2024-04-25T00:31:44.711690+0000","last_clean":"2024-04-25T00:31:44.711690+0000","last_became_active":"2024-04-25T00:30:48.534710+0000","last_became_peered":"2024-04-25T00:30:48.534710+0000","last_unstale":"2024-04-25T00:31:44.711690+0000","last_undegraded":"2024-04-25T00:31:44.711690+0000","last_fullsized":"2024-04-25T00:31:44.711690+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:57:18.582569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.707210+0000","last_change":"2024-04-25T00:30:48.541921+0000","last_active":"2024-04-25T00:31:44.707210+0000","last_peered":"2024-04-25T00:31:44.707210+0000","last_clean":"2024-04-25T00:31:44.707210+0000","last_became_active":"2024-04-25T00:30:48.541348+0000","last_became_peered":"2024-04-25T00:30:48.541348+0000","last_unstale":"2024-04-25T00:31:44.707210+0000","last_undegraded":"2024-04-25T00:31:44.707210+0000","last_fullsized":"2024-04-25T00:31:44.707210+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:07:00.001504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.169960+0000","last_change":"2024-04-25T00:30:48.536902+0000","last_active":"2024-04-25T00:31:45.169960+0000","last_peered":"2024-04-25T00:31:45.169960+0000","last_clean":"2024-04-25T00:31:45.169960+0000","last_became_active":"2024-04-25T00:30:48.536008+0000","last_became_peered":"2024-04-25T00:30:48.536008+0000","last_unstale":"2024-04-25T00:31:45.169960+0000","last_undegraded":"2024-04-25T00:31:45.169960+0000","last_fullsized":"2024-04-25T00:31:45.169960+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:19:15.050142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.029352+0000","last_change":"2024-04-25T00:30:48.533104+0000","last_active":"2024-04-25T00:31:45.029352+0000","last_peered":"2024-04-25T00:31:45.029352+0000","last_clean":"2024-04-25T00:31:45.029352+0000","last_became_active":"2024-04-25T00:30:48.532124+0000","last_became_peered":"2024-04-25T00:30:48.532124+0000","last_unstale":"2024-04-25T00:31:45.029352+0000","last_undegraded":"2024-04-25T00:31:45.029352+0000","last_fullsized":"2024-04-25T00:31:45.029352+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:54:57.391433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.169213+0000","last_change":"2024-04-25T00:30:48.542074+0000","last_active":"2024-04-25T00:31:45.169213+0000","last_peered":"2024-04-25T00:31:45.169213+0000","last_clean":"2024-04-25T00:31:45.169213+0000","last_became_active":"2024-04-25T00:30:48.538752+0000","last_became_peered":"2024-04-25T00:30:48.538752+0000","last_unstale":"2024-04-25T00:31:45.169213+0000","last_undegraded":"2024-04-25T00:31:45.169213+0000","last_fullsized":"2024-04-25T00:31:45.169213+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:55:30.613616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.704798+0000","last_change":"2024-04-25T00:30:48.534158+0000","last_active":"2024-04-25T00:31:44.704798+0000","last_peered":"2024-04-25T00:31:44.704798+0000","last_clean":"2024-04-25T00:31:44.704798+0000","last_became_active":"2024-04-25T00:30:48.533821+0000","last_became_peered":"2024-04-25T00:30:48.533821+0000","last_unstale":"2024-04-25T00:31:44.704798+0000","last_undegraded":"2024-04-25T00:31:44.704798+0000","last_fullsized":"2024-04-25T00:31:44.704798+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:05:19.327608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:45.027512+0000","last_change":"2024-04-25T00:30:48.531718+0000","last_active":"2024-04-25T00:31:45.027512+0000","last_peered":"2024-04-25T00:31:45.027512+0000","last_clean":"2024-04-25T00:31:45.027512+0000","last_became_active":"2024-04-25T00:30:48.531291+0000","last_became_peered":"2024-04-25T00:30:48.531291+0000","last_unstale":"2024-04-25T00:31:45.027512+0000","last_undegraded":"2024-04-25T00:31:45.027512+0000","last_fullsized":"2024-04-25T00:31:45.027512+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:56:49.032937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":31,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-04-25T00:31:44.713036+0000","last_change":"2024-04-25T00:31:44.713036+0000","last_active":"2024-04-25T00:31:44.713036+0000","last_peered":"2024-04-25T00:31:44.713036+0000","last_clean":"2024-04-25T00:31:44.713036+0000","last_became_active":"2024-04-25T00:31:44.712651+0000","last_became_peered":"2024-04-25T00:31:44.712651+0000","last_unstale":"2024-04-25T00:31:44.713036+0000","last_undegraded":"2024-04-25T00:31:44.713036+0000","last_fullsized":"2024-04-25T00:31:44.713036+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:05:04.811510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,9,8],"acting":[11,9,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":234,"num_objects":2,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":117,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":32768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2,"ondisk_log_size":2,"up":128,"acting":128,"num_store_stats":12},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":65776,"num_objects":60,"num_object_clones":0,"num_object_copies":180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":60,"num_whiteouts":0,"num_read":487,"num_read_kb":311,"num_write":159,"num_write_kb":113,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":417792,"data_stored":197328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138,"ondisk_log_size":138,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":80,"seq":343597383714,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32176,"kb_used_data":1000,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712976,"statfs":{"total":95995035648,"available":95962087424,"internally_reserved":0,"allocated":1024000,"data_stored":500815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1596,"internal_metadata":31914436},"hb_peers":[0,1,2,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":10,"up_from":72,"seq":309237645352,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28696,"kb_used_data":1676,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716456,"statfs":{"total":95995035648,"available":95965650944,"internally_reserved":0,"allocated":1716224,"data_stored":1154345,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1600,"internal_metadata":27654592},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":9,"up_from":67,"seq":287762808877,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36356,"kb_used_data":1016,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708796,"statfs":{"total":95995035648,"available":95957807104,"internally_reserved":0,"allocated":1040384,"data_stored":501051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1600,"internal_metadata":36174272},"hb_peers":[0,1,2,3,4,5,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":8,"up_from":59,"seq":253403070513,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32192,"kb_used_data":1016,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712960,"statfs":{"total":95995035648,"available":95962071040,"internally_reserved":0,"allocated":1040384,"data_stored":500971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":7,"up_from":51,"seq":219043332150,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32780,"kb_used_data":1596,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712372,"statfs":{"total":95995035648,"available":95961468928,"internally_reserved":0,"allocated":1634304,"data_stored":1096880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495676,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36336,"kb_used_data":996,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708816,"statfs":{"total":95995035648,"available":95957827584,"internally_reserved":0,"allocated":1019904,"data_stored":496227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":36174278},"hb_peers":[0,1,2,3,4,5,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574932,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28080,"kb_used_data":1060,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717072,"statfs":{"total":95995035648,"available":95966281728,"internally_reserved":0,"allocated":1085440,"data_stored":557912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738458,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28044,"kb_used_data":1020,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717108,"statfs":{"total":95995035648,"available":95966318592,"internally_reserved":0,"allocated":1044480,"data_stored":497615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":18,"seq":77309411408,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28108,"kb_used_data":1084,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717044,"statfs":{"total":95995035648,"available":95966253056,"internally_reserved":0,"allocated":1110016,"data_stored":559262,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":25,"seq":107374182475,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36348,"kb_used_data":1008,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708804,"statfs":{"total":95995035648,"available":95957815296,"internally_reserved":0,"allocated":1032192,"data_stored":496421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":36174279},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":33,"seq":141733920839,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28032,"kb_used_data":1008,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717120,"statfs":{"total":95995035648,"available":95966330880,"internally_reserved":0,"allocated":1032192,"data_stored":501268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757314,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28628,"kb_used_data":1604,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716524,"statfs":{"total":95995035648,"available":95965720576,"internally_reserved":0,"allocated":1642496,"data_stored":1092077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":69632,"data_stored":61775,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":94208,"data_stored":63125,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2183,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":63744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":8192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-25T00:32:35.065 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-25T00:32:35.065 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-25T00:32:35.065 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-25T00:32:35.065 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-25T00:32:35.318 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:32:35.734 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:35 smithi032 ceph-mon[29357]: from='client.15147 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:32:35.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:35 smithi121 ceph-mon[32231]: from='client.15147 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:32:35.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:35 smithi130 ceph-mon[32525]: from='client.15147 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:32:36.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:36 smithi032 ceph-mon[29357]: pgmap v347: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-25T00:32:36.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:36 smithi121 ceph-mon[32231]: pgmap v347: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-25T00:32:36.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:36 smithi130 ceph-mon[32525]: pgmap v347: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-25T00:32:37.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.153+0000 7f9d3ab6b700 1 -- 172.21.15.32:0/723934118 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9d34074460 msgr2=0x7f9d34074840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:37.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.153+0000 7f9d3ab6b700 1 --2- 172.21.15.32:0/723934118 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9d34074460 0x7f9d34074840 secure :-1 s=READY pgs=344 cs=0 l=1 rev1=1 crypto rx=0x7f9d24009990 tx=0x7f9d24009ca0 comp rx=0 tx=0).stop 2024-04-25T00:32:37.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.153+0000 7f9d3ab6b700 1 -- 172.21.15.32:0/723934118 shutdown_connections 2024-04-25T00:32:37.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.153+0000 7f9d3ab6b700 1 --2- 172.21.15.32:0/723934118 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9d3410ec60 0x7f9d34111050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:37.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.153+0000 7f9d3ab6b700 1 --2- 172.21.15.32:0/723934118 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9d34074460 0x7f9d34074840 unknown :-1 s=CLOSED pgs=344 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:37.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.153+0000 7f9d3ab6b700 1 --2- 172.21.15.32:0/723934118 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9d34074d80 0x7f9d3410e720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:37.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.153+0000 7f9d3ab6b700 1 -- 172.21.15.32:0/723934118 >> 172.21.15.32:0/723934118 conn(0x7f9d340793a0 msgr2=0x7f9d340797b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:37.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.153+0000 7f9d3ab6b700 1 -- 172.21.15.32:0/723934118 shutdown_connections 2024-04-25T00:32:37.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.153+0000 7f9d3ab6b700 1 -- 172.21.15.32:0/723934118 wait complete. 2024-04-25T00:32:37.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.154+0000 7f9d3ab6b700 1 Processor -- start 2024-04-25T00:32:37.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.154+0000 7f9d3ab6b700 1 -- start start 2024-04-25T00:32:37.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.154+0000 7f9d3ab6b700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9d34074460 0x7f9d341a0880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:37.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.154+0000 7f9d3ab6b700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9d34074d80 0x7f9d341a0dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:37.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.154+0000 7f9d3ab6b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9d3410ec60 0x7f9d3419aa40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:37.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.154+0000 7f9d3ab6b700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f9d34114780 con 0x7f9d34074d80 2024-04-25T00:32:37.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.154+0000 7f9d3ab6b700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f9d34114600 con 0x7f9d34074460 2024-04-25T00:32:37.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.154+0000 7f9d3ab6b700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f9d34114900 con 0x7f9d3410ec60 2024-04-25T00:32:37.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.154+0000 7f9d33fff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9d34074d80 0x7f9d341a0dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:37.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.154+0000 7f9d33fff700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9d34074d80 0x7f9d341a0dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40746/0 (socket says 172.21.15.32:40746) 2024-04-25T00:32:37.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.154+0000 7f9d33fff700 1 -- 172.21.15.32:0/3701750454 learned_addr learned my addr 172.21.15.32:0/3701750454 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:37.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.154+0000 7f9d39108700 1 --2- 172.21.15.32:0/3701750454 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9d3410ec60 0x7f9d3419aa40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:37.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.154+0000 7f9d38907700 1 --2- 172.21.15.32:0/3701750454 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9d34074460 0x7f9d341a0880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:37.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.154+0000 7f9d33fff700 1 -- 172.21.15.32:0/3701750454 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9d3410ec60 msgr2=0x7f9d3419aa40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:37.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.154+0000 7f9d33fff700 1 --2- 172.21.15.32:0/3701750454 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9d3410ec60 0x7f9d3419aa40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:37.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.154+0000 7f9d33fff700 1 -- 172.21.15.32:0/3701750454 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9d34074460 msgr2=0x7f9d341a0880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:37.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.154+0000 7f9d33fff700 1 --2- 172.21.15.32:0/3701750454 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9d34074460 0x7f9d341a0880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:37.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.154+0000 7f9d33fff700 1 -- 172.21.15.32:0/3701750454 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9d3419b1a0 con 0x7f9d34074d80 2024-04-25T00:32:37.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.155+0000 7f9d33fff700 1 --2- 172.21.15.32:0/3701750454 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9d34074d80 0x7f9d341a0dc0 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7f9d2800d950 tx=0x7f9d2800dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:37.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.155+0000 7f9d31ffb700 1 -- 172.21.15.32:0/3701750454 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9d28016070 con 0x7f9d34074d80 2024-04-25T00:32:37.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.155+0000 7f9d3ab6b700 1 -- 172.21.15.32:0/3701750454 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9d3419b490 con 0x7f9d34074d80 2024-04-25T00:32:37.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.155+0000 7f9d31ffb700 1 -- 172.21.15.32:0/3701750454 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9d2800fd70 con 0x7f9d34074d80 2024-04-25T00:32:37.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.155+0000 7f9d31ffb700 1 -- 172.21.15.32:0/3701750454 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9d2800e610 con 0x7f9d34074d80 2024-04-25T00:32:37.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.155+0000 7f9d3ab6b700 1 -- 172.21.15.32:0/3701750454 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9d341a7750 con 0x7f9d34074d80 2024-04-25T00:32:37.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.156+0000 7f9d3ab6b700 1 -- 172.21.15.32:0/3701750454 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9d34068f20 con 0x7f9d34074d80 2024-04-25T00:32:37.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.157+0000 7f9d31ffb700 1 -- 172.21.15.32:0/3701750454 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f9d28019070 con 0x7f9d34074d80 2024-04-25T00:32:37.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.157+0000 7f9d31ffb700 1 --2- 172.21.15.32:0/3701750454 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f9d1c0a2cb0 0x7f9d1c0a5170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:37.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.158+0000 7f9d31ffb700 1 -- 172.21.15.32:0/3701750454 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f9d280cefe0 con 0x7f9d34074d80 2024-04-25T00:32:37.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.158+0000 7f9d38907700 1 --2- 172.21.15.32:0/3701750454 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f9d1c0a2cb0 0x7f9d1c0a5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:37.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.158+0000 7f9d38907700 1 --2- 172.21.15.32:0/3701750454 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f9d1c0a2cb0 0x7f9d1c0a5170 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f9d2400b5c0 tx=0x7f9d24005960 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:37.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.162+0000 7f9d31ffb700 1 -- 172.21.15.32:0/3701750454 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f9d2809ba80 con 0x7f9d34074d80 2024-04-25T00:32:37.420 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.418+0000 7f9d3ab6b700 1 -- 172.21.15.32:0/3701750454 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "health", "format": "json"} v 0) -- 0x7f9d341a7950 con 0x7f9d34074d80 2024-04-25T00:32:37.420 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.418+0000 7f9d31ffb700 1 -- 172.21.15.32:0/3701750454 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "health", "format": "json"}]=0 v0) ==== 72+0+46 (secure 0 0 0) 0x7f9d2809b380 con 0x7f9d34074d80 2024-04-25T00:32:37.420 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:32:37.420 INFO:teuthology.orchestra.run.smithi032.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-25T00:32:37.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.421+0000 7f9d3ab6b700 1 -- 172.21.15.32:0/3701750454 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f9d1c0a2cb0 msgr2=0x7f9d1c0a5170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:37.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.421+0000 7f9d3ab6b700 1 --2- 172.21.15.32:0/3701750454 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f9d1c0a2cb0 0x7f9d1c0a5170 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f9d2400b5c0 tx=0x7f9d24005960 comp rx=0 tx=0).stop 2024-04-25T00:32:37.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.421+0000 7f9d3ab6b700 1 -- 172.21.15.32:0/3701750454 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9d34074d80 msgr2=0x7f9d341a0dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:37.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.421+0000 7f9d3ab6b700 1 --2- 172.21.15.32:0/3701750454 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9d34074d80 0x7f9d341a0dc0 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7f9d2800d950 tx=0x7f9d2800dc60 comp rx=0 tx=0).stop 2024-04-25T00:32:37.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.421+0000 7f9d3ab6b700 1 -- 172.21.15.32:0/3701750454 shutdown_connections 2024-04-25T00:32:37.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.421+0000 7f9d3ab6b700 1 --2- 172.21.15.32:0/3701750454 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f9d1c0a2cb0 0x7f9d1c0a5170 unknown :-1 s=CLOSED pgs=221 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:37.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.421+0000 7f9d3ab6b700 1 --2- 172.21.15.32:0/3701750454 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9d3410ec60 0x7f9d3419aa40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:37.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.421+0000 7f9d3ab6b700 1 --2- 172.21.15.32:0/3701750454 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9d34074460 0x7f9d341a0880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:37.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.421+0000 7f9d3ab6b700 1 --2- 172.21.15.32:0/3701750454 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9d34074d80 0x7f9d341a0dc0 unknown :-1 s=CLOSED pgs=345 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:37.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.421+0000 7f9d3ab6b700 1 -- 172.21.15.32:0/3701750454 >> 172.21.15.32:0/3701750454 conn(0x7f9d340793a0 msgr2=0x7f9d340ff430 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:37.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.421+0000 7f9d3ab6b700 1 -- 172.21.15.32:0/3701750454 shutdown_connections 2024-04-25T00:32:37.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:37.421+0000 7f9d3ab6b700 1 -- 172.21.15.32:0/3701750454 wait complete. 2024-04-25T00:32:37.719 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:37 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3701750454' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-25T00:32:37.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:37 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3701750454' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-25T00:32:37.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:37 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3701750454' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-25T00:32:38.227 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-25T00:32:38.228 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-25T00:32:38.335 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.332+0000 7f5554ee1640 1 -- 172.21.15.32:0/1461054377 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5550100680 msgr2=0x7f5550100aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:38.336 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.332+0000 7f5554ee1640 1 --2- 172.21.15.32:0/1461054377 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5550100680 0x7f5550100aa0 secure :-1 s=READY pgs=346 cs=0 l=1 rev1=1 crypto rx=0x7f5540009900 tx=0x7f554002a410 comp rx=0 tx=0).stop 2024-04-25T00:32:38.336 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.332+0000 7f5554ee1640 1 -- 172.21.15.32:0/1461054377 shutdown_connections 2024-04-25T00:32:38.336 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.332+0000 7f5554ee1640 1 --2- 172.21.15.32:0/1461054377 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5550102230 0x7f5550108ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.336 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.332+0000 7f5554ee1640 1 --2- 172.21.15.32:0/1461054377 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5550101870 0x7f5550101cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.336 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.332+0000 7f5554ee1640 1 --2- 172.21.15.32:0/1461054377 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5550100680 0x7f5550100aa0 unknown :-1 s=CLOSED pgs=346 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.336 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.332+0000 7f5554ee1640 1 -- 172.21.15.32:0/1461054377 >> 172.21.15.32:0/1461054377 conn(0x7f55500fbbf0 msgr2=0x7f55500fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:38.336 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.332+0000 7f5554ee1640 1 -- 172.21.15.32:0/1461054377 shutdown_connections 2024-04-25T00:32:38.336 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.332+0000 7f5554ee1640 1 -- 172.21.15.32:0/1461054377 wait complete. 2024-04-25T00:32:38.336 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.333+0000 7f5554ee1640 1 Processor -- start 2024-04-25T00:32:38.336 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.333+0000 7f5554ee1640 1 -- start start 2024-04-25T00:32:38.336 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.333+0000 7f5554ee1640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5550100680 0x7f5550199d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:38.336 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.333+0000 7f5554ee1640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5550101870 0x7f555019a280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:38.337 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.333+0000 7f5554ee1640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5550102230 0x7f55501a12b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:38.337 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.333+0000 7f5554ee1640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f555010d460 con 0x7f5550101870 2024-04-25T00:32:38.337 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.333+0000 7f5554ee1640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f555010d2e0 con 0x7f5550102230 2024-04-25T00:32:38.337 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.333+0000 7f5554ee1640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f555010d5e0 con 0x7f5550100680 2024-04-25T00:32:38.337 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.333+0000 7f554dd9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5550101870 0x7f555019a280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:38.337 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.333+0000 7f554dd9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5550101870 0x7f555019a280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40762/0 (socket says 172.21.15.32:40762) 2024-04-25T00:32:38.337 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.333+0000 7f554dd9b640 1 -- 172.21.15.32:0/2070204227 learned_addr learned my addr 172.21.15.32:0/2070204227 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:38.337 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.334+0000 7f554e59c640 1 --2- 172.21.15.32:0/2070204227 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5550100680 0x7f5550199d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:38.337 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.334+0000 7f554dd9b640 1 -- 172.21.15.32:0/2070204227 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5550100680 msgr2=0x7f5550199d40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:38.337 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.334+0000 7f554ed9d640 1 --2- 172.21.15.32:0/2070204227 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5550102230 0x7f55501a12b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:38.337 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.334+0000 7f554dd9b640 1 --2- 172.21.15.32:0/2070204227 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5550100680 0x7f5550199d40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.337 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.334+0000 7f554dd9b640 1 -- 172.21.15.32:0/2070204227 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5550102230 msgr2=0x7f55501a12b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:38.337 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.334+0000 7f554dd9b640 1 --2- 172.21.15.32:0/2070204227 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5550102230 0x7f55501a12b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.337 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.334+0000 7f554dd9b640 1 -- 172.21.15.32:0/2070204227 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f55501a1950 con 0x7f5550101870 2024-04-25T00:32:38.338 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.334+0000 7f554dd9b640 1 --2- 172.21.15.32:0/2070204227 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5550101870 0x7f555019a280 secure :-1 s=READY pgs=347 cs=0 l=1 rev1=1 crypto rx=0x7f554400c890 tx=0x7f554400cd50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:38.338 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.334+0000 7f553f7fe640 1 -- 172.21.15.32:0/2070204227 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5544004220 con 0x7f5550101870 2024-04-25T00:32:38.338 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.334+0000 7f553f7fe640 1 -- 172.21.15.32:0/2070204227 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f55440043c0 con 0x7f5550101870 2024-04-25T00:32:38.338 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.334+0000 7f5554ee1640 1 -- 172.21.15.32:0/2070204227 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f55501a1c40 con 0x7f5550101870 2024-04-25T00:32:38.338 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.334+0000 7f553f7fe640 1 -- 172.21.15.32:0/2070204227 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f55440049e0 con 0x7f5550101870 2024-04-25T00:32:38.338 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.334+0000 7f5554ee1640 1 -- 172.21.15.32:0/2070204227 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f55501a2190 con 0x7f5550101870 2024-04-25T00:32:38.339 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.335+0000 7f5554ee1640 1 -- 172.21.15.32:0/2070204227 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f55501a23c0 con 0x7f5550101870 2024-04-25T00:32:38.339 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.336+0000 7f553f7fe640 1 -- 172.21.15.32:0/2070204227 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f55440079b0 con 0x7f5550101870 2024-04-25T00:32:38.339 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.337+0000 7f553f7fe640 1 --2- 172.21.15.32:0/2070204227 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f55380a2c20 0x7f55380a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:38.339 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.337+0000 7f554e59c640 1 --2- 172.21.15.32:0/2070204227 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f55380a2c20 0x7f55380a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:38.339 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.337+0000 7f553f7fe640 1 -- 172.21.15.32:0/2070204227 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f55440cf6d0 con 0x7f5550101870 2024-04-25T00:32:38.339 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.338+0000 7f554e59c640 1 --2- 172.21.15.32:0/2070204227 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f55380a2c20 0x7f55380a50e0 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f5540006010 tx=0x7f5540000bc0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:38.343 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.342+0000 7f553f7fe640 1 -- 172.21.15.32:0/2070204227 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f5544014030 con 0x7f5550101870 2024-04-25T00:32:38.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:38 smithi032 ceph-mon[29357]: pgmap v348: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:32:38.531 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.530+0000 7f5554ee1640 1 -- 172.21.15.32:0/2070204227 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f55501a2720 con 0x7f5550101870 2024-04-25T00:32:38.533 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.531+0000 7f553f7fe640 1 -- 172.21.15.32:0/2070204227 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v91) ==== 74+0+24600 (secure 0 0 0) 0x7f554409c170 con 0x7f5550101870 2024-04-25T00:32:38.533 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:32:38.533 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":91,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:31:44.700076+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-25T00:30:46.239556+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-25T00:30:47.257182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2024-04-25T00:30:51.193975+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.6","mappings":[{"from":3,"to":6}]},{"pgid":"3.17","mappings":[{"from":4,"to":7}]},{"pgid":"3.1b","mappings":[{"from":5,"to":9}]},{"pgid":"3.20","mappings":[{"from":5,"to":9}]},{"pgid":"3.23","mappings":[{"from":4,"to":8}]},{"pgid":"3.2c","mappings":[{"from":4,"to":9}]},{"pgid":"4.c","mappings":[{"from":9,"to":10}]},{"pgid":"4.12","mappings":[{"from":9,"to":10}]},{"pgid":"4.1c","mappings":[{"from":9,"to":8}]},{"pgid":"4.1d","mappings":[{"from":5,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-num-failure-domains":"0","crush-osds-per-failure-domain":"0","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:32:38.535 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.534+0000 7f5554ee1640 1 -- 172.21.15.32:0/2070204227 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f55380a2c20 msgr2=0x7f55380a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:38.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.534+0000 7f5554ee1640 1 --2- 172.21.15.32:0/2070204227 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f55380a2c20 0x7f55380a50e0 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f5540006010 tx=0x7f5540000bc0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.534+0000 7f5554ee1640 1 -- 172.21.15.32:0/2070204227 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5550101870 msgr2=0x7f555019a280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:38.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.534+0000 7f5554ee1640 1 --2- 172.21.15.32:0/2070204227 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5550101870 0x7f555019a280 secure :-1 s=READY pgs=347 cs=0 l=1 rev1=1 crypto rx=0x7f554400c890 tx=0x7f554400cd50 comp rx=0 tx=0).stop 2024-04-25T00:32:38.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.534+0000 7f5554ee1640 1 -- 172.21.15.32:0/2070204227 shutdown_connections 2024-04-25T00:32:38.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.534+0000 7f5554ee1640 1 --2- 172.21.15.32:0/2070204227 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f55380a2c20 0x7f55380a50e0 unknown :-1 s=CLOSED pgs=222 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.534+0000 7f5554ee1640 1 --2- 172.21.15.32:0/2070204227 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5550102230 0x7f55501a12b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.534+0000 7f5554ee1640 1 --2- 172.21.15.32:0/2070204227 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5550101870 0x7f555019a280 unknown :-1 s=CLOSED pgs=347 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.534+0000 7f5554ee1640 1 --2- 172.21.15.32:0/2070204227 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5550100680 0x7f5550199d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.534+0000 7f5554ee1640 1 -- 172.21.15.32:0/2070204227 >> 172.21.15.32:0/2070204227 conn(0x7f55500fbbf0 msgr2=0x7f5550109480 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:38.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.534+0000 7f5554ee1640 1 -- 172.21.15.32:0/2070204227 shutdown_connections 2024-04-25T00:32:38.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.534+0000 7f5554ee1640 1 -- 172.21.15.32:0/2070204227 wait complete. 2024-04-25T00:32:38.547 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-25T00:26:07.306839+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-04-25T00:30:46.239556+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '84', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-04-25T00:30:47.257182+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '85', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.05000000074505806}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_data_ec', 'create_time': '2024-04-25T00:30:51.193975+0000', 'flags': 5, 'flags_names': 'hashpspool,ec_overwrites', 'type': 3, 'size': 4, 'min_size': 3, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '89', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'cephfs_data_ec', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 8192, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}] 2024-04-25T00:32:38.547 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-25T00:32:38.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.652+0000 7f861609a640 1 -- 172.21.15.32:0/3938179580 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8610102230 msgr2=0x7f8610108ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:38.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.652+0000 7f861609a640 1 --2- 172.21.15.32:0/3938179580 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8610102230 0x7f8610108ac0 secure :-1 s=READY pgs=348 cs=0 l=1 rev1=1 crypto rx=0x7f8608009ca0 tx=0x7f860802a8c0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.652+0000 7f861609a640 1 -- 172.21.15.32:0/3938179580 shutdown_connections 2024-04-25T00:32:38.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.652+0000 7f861609a640 1 --2- 172.21.15.32:0/3938179580 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8610102230 0x7f8610108ac0 unknown :-1 s=CLOSED pgs=348 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.652+0000 7f861609a640 1 --2- 172.21.15.32:0/3938179580 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8610101870 0x7f8610101cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.652+0000 7f861609a640 1 --2- 172.21.15.32:0/3938179580 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8610100680 0x7f8610100aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.652+0000 7f861609a640 1 -- 172.21.15.32:0/3938179580 >> 172.21.15.32:0/3938179580 conn(0x7f86100fbbf0 msgr2=0x7f86100fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:38.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.652+0000 7f861609a640 1 -- 172.21.15.32:0/3938179580 shutdown_connections 2024-04-25T00:32:38.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.652+0000 7f861609a640 1 -- 172.21.15.32:0/3938179580 wait complete. 2024-04-25T00:32:38.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.652+0000 7f861609a640 1 Processor -- start 2024-04-25T00:32:38.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.653+0000 7f861609a640 1 -- start start 2024-04-25T00:32:38.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.653+0000 7f861609a640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8610100680 0x7f861010b000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:38.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.653+0000 7f861609a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8610101870 0x7f8610109640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:38.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.653+0000 7f861609a640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8610102230 0x7f8610109b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:38.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.653+0000 7f861609a640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f861010d420 con 0x7f8610101870 2024-04-25T00:32:38.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.653+0000 7f861609a640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f861010d2a0 con 0x7f8610100680 2024-04-25T00:32:38.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.653+0000 7f861609a640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f861010d5a0 con 0x7f8610102230 2024-04-25T00:32:38.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.653+0000 7f860effd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8610101870 0x7f8610109640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:38.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.653+0000 7f860effd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8610101870 0x7f8610109640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40776/0 (socket says 172.21.15.32:40776) 2024-04-25T00:32:38.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.653+0000 7f860effd640 1 -- 172.21.15.32:0/779446494 learned_addr learned my addr 172.21.15.32:0/779446494 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:38.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.653+0000 7f860ffff640 1 --2- 172.21.15.32:0/779446494 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8610102230 0x7f8610109b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:38.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.653+0000 7f860f7fe640 1 --2- 172.21.15.32:0/779446494 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8610100680 0x7f861010b000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:38.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.653+0000 7f860effd640 1 -- 172.21.15.32:0/779446494 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8610102230 msgr2=0x7f8610109b80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:38.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.653+0000 7f860effd640 1 --2- 172.21.15.32:0/779446494 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8610102230 0x7f8610109b80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.653+0000 7f860effd640 1 -- 172.21.15.32:0/779446494 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8610100680 msgr2=0x7f861010b000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:38.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.653+0000 7f860effd640 1 --2- 172.21.15.32:0/779446494 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8610100680 0x7f861010b000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.653+0000 7f860effd640 1 -- 172.21.15.32:0/779446494 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f861010a410 con 0x7f8610101870 2024-04-25T00:32:38.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.653+0000 7f860f7fe640 1 --2- 172.21.15.32:0/779446494 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8610100680 0x7f861010b000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:32:38.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.654+0000 7f860effd640 1 --2- 172.21.15.32:0/779446494 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8610101870 0x7f8610109640 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7f860000c920 tx=0x7f860000cde0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:38.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.654+0000 7f860cff9640 1 -- 172.21.15.32:0/779446494 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8600007c70 con 0x7f8610101870 2024-04-25T00:32:38.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.654+0000 7f860cff9640 1 -- 172.21.15.32:0/779446494 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8600007e10 con 0x7f8610101870 2024-04-25T00:32:38.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.654+0000 7f861609a640 1 -- 172.21.15.32:0/779446494 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f86101a1b10 con 0x7f8610101870 2024-04-25T00:32:38.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.654+0000 7f860cff9640 1 -- 172.21.15.32:0/779446494 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8600018890 con 0x7f8610101870 2024-04-25T00:32:38.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.654+0000 7f861609a640 1 -- 172.21.15.32:0/779446494 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f86101a2060 con 0x7f8610101870 2024-04-25T00:32:38.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.655+0000 7f861609a640 1 -- 172.21.15.32:0/779446494 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8610107940 con 0x7f8610101870 2024-04-25T00:32:38.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.657+0000 7f860cff9640 1 -- 172.21.15.32:0/779446494 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f8600018a50 con 0x7f8610101870 2024-04-25T00:32:38.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.657+0000 7f860cff9640 1 --2- 172.21.15.32:0/779446494 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f85fc0a2c20 0x7f85fc0a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:38.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.658+0000 7f860cff9640 1 -- 172.21.15.32:0/779446494 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f86000cf9d0 con 0x7f8610101870 2024-04-25T00:32:38.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.658+0000 7f860f7fe640 1 --2- 172.21.15.32:0/779446494 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f85fc0a2c20 0x7f85fc0a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:38.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.658+0000 7f860f7fe640 1 --2- 172.21.15.32:0/779446494 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f85fc0a2c20 0x7f85fc0a50e0 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f86101016d0 tx=0x7f85f8005f90 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:38.661 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.660+0000 7f860cff9640 1 -- 172.21.15.32:0/779446494 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f8600014030 con 0x7f8610101870 2024-04-25T00:32:38.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:38 smithi121 ceph-mon[32231]: pgmap v348: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:32:38.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.846+0000 7f861609a640 1 -- 172.21.15.32:0/779446494 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7f861002d300 con 0x7f8610101870 2024-04-25T00:32:38.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.847+0000 7f860cff9640 1 -- 172.21.15.32:0/779446494 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v91) ==== 93+0+10 (secure 0 0 0) 0x7f860009c4a0 con 0x7f8610101870 2024-04-25T00:32:38.848 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 1 2024-04-25T00:32:38.850 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.849+0000 7f861609a640 1 -- 172.21.15.32:0/779446494 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f85fc0a2c20 msgr2=0x7f85fc0a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:38.851 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.849+0000 7f861609a640 1 --2- 172.21.15.32:0/779446494 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f85fc0a2c20 0x7f85fc0a50e0 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f86101016d0 tx=0x7f85f8005f90 comp rx=0 tx=0).stop 2024-04-25T00:32:38.851 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.849+0000 7f861609a640 1 -- 172.21.15.32:0/779446494 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8610101870 msgr2=0x7f8610109640 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:38.851 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.849+0000 7f861609a640 1 --2- 172.21.15.32:0/779446494 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8610101870 0x7f8610109640 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7f860000c920 tx=0x7f860000cde0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.851 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.849+0000 7f861609a640 1 -- 172.21.15.32:0/779446494 shutdown_connections 2024-04-25T00:32:38.851 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.849+0000 7f861609a640 1 --2- 172.21.15.32:0/779446494 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f85fc0a2c20 0x7f85fc0a50e0 unknown :-1 s=CLOSED pgs=223 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.851 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.849+0000 7f861609a640 1 --2- 172.21.15.32:0/779446494 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8610102230 0x7f8610109b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.851 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.849+0000 7f861609a640 1 --2- 172.21.15.32:0/779446494 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8610101870 0x7f8610109640 unknown :-1 s=CLOSED pgs=349 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.851 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.849+0000 7f861609a640 1 --2- 172.21.15.32:0/779446494 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8610100680 0x7f861010b000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.851 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.849+0000 7f861609a640 1 -- 172.21.15.32:0/779446494 >> 172.21.15.32:0/779446494 conn(0x7f86100fbbf0 msgr2=0x7f8610104b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:38.851 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.849+0000 7f861609a640 1 -- 172.21.15.32:0/779446494 shutdown_connections 2024-04-25T00:32:38.851 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.849+0000 7f861609a640 1 -- 172.21.15.32:0/779446494 wait complete. 2024-04-25T00:32:38.862 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-04-25T00:32:38.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:38 smithi130 ceph-mon[32525]: pgmap v348: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:32:38.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.966+0000 7f611a0df640 1 -- 172.21.15.32:0/3260067376 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6114102250 msgr2=0x7f6114108ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:38.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.966+0000 7f611a0df640 1 --2- 172.21.15.32:0/3260067376 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6114102250 0x7f6114108ae0 secure :-1 s=READY pgs=350 cs=0 l=1 rev1=1 crypto rx=0x7f610c009900 tx=0x7f610c02a410 comp rx=0 tx=0).stop 2024-04-25T00:32:38.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.966+0000 7f611a0df640 1 -- 172.21.15.32:0/3260067376 shutdown_connections 2024-04-25T00:32:38.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.966+0000 7f611a0df640 1 --2- 172.21.15.32:0/3260067376 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6114102250 0x7f6114108ae0 unknown :-1 s=CLOSED pgs=350 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.966+0000 7f611a0df640 1 --2- 172.21.15.32:0/3260067376 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6114101890 0x7f6114101d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.966+0000 7f611a0df640 1 --2- 172.21.15.32:0/3260067376 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f61141006a0 0x7f6114100ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.966+0000 7f611a0df640 1 -- 172.21.15.32:0/3260067376 >> 172.21.15.32:0/3260067376 conn(0x7f61140fbc30 msgr2=0x7f61140fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:38.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.966+0000 7f611a0df640 1 -- 172.21.15.32:0/3260067376 shutdown_connections 2024-04-25T00:32:38.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.966+0000 7f611a0df640 1 -- 172.21.15.32:0/3260067376 wait complete. 2024-04-25T00:32:38.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.967+0000 7f611a0df640 1 Processor -- start 2024-04-25T00:32:38.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.967+0000 7f611a0df640 1 -- start start 2024-04-25T00:32:38.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.967+0000 7f611a0df640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f61141006a0 0x7f61141968b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:38.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.967+0000 7f611a0df640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6114101890 0x7f6114196df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:38.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.967+0000 7f611a0df640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6114102250 0x7f611419de60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:38.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.967+0000 7f611a0df640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f611410c300 con 0x7f6114101890 2024-04-25T00:32:38.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.967+0000 7f611a0df640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f611410c180 con 0x7f61141006a0 2024-04-25T00:32:38.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.967+0000 7f611a0df640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f611410c480 con 0x7f6114102250 2024-04-25T00:32:38.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.967+0000 7f6112ffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6114101890 0x7f6114196df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:38.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.967+0000 7f6112ffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6114101890 0x7f6114196df0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40800/0 (socket says 172.21.15.32:40800) 2024-04-25T00:32:38.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.967+0000 7f6112ffd640 1 -- 172.21.15.32:0/3668627901 learned_addr learned my addr 172.21.15.32:0/3668627901 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:38.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.967+0000 7f61137fe640 1 --2- 172.21.15.32:0/3668627901 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f61141006a0 0x7f61141968b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:38.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.968+0000 7f6113fff640 1 --2- 172.21.15.32:0/3668627901 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6114102250 0x7f611419de60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:38.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.968+0000 7f6112ffd640 1 -- 172.21.15.32:0/3668627901 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6114102250 msgr2=0x7f611419de60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:38.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.968+0000 7f6112ffd640 1 --2- 172.21.15.32:0/3668627901 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6114102250 0x7f611419de60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.968+0000 7f6112ffd640 1 -- 172.21.15.32:0/3668627901 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f61141006a0 msgr2=0x7f61141968b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:38.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.968+0000 7f6112ffd640 1 --2- 172.21.15.32:0/3668627901 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f61141006a0 0x7f61141968b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:38.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.968+0000 7f6112ffd640 1 -- 172.21.15.32:0/3668627901 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f611419e4d0 con 0x7f6114101890 2024-04-25T00:32:38.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.968+0000 7f6113fff640 1 --2- 172.21.15.32:0/3668627901 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6114102250 0x7f611419de60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:32:38.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.968+0000 7f6112ffd640 1 --2- 172.21.15.32:0/3668627901 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6114101890 0x7f6114196df0 secure :-1 s=READY pgs=351 cs=0 l=1 rev1=1 crypto rx=0x7f610800c920 tx=0x7f610800cde0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:38.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.968+0000 7f6110ff9640 1 -- 172.21.15.32:0/3668627901 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6108007c70 con 0x7f6114101890 2024-04-25T00:32:38.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.968+0000 7f611a0df640 1 -- 172.21.15.32:0/3668627901 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f611419e7c0 con 0x7f6114101890 2024-04-25T00:32:38.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.968+0000 7f6110ff9640 1 -- 172.21.15.32:0/3668627901 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6108007e10 con 0x7f6114101890 2024-04-25T00:32:38.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.968+0000 7f6110ff9640 1 -- 172.21.15.32:0/3668627901 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6108018890 con 0x7f6114101890 2024-04-25T00:32:38.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.968+0000 7f611a0df640 1 -- 172.21.15.32:0/3668627901 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f611419ed60 con 0x7f6114101890 2024-04-25T00:32:38.973 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.969+0000 7f611a0df640 1 -- 172.21.15.32:0/3668627901 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f61141078f0 con 0x7f6114101890 2024-04-25T00:32:38.973 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.970+0000 7f6110ff9640 1 -- 172.21.15.32:0/3668627901 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f6108018a70 con 0x7f6114101890 2024-04-25T00:32:38.974 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.971+0000 7f6110ff9640 1 --2- 172.21.15.32:0/3668627901 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f60fc0a2c20 0x7f60fc0a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:38.974 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.971+0000 7f61137fe640 1 --2- 172.21.15.32:0/3668627901 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f60fc0a2c20 0x7f60fc0a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:38.974 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.971+0000 7f6110ff9640 1 -- 172.21.15.32:0/3668627901 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f61080cf730 con 0x7f6114101890 2024-04-25T00:32:38.974 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.971+0000 7f61137fe640 1 --2- 172.21.15.32:0/3668627901 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f60fc0a2c20 0x7f60fc0a50e0 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f61141016f0 tx=0x7f6104009870 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:38.977 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:38.975+0000 7f6110ff9640 1 -- 172.21.15.32:0/3668627901 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f6108014030 con 0x7f6114101890 2024-04-25T00:32:39.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.163+0000 7f611a0df640 1 -- 172.21.15.32:0/3668627901 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) -- 0x7f611402d380 con 0x7f6114101890 2024-04-25T00:32:39.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.163+0000 7f6110ff9640 1 -- 172.21.15.32:0/3668627901 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v91) ==== 104+0+11 (secure 0 0 0) 0x7f610800de20 con 0x7f6114101890 2024-04-25T00:32:39.165 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 32 2024-04-25T00:32:39.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.165+0000 7f611a0df640 1 -- 172.21.15.32:0/3668627901 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f60fc0a2c20 msgr2=0x7f60fc0a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:39.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.165+0000 7f611a0df640 1 --2- 172.21.15.32:0/3668627901 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f60fc0a2c20 0x7f60fc0a50e0 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f61141016f0 tx=0x7f6104009870 comp rx=0 tx=0).stop 2024-04-25T00:32:39.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.165+0000 7f611a0df640 1 -- 172.21.15.32:0/3668627901 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6114101890 msgr2=0x7f6114196df0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:39.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.165+0000 7f611a0df640 1 --2- 172.21.15.32:0/3668627901 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6114101890 0x7f6114196df0 secure :-1 s=READY pgs=351 cs=0 l=1 rev1=1 crypto rx=0x7f610800c920 tx=0x7f610800cde0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.166+0000 7f611a0df640 1 -- 172.21.15.32:0/3668627901 shutdown_connections 2024-04-25T00:32:39.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.166+0000 7f611a0df640 1 --2- 172.21.15.32:0/3668627901 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f60fc0a2c20 0x7f60fc0a50e0 unknown :-1 s=CLOSED pgs=224 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.166+0000 7f611a0df640 1 --2- 172.21.15.32:0/3668627901 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6114102250 0x7f611419de60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.166+0000 7f611a0df640 1 --2- 172.21.15.32:0/3668627901 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6114101890 0x7f6114196df0 unknown :-1 s=CLOSED pgs=351 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.166+0000 7f611a0df640 1 --2- 172.21.15.32:0/3668627901 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f61141006a0 0x7f61141968b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.166+0000 7f611a0df640 1 -- 172.21.15.32:0/3668627901 >> 172.21.15.32:0/3668627901 conn(0x7f61140fbc30 msgr2=0x7f6114104ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:39.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.166+0000 7f611a0df640 1 -- 172.21.15.32:0/3668627901 shutdown_connections 2024-04-25T00:32:39.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.166+0000 7f611a0df640 1 -- 172.21.15.32:0/3668627901 wait complete. 2024-04-25T00:32:39.178 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-04-25T00:32:39.287 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.283+0000 7f1dfc961640 1 -- 172.21.15.32:0/3186004109 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1df8102990 msgr2=0x7f1df8102e10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:39.287 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.283+0000 7f1dfc961640 1 --2- 172.21.15.32:0/3186004109 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1df8102990 0x7f1df8102e10 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 crypto rx=0x7f1de0009900 tx=0x7f1de002a410 comp rx=0 tx=0).stop 2024-04-25T00:32:39.287 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.283+0000 7f1dfc961640 1 -- 172.21.15.32:0/3186004109 shutdown_connections 2024-04-25T00:32:39.287 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.283+0000 7f1dfc961640 1 --2- 172.21.15.32:0/3186004109 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1df8103350 0x7f1df8109be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.283+0000 7f1dfc961640 1 --2- 172.21.15.32:0/3186004109 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1df8102990 0x7f1df8102e10 unknown :-1 s=CLOSED pgs=352 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.283+0000 7f1dfc961640 1 --2- 172.21.15.32:0/3186004109 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1df81017a0 0x7f1df8101bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.283+0000 7f1dfc961640 1 -- 172.21.15.32:0/3186004109 >> 172.21.15.32:0/3186004109 conn(0x7f1df80fcd10 msgr2=0x7f1df80ff150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:39.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.283+0000 7f1dfc961640 1 -- 172.21.15.32:0/3186004109 shutdown_connections 2024-04-25T00:32:39.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.283+0000 7f1dfc961640 1 -- 172.21.15.32:0/3186004109 wait complete. 2024-04-25T00:32:39.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.284+0000 7f1dfc961640 1 Processor -- start 2024-04-25T00:32:39.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.284+0000 7f1dfc961640 1 -- start start 2024-04-25T00:32:39.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.284+0000 7f1dfc961640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1df81017a0 0x7f1df8197a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:39.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.284+0000 7f1dfc961640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1df8102990 0x7f1df8197f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:39.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.284+0000 7f1dfc961640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1df8103350 0x7f1df819f000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:39.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.284+0000 7f1dfc961640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f1df810d400 con 0x7f1df8103350 2024-04-25T00:32:39.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.284+0000 7f1dfc961640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f1df810d280 con 0x7f1df8102990 2024-04-25T00:32:39.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.284+0000 7f1dfc961640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f1df810d580 con 0x7f1df81017a0 2024-04-25T00:32:39.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.284+0000 7f1df6d9d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1df8103350 0x7f1df819f000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:39.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.284+0000 7f1df6d9d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1df8103350 0x7f1df819f000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40810/0 (socket says 172.21.15.32:40810) 2024-04-25T00:32:39.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.284+0000 7f1df6d9d640 1 -- 172.21.15.32:0/586742952 learned_addr learned my addr 172.21.15.32:0/586742952 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:39.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.284+0000 7f1dedd9b640 1 --2- 172.21.15.32:0/586742952 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1df8102990 0x7f1df8197f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:39.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.284+0000 7f1df659c640 1 --2- 172.21.15.32:0/586742952 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1df81017a0 0x7f1df8197a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:39.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.284+0000 7f1df6d9d640 1 -- 172.21.15.32:0/586742952 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1df81017a0 msgr2=0x7f1df8197a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:39.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.284+0000 7f1df6d9d640 1 --2- 172.21.15.32:0/586742952 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1df81017a0 0x7f1df8197a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.284+0000 7f1df6d9d640 1 -- 172.21.15.32:0/586742952 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1df8102990 msgr2=0x7f1df8197f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:39.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.284+0000 7f1df6d9d640 1 --2- 172.21.15.32:0/586742952 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1df8102990 0x7f1df8197f90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.284+0000 7f1df6d9d640 1 -- 172.21.15.32:0/586742952 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1df819f670 con 0x7f1df8103350 2024-04-25T00:32:39.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.285+0000 7f1df6d9d640 1 --2- 172.21.15.32:0/586742952 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1df8103350 0x7f1df819f000 secure :-1 s=READY pgs=353 cs=0 l=1 rev1=1 crypto rx=0x7f1df000cad0 tx=0x7f1df000cf90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:39.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.285+0000 7f1deffff640 1 -- 172.21.15.32:0/586742952 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1df000ae20 con 0x7f1df8103350 2024-04-25T00:32:39.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.285+0000 7f1deffff640 1 -- 172.21.15.32:0/586742952 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1df0005020 con 0x7f1df8103350 2024-04-25T00:32:39.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.285+0000 7f1dfc961640 1 -- 172.21.15.32:0/586742952 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1df819f960 con 0x7f1df8103350 2024-04-25T00:32:39.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.285+0000 7f1deffff640 1 -- 172.21.15.32:0/586742952 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1df000f6f0 con 0x7f1df8103350 2024-04-25T00:32:39.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.285+0000 7f1dfc961640 1 -- 172.21.15.32:0/586742952 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1df819fe30 con 0x7f1df8103350 2024-04-25T00:32:39.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.286+0000 7f1dfc961640 1 -- 172.21.15.32:0/586742952 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1df8108ac0 con 0x7f1df8103350 2024-04-25T00:32:39.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.287+0000 7f1deffff640 1 -- 172.21.15.32:0/586742952 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f1df00051d0 con 0x7f1df8103350 2024-04-25T00:32:39.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.288+0000 7f1deffff640 1 --2- 172.21.15.32:0/586742952 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f1de40a2c20 0x7f1de40a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:39.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.288+0000 7f1df659c640 1 --2- 172.21.15.32:0/586742952 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f1de40a2c20 0x7f1de40a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:39.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.288+0000 7f1deffff640 1 -- 172.21.15.32:0/586742952 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f1df00cf6a0 con 0x7f1df8103350 2024-04-25T00:32:39.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.288+0000 7f1df659c640 1 --2- 172.21.15.32:0/586742952 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f1de40a2c20 0x7f1de40a50e0 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7f1de80058c0 tx=0x7f1de8005850 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:39.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.293+0000 7f1deffff640 1 -- 172.21.15.32:0/586742952 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f1df0014030 con 0x7f1df8103350 2024-04-25T00:32:39.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.479+0000 7f1dfc961640 1 -- 172.21.15.32:0/586742952 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) -- 0x7f1df802d170 con 0x7f1df8103350 2024-04-25T00:32:39.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.480+0000 7f1deffff640 1 -- 172.21.15.32:0/586742952 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v91) ==== 100+0+11 (secure 0 0 0) 0x7f1df009c140 con 0x7f1df8103350 2024-04-25T00:32:39.481 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 64 2024-04-25T00:32:39.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.481+0000 7f1dfc961640 1 -- 172.21.15.32:0/586742952 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f1de40a2c20 msgr2=0x7f1de40a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:39.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.481+0000 7f1dfc961640 1 --2- 172.21.15.32:0/586742952 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f1de40a2c20 0x7f1de40a50e0 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7f1de80058c0 tx=0x7f1de8005850 comp rx=0 tx=0).stop 2024-04-25T00:32:39.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.481+0000 7f1dfc961640 1 -- 172.21.15.32:0/586742952 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1df8103350 msgr2=0x7f1df819f000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:39.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.481+0000 7f1dfc961640 1 --2- 172.21.15.32:0/586742952 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1df8103350 0x7f1df819f000 secure :-1 s=READY pgs=353 cs=0 l=1 rev1=1 crypto rx=0x7f1df000cad0 tx=0x7f1df000cf90 comp rx=0 tx=0).stop 2024-04-25T00:32:39.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.482+0000 7f1dfc961640 1 -- 172.21.15.32:0/586742952 shutdown_connections 2024-04-25T00:32:39.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.482+0000 7f1dfc961640 1 --2- 172.21.15.32:0/586742952 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f1de40a2c20 0x7f1de40a50e0 unknown :-1 s=CLOSED pgs=225 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.482+0000 7f1dfc961640 1 --2- 172.21.15.32:0/586742952 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1df8103350 0x7f1df819f000 unknown :-1 s=CLOSED pgs=353 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.482+0000 7f1dfc961640 1 --2- 172.21.15.32:0/586742952 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1df8102990 0x7f1df8197f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.482+0000 7f1dfc961640 1 --2- 172.21.15.32:0/586742952 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1df81017a0 0x7f1df8197a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.482+0000 7f1dfc961640 1 -- 172.21.15.32:0/586742952 >> 172.21.15.32:0/586742952 conn(0x7f1df80fcd10 msgr2=0x7f1df8105c90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:39.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.482+0000 7f1dfc961640 1 -- 172.21.15.32:0/586742952 shutdown_connections 2024-04-25T00:32:39.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.482+0000 7f1dfc961640 1 -- 172.21.15.32:0/586742952 wait complete. 2024-04-25T00:32:39.494 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data_ec pg_num 2024-04-25T00:32:39.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.600+0000 7f554df6a640 1 -- 172.21.15.32:0/1880122699 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5548102250 msgr2=0x7f5548108ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:39.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.600+0000 7f554df6a640 1 --2- 172.21.15.32:0/1880122699 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5548102250 0x7f5548108ae0 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 crypto rx=0x7f5540009ca0 tx=0x7f554002a8c0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.600+0000 7f554df6a640 1 -- 172.21.15.32:0/1880122699 shutdown_connections 2024-04-25T00:32:39.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.600+0000 7f554df6a640 1 --2- 172.21.15.32:0/1880122699 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5548102250 0x7f5548108ae0 unknown :-1 s=CLOSED pgs=354 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.600+0000 7f554df6a640 1 --2- 172.21.15.32:0/1880122699 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5548101890 0x7f5548101d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.600+0000 7f554df6a640 1 --2- 172.21.15.32:0/1880122699 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f55481006a0 0x7f5548100ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.600+0000 7f554df6a640 1 -- 172.21.15.32:0/1880122699 >> 172.21.15.32:0/1880122699 conn(0x7f55480fbc30 msgr2=0x7f55480fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:39.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.600+0000 7f554df6a640 1 -- 172.21.15.32:0/1880122699 shutdown_connections 2024-04-25T00:32:39.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.600+0000 7f554df6a640 1 -- 172.21.15.32:0/1880122699 wait complete. 2024-04-25T00:32:39.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.600+0000 7f554df6a640 1 Processor -- start 2024-04-25T00:32:39.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.600+0000 7f554df6a640 1 -- start start 2024-04-25T00:32:39.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.601+0000 7f554df6a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f55481006a0 0x7f5548197bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:39.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.601+0000 7f554df6a640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5548101890 0x7f5548198110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:39.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.601+0000 7f554df6a640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5548102250 0x7f554819f180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:39.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.601+0000 7f554df6a640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f554810c410 con 0x7f55481006a0 2024-04-25T00:32:39.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.601+0000 7f554df6a640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f554810c290 con 0x7f5548101890 2024-04-25T00:32:39.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.601+0000 7f554df6a640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f554810c590 con 0x7f5548102250 2024-04-25T00:32:39.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.601+0000 7f55477fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f55481006a0 0x7f5548197bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:39.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.601+0000 7f55477fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f55481006a0 0x7f5548197bd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40828/0 (socket says 172.21.15.32:40828) 2024-04-25T00:32:39.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.601+0000 7f55477fe640 1 -- 172.21.15.32:0/1024939214 learned_addr learned my addr 172.21.15.32:0/1024939214 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:39.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.601+0000 7f5547fff640 1 --2- 172.21.15.32:0/1024939214 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5548102250 0x7f554819f180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:39.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.601+0000 7f5546ffd640 1 --2- 172.21.15.32:0/1024939214 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5548101890 0x7f5548198110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:39.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.601+0000 7f55477fe640 1 -- 172.21.15.32:0/1024939214 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5548102250 msgr2=0x7f554819f180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:39.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.601+0000 7f55477fe640 1 --2- 172.21.15.32:0/1024939214 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5548102250 0x7f554819f180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.601+0000 7f55477fe640 1 -- 172.21.15.32:0/1024939214 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5548101890 msgr2=0x7f5548198110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:39.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.601+0000 7f55477fe640 1 --2- 172.21.15.32:0/1024939214 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5548101890 0x7f5548198110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.601+0000 7f55477fe640 1 -- 172.21.15.32:0/1024939214 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f554819f7f0 con 0x7f55481006a0 2024-04-25T00:32:39.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.601+0000 7f5546ffd640 1 --2- 172.21.15.32:0/1024939214 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5548101890 0x7f5548198110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:32:39.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.601+0000 7f55477fe640 1 --2- 172.21.15.32:0/1024939214 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f55481006a0 0x7f5548197bd0 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7f553800b450 tx=0x7f553800b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:39.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.601+0000 7f5544ff9640 1 -- 172.21.15.32:0/1024939214 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5538015020 con 0x7f55481006a0 2024-04-25T00:32:39.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.601+0000 7f5544ff9640 1 -- 172.21.15.32:0/1024939214 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5538011d70 con 0x7f55481006a0 2024-04-25T00:32:39.606 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.601+0000 7f554df6a640 1 -- 172.21.15.32:0/1024939214 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f554819fae0 con 0x7f55481006a0 2024-04-25T00:32:39.606 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.601+0000 7f5544ff9640 1 -- 172.21.15.32:0/1024939214 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f553800f340 con 0x7f55481006a0 2024-04-25T00:32:39.606 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.602+0000 7f554df6a640 1 -- 172.21.15.32:0/1024939214 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f55481a0030 con 0x7f55481006a0 2024-04-25T00:32:39.607 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.602+0000 7f554df6a640 1 -- 172.21.15.32:0/1024939214 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f55481a0260 con 0x7f55481006a0 2024-04-25T00:32:39.607 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.604+0000 7f5544ff9640 1 -- 172.21.15.32:0/1024939214 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f5538018020 con 0x7f55481006a0 2024-04-25T00:32:39.607 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.604+0000 7f5544ff9640 1 --2- 172.21.15.32:0/1024939214 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f55300a2c20 0x7f55300a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:39.607 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.604+0000 7f5546ffd640 1 --2- 172.21.15.32:0/1024939214 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f55300a2c20 0x7f55300a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:39.607 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.604+0000 7f5544ff9640 1 -- 172.21.15.32:0/1024939214 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f55380cf300 con 0x7f55481006a0 2024-04-25T00:32:39.607 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.605+0000 7f5546ffd640 1 --2- 172.21.15.32:0/1024939214 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f55300a2c20 0x7f55300a50e0 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7f553c006fd0 tx=0x7f553c008040 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:39.609 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.608+0000 7f5544ff9640 1 -- 172.21.15.32:0/1024939214 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f553801f020 con 0x7f55481006a0 2024-04-25T00:32:39.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:39 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2070204227' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:32:39.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:39 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/779446494' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:32:39.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:39 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3668627901' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:32:39.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.792+0000 7f554df6a640 1 -- 172.21.15.32:0/1024939214 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"} v 0) -- 0x7f55481a0430 con 0x7f55481006a0 2024-04-25T00:32:39.794 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.792+0000 7f5544ff9640 1 -- 172.21.15.32:0/1024939214 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]=0 v91) ==== 103+0+11 (secure 0 0 0) 0x7f553800d040 con 0x7f55481006a0 2024-04-25T00:32:39.794 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 32 2024-04-25T00:32:39.796 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.794+0000 7f554df6a640 1 -- 172.21.15.32:0/1024939214 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f55300a2c20 msgr2=0x7f55300a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:39.796 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.794+0000 7f554df6a640 1 --2- 172.21.15.32:0/1024939214 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f55300a2c20 0x7f55300a50e0 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7f553c006fd0 tx=0x7f553c008040 comp rx=0 tx=0).stop 2024-04-25T00:32:39.796 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.794+0000 7f554df6a640 1 -- 172.21.15.32:0/1024939214 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f55481006a0 msgr2=0x7f5548197bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:39.796 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.794+0000 7f554df6a640 1 --2- 172.21.15.32:0/1024939214 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f55481006a0 0x7f5548197bd0 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7f553800b450 tx=0x7f553800b910 comp rx=0 tx=0).stop 2024-04-25T00:32:39.796 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.794+0000 7f554df6a640 1 -- 172.21.15.32:0/1024939214 shutdown_connections 2024-04-25T00:32:39.796 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.794+0000 7f554df6a640 1 --2- 172.21.15.32:0/1024939214 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f55300a2c20 0x7f55300a50e0 unknown :-1 s=CLOSED pgs=226 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.796 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.794+0000 7f554df6a640 1 --2- 172.21.15.32:0/1024939214 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5548102250 0x7f554819f180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.794+0000 7f554df6a640 1 --2- 172.21.15.32:0/1024939214 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5548101890 0x7f5548198110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.794+0000 7f554df6a640 1 --2- 172.21.15.32:0/1024939214 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f55481006a0 0x7f5548197bd0 unknown :-1 s=CLOSED pgs=355 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.794+0000 7f554df6a640 1 -- 172.21.15.32:0/1024939214 >> 172.21.15.32:0/1024939214 conn(0x7f55480fbc30 msgr2=0x7f5548069d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:39.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.795+0000 7f554df6a640 1 -- 172.21.15.32:0/1024939214 shutdown_connections 2024-04-25T00:32:39.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.795+0000 7f554df6a640 1 -- 172.21.15.32:0/1024939214 wait complete. 2024-04-25T00:32:39.807 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:32:39.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:39 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2070204227' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:32:39.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:39 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/779446494' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:32:39.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:39 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3668627901' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:32:39.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:39 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2070204227' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:32:39.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:39 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/779446494' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:32:39.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:39 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3668627901' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:32:39.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.910+0000 7f99d724d640 1 -- 172.21.15.32:0/843130068 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f99d01022c0 msgr2=0x7f99d01046b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:39.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.910+0000 7f99d724d640 1 --2- 172.21.15.32:0/843130068 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f99d01022c0 0x7f99d01046b0 secure :-1 s=READY pgs=356 cs=0 l=1 rev1=1 crypto rx=0x7f99c4009900 tx=0x7f99c402a410 comp rx=0 tx=0).stop 2024-04-25T00:32:39.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.910+0000 7f99d724d640 1 -- 172.21.15.32:0/843130068 shutdown_connections 2024-04-25T00:32:39.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.910+0000 7f99d724d640 1 --2- 172.21.15.32:0/843130068 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f99d0107520 0x7f99d0109910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.910+0000 7f99d724d640 1 --2- 172.21.15.32:0/843130068 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f99d0104bf0 0x7f99d0106fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.910+0000 7f99d724d640 1 --2- 172.21.15.32:0/843130068 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f99d01022c0 0x7f99d01046b0 unknown :-1 s=CLOSED pgs=356 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.910+0000 7f99d724d640 1 -- 172.21.15.32:0/843130068 >> 172.21.15.32:0/843130068 conn(0x7f99d00fbc30 msgr2=0x7f99d00fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:39.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.910+0000 7f99d724d640 1 -- 172.21.15.32:0/843130068 shutdown_connections 2024-04-25T00:32:39.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.910+0000 7f99d724d640 1 -- 172.21.15.32:0/843130068 wait complete. 2024-04-25T00:32:39.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.910+0000 7f99d724d640 1 Processor -- start 2024-04-25T00:32:39.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.910+0000 7f99d724d640 1 -- start start 2024-04-25T00:32:39.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.911+0000 7f99d724d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f99d01022c0 0x7f99d0197b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:39.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.911+0000 7f99d724d640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f99d0104bf0 0x7f99d0198050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:39.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.911+0000 7f99d724d640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f99d0107520 0x7f99d019f0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:39.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.911+0000 7f99d724d640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f99d010d410 con 0x7f99d01022c0 2024-04-25T00:32:39.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.911+0000 7f99d724d640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f99d010d290 con 0x7f99d0107520 2024-04-25T00:32:39.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.911+0000 7f99d724d640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f99d010d590 con 0x7f99d0104bf0 2024-04-25T00:32:39.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.911+0000 7f99d4fe9640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f99d01022c0 0x7f99d0197b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:39.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.911+0000 7f99d4fe9640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f99d01022c0 0x7f99d0197b10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40854/0 (socket says 172.21.15.32:40854) 2024-04-25T00:32:39.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.911+0000 7f99d4fe9640 1 -- 172.21.15.32:0/3540670235 learned_addr learned my addr 172.21.15.32:0/3540670235 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:39.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.911+0000 7f99cffff640 1 --2- 172.21.15.32:0/3540670235 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f99d0104bf0 0x7f99d0198050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:39.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.911+0000 7f99d57ea640 1 --2- 172.21.15.32:0/3540670235 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f99d0107520 0x7f99d019f0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:39.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.911+0000 7f99d4fe9640 1 -- 172.21.15.32:0/3540670235 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f99d0104bf0 msgr2=0x7f99d0198050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:39.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.911+0000 7f99d4fe9640 1 --2- 172.21.15.32:0/3540670235 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f99d0104bf0 0x7f99d0198050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.911+0000 7f99d4fe9640 1 -- 172.21.15.32:0/3540670235 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f99d0107520 msgr2=0x7f99d019f0c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:39.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.911+0000 7f99d4fe9640 1 --2- 172.21.15.32:0/3540670235 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f99d0107520 0x7f99d019f0c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:39.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.911+0000 7f99d4fe9640 1 -- 172.21.15.32:0/3540670235 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f99d019f730 con 0x7f99d01022c0 2024-04-25T00:32:39.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.911+0000 7f99d57ea640 1 --2- 172.21.15.32:0/3540670235 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f99d0107520 0x7f99d019f0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:32:39.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.911+0000 7f99d4fe9640 1 --2- 172.21.15.32:0/3540670235 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f99d01022c0 0x7f99d0197b10 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 crypto rx=0x7f99c4009230 tx=0x7f99c4031b60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:39.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.911+0000 7f99cdffb640 1 -- 172.21.15.32:0/3540670235 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f99c403f030 con 0x7f99d01022c0 2024-04-25T00:32:39.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.911+0000 7f99cdffb640 1 -- 172.21.15.32:0/3540670235 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f99c4003ad0 con 0x7f99d01022c0 2024-04-25T00:32:39.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.911+0000 7f99d724d640 1 -- 172.21.15.32:0/3540670235 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f99d019f9c0 con 0x7f99d01022c0 2024-04-25T00:32:39.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.912+0000 7f99cdffb640 1 -- 172.21.15.32:0/3540670235 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f99c4035070 con 0x7f99d01022c0 2024-04-25T00:32:39.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.912+0000 7f99d724d640 1 -- 172.21.15.32:0/3540670235 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f99d019feb0 con 0x7f99d01022c0 2024-04-25T00:32:39.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.913+0000 7f99d724d640 1 -- 172.21.15.32:0/3540670235 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f99d00fd720 con 0x7f99d01022c0 2024-04-25T00:32:39.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.914+0000 7f99cdffb640 1 -- 172.21.15.32:0/3540670235 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f99c4042030 con 0x7f99d01022c0 2024-04-25T00:32:39.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.915+0000 7f99cdffb640 1 --2- 172.21.15.32:0/3540670235 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f99c00a2c20 0x7f99c00a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:39.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.915+0000 7f99cffff640 1 --2- 172.21.15.32:0/3540670235 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f99c00a2c20 0x7f99c00a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:39.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.915+0000 7f99cdffb640 1 -- 172.21.15.32:0/3540670235 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f99c40efb70 con 0x7f99d01022c0 2024-04-25T00:32:39.917 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.915+0000 7f99cffff640 1 --2- 172.21.15.32:0/3540670235 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f99c00a2c20 0x7f99c00a50e0 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7f99bc005fd0 tx=0x7f99bc005850 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:39.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:39.919+0000 7f99cdffb640 1 -- 172.21.15.32:0/3540670235 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f99c40bc610 con 0x7f99d01022c0 2024-04-25T00:32:40.144 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.143+0000 7f99d724d640 1 -- 172.21.15.32:0/3540670235 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f99d002d3b0 con 0x7f99d01022c0 2024-04-25T00:32:40.144 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.144+0000 7f99cdffb640 1 -- 172.21.15.32:0/3540670235 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) ==== 94+0+11876 (secure 0 0 0) 0x7f99c40bbf10 con 0x7f99d01022c0 2024-04-25T00:32:40.144 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:32:40.145 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14643,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6839/187492674","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6838","nonce":187492674},{"type":"v1","addr":"172.21.15.32:6839","nonce":187492674}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14655,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6841/1948785545","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6840","nonce":1948785545},{"type":"v1","addr":"172.21.15.32:6841","nonce":1948785545}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-25T00:30:47.866875+0000","modified":"2024-04-25T00:31:14.972514+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24506,"mds_1":24481,"mds_2":24476},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24443":{"gid":24443,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24445":{"gid":24445,"name":"f","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24457":{"gid":24457,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24476":{"gid":24476,"name":"h","rank":2,"incarnation":22,"state":"up:active","state_seq":12,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24481":{"gid":24481,"name":"i","rank":1,"incarnation":20,"state":"up:active","state_seq":11,"addr":"172.21.15.130:6839/4267125809","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":4267125809},{"type":"v1","addr":"172.21.15.130:6839","nonce":4267125809}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24506":{"gid":24506,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.121:6839/258444853","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6838","nonce":258444853},{"type":"v1","addr":"172.21.15.121:6839","nonce":258444853}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24506,"qdb_cluster":[24476,24481,24506]},"id":1}]} 2024-04-25T00:32:40.145 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 24 2024-04-25T00:32:40.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.146+0000 7f99d724d640 1 -- 172.21.15.32:0/3540670235 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f99c00a2c20 msgr2=0x7f99c00a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:40.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.146+0000 7f99d724d640 1 --2- 172.21.15.32:0/3540670235 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f99c00a2c20 0x7f99c00a50e0 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7f99bc005fd0 tx=0x7f99bc005850 comp rx=0 tx=0).stop 2024-04-25T00:32:40.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.146+0000 7f99d724d640 1 -- 172.21.15.32:0/3540670235 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f99d01022c0 msgr2=0x7f99d0197b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:40.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.146+0000 7f99d724d640 1 --2- 172.21.15.32:0/3540670235 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f99d01022c0 0x7f99d0197b10 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 crypto rx=0x7f99c4009230 tx=0x7f99c4031b60 comp rx=0 tx=0).stop 2024-04-25T00:32:40.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.146+0000 7f99d724d640 1 -- 172.21.15.32:0/3540670235 shutdown_connections 2024-04-25T00:32:40.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.146+0000 7f99d724d640 1 --2- 172.21.15.32:0/3540670235 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f99c00a2c20 0x7f99c00a50e0 unknown :-1 s=CLOSED pgs=227 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.146+0000 7f99d724d640 1 --2- 172.21.15.32:0/3540670235 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f99d0107520 0x7f99d019f0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.146+0000 7f99d724d640 1 --2- 172.21.15.32:0/3540670235 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f99d0104bf0 0x7f99d0198050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.146+0000 7f99d724d640 1 --2- 172.21.15.32:0/3540670235 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f99d01022c0 0x7f99d0197b10 unknown :-1 s=CLOSED pgs=357 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.148 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.146+0000 7f99d724d640 1 -- 172.21.15.32:0/3540670235 >> 172.21.15.32:0/3540670235 conn(0x7f99d00fbc30 msgr2=0x7f99d0102b60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:40.148 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.146+0000 7f99d724d640 1 -- 172.21.15.32:0/3540670235 shutdown_connections 2024-04-25T00:32:40.148 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.146+0000 7f99d724d640 1 -- 172.21.15.32:0/3540670235 wait complete. 2024-04-25T00:32:40.159 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-25T00:32:40.264 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.260+0000 7f7411f1b640 1 -- 172.21.15.32:0/905642139 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f740c102990 msgr2=0x7f740c102e10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:40.264 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.260+0000 7f7411f1b640 1 --2- 172.21.15.32:0/905642139 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f740c102990 0x7f740c102e10 secure :-1 s=READY pgs=358 cs=0 l=1 rev1=1 crypto rx=0x7f73f4009900 tx=0x7f73f402a410 comp rx=0 tx=0).stop 2024-04-25T00:32:40.264 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.260+0000 7f7411f1b640 1 -- 172.21.15.32:0/905642139 shutdown_connections 2024-04-25T00:32:40.264 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.260+0000 7f7411f1b640 1 --2- 172.21.15.32:0/905642139 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f740c103350 0x7f740c109be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.264 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.260+0000 7f7411f1b640 1 --2- 172.21.15.32:0/905642139 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f740c102990 0x7f740c102e10 unknown :-1 s=CLOSED pgs=358 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.264 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.260+0000 7f7411f1b640 1 --2- 172.21.15.32:0/905642139 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f740c1017a0 0x7f740c101bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.260+0000 7f7411f1b640 1 -- 172.21.15.32:0/905642139 >> 172.21.15.32:0/905642139 conn(0x7f740c0fccf0 msgr2=0x7f740c0ff150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:40.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.260+0000 7f7411f1b640 1 -- 172.21.15.32:0/905642139 shutdown_connections 2024-04-25T00:32:40.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.260+0000 7f7411f1b640 1 -- 172.21.15.32:0/905642139 wait complete. 2024-04-25T00:32:40.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.261+0000 7f7411f1b640 1 Processor -- start 2024-04-25T00:32:40.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.261+0000 7f7411f1b640 1 -- start start 2024-04-25T00:32:40.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.261+0000 7f7411f1b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f740c1017a0 0x7f740c0754f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:40.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.261+0000 7f7411f1b640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f740c102990 0x7f740c073b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:40.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.261+0000 7f7411f1b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f740c103350 0x7f740c074070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:40.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.261+0000 7f7411f1b640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f740c10d490 con 0x7f740c1017a0 2024-04-25T00:32:40.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.261+0000 7f7411f1b640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f740c10d310 con 0x7f740c103350 2024-04-25T00:32:40.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.261+0000 7f7411f1b640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f740c10d610 con 0x7f740c102990 2024-04-25T00:32:40.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.261+0000 7f740b7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f740c1017a0 0x7f740c0754f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:40.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.261+0000 7f740b7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f740c1017a0 0x7f740c0754f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40886/0 (socket says 172.21.15.32:40886) 2024-04-25T00:32:40.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.262+0000 7f740b7fe640 1 -- 172.21.15.32:0/3648278663 learned_addr learned my addr 172.21.15.32:0/3648278663 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:40.266 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.262+0000 7f740affd640 1 --2- 172.21.15.32:0/3648278663 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f740c102990 0x7f740c073b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:40.266 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.262+0000 7f740b7fe640 1 -- 172.21.15.32:0/3648278663 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f740c102990 msgr2=0x7f740c073b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:40.266 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.262+0000 7f740b7fe640 1 --2- 172.21.15.32:0/3648278663 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f740c102990 0x7f740c073b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.266 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.262+0000 7f740bfff640 1 --2- 172.21.15.32:0/3648278663 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f740c103350 0x7f740c074070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:40.266 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.262+0000 7f740b7fe640 1 -- 172.21.15.32:0/3648278663 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f740c103350 msgr2=0x7f740c074070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:40.266 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.262+0000 7f740b7fe640 1 --2- 172.21.15.32:0/3648278663 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f740c103350 0x7f740c074070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.266 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.262+0000 7f740b7fe640 1 -- 172.21.15.32:0/3648278663 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f740c074900 con 0x7f740c1017a0 2024-04-25T00:32:40.266 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.262+0000 7f740affd640 1 --2- 172.21.15.32:0/3648278663 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f740c102990 0x7f740c073b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:32:40.266 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.262+0000 7f740b7fe640 1 --2- 172.21.15.32:0/3648278663 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f740c1017a0 0x7f740c0754f0 secure :-1 s=READY pgs=359 cs=0 l=1 rev1=1 crypto rx=0x7f740400b450 tx=0x7f740400b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:40.266 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.262+0000 7f7408ff9640 1 -- 172.21.15.32:0/3648278663 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7404015020 con 0x7f740c1017a0 2024-04-25T00:32:40.266 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.262+0000 7f7408ff9640 1 -- 172.21.15.32:0/3648278663 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7404011d70 con 0x7f740c1017a0 2024-04-25T00:32:40.266 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.262+0000 7f7411f1b640 1 -- 172.21.15.32:0/3648278663 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f740c1a1c60 con 0x7f740c1017a0 2024-04-25T00:32:40.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.262+0000 7f7408ff9640 1 -- 172.21.15.32:0/3648278663 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f740400f340 con 0x7f740c1017a0 2024-04-25T00:32:40.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.262+0000 7f7411f1b640 1 -- 172.21.15.32:0/3648278663 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f740c1a21b0 con 0x7f740c1017a0 2024-04-25T00:32:40.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.263+0000 7f7411f1b640 1 -- 172.21.15.32:0/3648278663 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f740c108a60 con 0x7f740c1017a0 2024-04-25T00:32:40.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.264+0000 7f7408ff9640 1 -- 172.21.15.32:0/3648278663 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f7404018020 con 0x7f740c1017a0 2024-04-25T00:32:40.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.265+0000 7f7408ff9640 1 --2- 172.21.15.32:0/3648278663 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f73f80a2c20 0x7f73f80a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:40.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.265+0000 7f7408ff9640 1 -- 172.21.15.32:0/3648278663 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f74040ceff0 con 0x7f740c1017a0 2024-04-25T00:32:40.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.265+0000 7f740affd640 1 --2- 172.21.15.32:0/3648278663 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f73f80a2c20 0x7f73f80a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:40.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.265+0000 7f740affd640 1 --2- 172.21.15.32:0/3648278663 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f73f80a2c20 0x7f73f80a50e0 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f73f40096c0 tx=0x7f73f4031400 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:40.270 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.269+0000 7f7408ff9640 1 -- 172.21.15.32:0/3648278663 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f740401f020 con 0x7f740c1017a0 2024-04-25T00:32:40.456 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.456+0000 7f7411f1b640 1 -- 172.21.15.32:0/3648278663 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f740c02d3e0 con 0x7f740c1017a0 2024-04-25T00:32:40.458 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.457+0000 7f7408ff9640 1 -- 172.21.15.32:0/3648278663 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v91) ==== 74+0+24600 (secure 0 0 0) 0x7f740400d040 con 0x7f740c1017a0 2024-04-25T00:32:40.458 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:32:40.458 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":91,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:31:44.700076+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-25T00:30:46.239556+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-25T00:30:47.257182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2024-04-25T00:30:51.193975+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.6","mappings":[{"from":3,"to":6}]},{"pgid":"3.17","mappings":[{"from":4,"to":7}]},{"pgid":"3.1b","mappings":[{"from":5,"to":9}]},{"pgid":"3.20","mappings":[{"from":5,"to":9}]},{"pgid":"3.23","mappings":[{"from":4,"to":8}]},{"pgid":"3.2c","mappings":[{"from":4,"to":9}]},{"pgid":"4.c","mappings":[{"from":9,"to":10}]},{"pgid":"4.12","mappings":[{"from":9,"to":10}]},{"pgid":"4.1c","mappings":[{"from":9,"to":8}]},{"pgid":"4.1d","mappings":[{"from":5,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-num-failure-domains":"0","crush-osds-per-failure-domain":"0","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:32:40.460 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.460+0000 7f7411f1b640 1 -- 172.21.15.32:0/3648278663 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f73f80a2c20 msgr2=0x7f73f80a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:40.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.460+0000 7f7411f1b640 1 --2- 172.21.15.32:0/3648278663 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f73f80a2c20 0x7f73f80a50e0 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f73f40096c0 tx=0x7f73f4031400 comp rx=0 tx=0).stop 2024-04-25T00:32:40.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.460+0000 7f7411f1b640 1 -- 172.21.15.32:0/3648278663 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f740c1017a0 msgr2=0x7f740c0754f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:40.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.460+0000 7f7411f1b640 1 --2- 172.21.15.32:0/3648278663 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f740c1017a0 0x7f740c0754f0 secure :-1 s=READY pgs=359 cs=0 l=1 rev1=1 crypto rx=0x7f740400b450 tx=0x7f740400b910 comp rx=0 tx=0).stop 2024-04-25T00:32:40.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.460+0000 7f7411f1b640 1 -- 172.21.15.32:0/3648278663 shutdown_connections 2024-04-25T00:32:40.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.460+0000 7f7411f1b640 1 --2- 172.21.15.32:0/3648278663 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f73f80a2c20 0x7f73f80a50e0 unknown :-1 s=CLOSED pgs=228 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.460+0000 7f7411f1b640 1 --2- 172.21.15.32:0/3648278663 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f740c103350 0x7f740c074070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.460+0000 7f7411f1b640 1 --2- 172.21.15.32:0/3648278663 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f740c102990 0x7f740c073b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.460+0000 7f7411f1b640 1 --2- 172.21.15.32:0/3648278663 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f740c1017a0 0x7f740c0754f0 unknown :-1 s=CLOSED pgs=359 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.460+0000 7f7411f1b640 1 -- 172.21.15.32:0/3648278663 >> 172.21.15.32:0/3648278663 conn(0x7f740c0fccf0 msgr2=0x7f740c105c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:40.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.460+0000 7f7411f1b640 1 -- 172.21.15.32:0/3648278663 shutdown_connections 2024-04-25T00:32:40.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.460+0000 7f7411f1b640 1 -- 172.21.15.32:0/3648278663 wait complete. 2024-04-25T00:32:40.472 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-25T00:26:07.306839+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-04-25T00:30:46.239556+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '84', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-04-25T00:30:47.257182+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '85', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.05000000074505806}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_data_ec', 'create_time': '2024-04-25T00:30:51.193975+0000', 'flags': 5, 'flags_names': 'hashpspool,ec_overwrites', 'type': 3, 'size': 4, 'min_size': 3, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '89', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'cephfs_data_ec', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 8192, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}] 2024-04-25T00:32:40.472 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-25T00:32:40.582 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.579+0000 7fac9b34f640 1 -- 172.21.15.32:0/3889398112 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fac94104bb0 msgr2=0x7fac94106fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:40.582 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.579+0000 7fac9b34f640 1 --2- 172.21.15.32:0/3889398112 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fac94104bb0 0x7fac94106fa0 secure :-1 s=READY pgs=360 cs=0 l=1 rev1=1 crypto rx=0x7fac84009900 tx=0x7fac8402a410 comp rx=0 tx=0).stop 2024-04-25T00:32:40.583 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.579+0000 7fac9b34f640 1 -- 172.21.15.32:0/3889398112 shutdown_connections 2024-04-25T00:32:40.583 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.579+0000 7fac9b34f640 1 --2- 172.21.15.32:0/3889398112 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fac941074e0 0x7fac941098d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.583 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.579+0000 7fac9b34f640 1 --2- 172.21.15.32:0/3889398112 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fac94104bb0 0x7fac94106fa0 unknown :-1 s=CLOSED pgs=360 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.583 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.579+0000 7fac9b34f640 1 --2- 172.21.15.32:0/3889398112 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fac94102280 0x7fac94104670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.583 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.579+0000 7fac9b34f640 1 -- 172.21.15.32:0/3889398112 >> 172.21.15.32:0/3889398112 conn(0x7fac940fbc10 msgr2=0x7fac940fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:40.583 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.579+0000 7fac9b34f640 1 -- 172.21.15.32:0/3889398112 shutdown_connections 2024-04-25T00:32:40.583 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.579+0000 7fac9b34f640 1 -- 172.21.15.32:0/3889398112 wait complete. 2024-04-25T00:32:40.583 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.580+0000 7fac9b34f640 1 Processor -- start 2024-04-25T00:32:40.583 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.580+0000 7fac9b34f640 1 -- start start 2024-04-25T00:32:40.583 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.580+0000 7fac9b34f640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fac94102280 0x7fac94197ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:40.583 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.580+0000 7fac9b34f640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fac94104bb0 0x7fac94198020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:40.583 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.580+0000 7fac9b34f640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fac941074e0 0x7fac9419f090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:40.583 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.580+0000 7fac9b34f640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fac9410d410 con 0x7fac94102280 2024-04-25T00:32:40.583 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.580+0000 7fac9b34f640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fac9410d290 con 0x7fac94104bb0 2024-04-25T00:32:40.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.580+0000 7fac9b34f640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fac9410d590 con 0x7fac941074e0 2024-04-25T00:32:40.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.580+0000 7fac990eb640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fac94102280 0x7fac94197ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:40.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.580+0000 7fac990eb640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fac94102280 0x7fac94197ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40914/0 (socket says 172.21.15.32:40914) 2024-04-25T00:32:40.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.580+0000 7fac990eb640 1 -- 172.21.15.32:0/490558493 learned_addr learned my addr 172.21.15.32:0/490558493 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:40.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.580+0000 7fac998ec640 1 --2- 172.21.15.32:0/490558493 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fac941074e0 0x7fac9419f090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:40.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.580+0000 7fac988ea640 1 --2- 172.21.15.32:0/490558493 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fac94104bb0 0x7fac94198020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:40.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.580+0000 7fac990eb640 1 -- 172.21.15.32:0/490558493 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fac941074e0 msgr2=0x7fac9419f090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:40.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.580+0000 7fac990eb640 1 --2- 172.21.15.32:0/490558493 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fac941074e0 0x7fac9419f090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.580+0000 7fac990eb640 1 -- 172.21.15.32:0/490558493 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fac94104bb0 msgr2=0x7fac94198020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:40.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.580+0000 7fac990eb640 1 --2- 172.21.15.32:0/490558493 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fac94104bb0 0x7fac94198020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.580+0000 7fac990eb640 1 -- 172.21.15.32:0/490558493 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fac9419f700 con 0x7fac94102280 2024-04-25T00:32:40.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.581+0000 7fac990eb640 1 --2- 172.21.15.32:0/490558493 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fac94102280 0x7fac94197ae0 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 crypto rx=0x7fac9000d770 tx=0x7fac9000dc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:40.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.581+0000 7fac8a7fc640 1 -- 172.21.15.32:0/490558493 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fac90017070 con 0x7fac94102280 2024-04-25T00:32:40.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.581+0000 7fac8a7fc640 1 -- 172.21.15.32:0/490558493 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fac90005500 con 0x7fac94102280 2024-04-25T00:32:40.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.581+0000 7fac9b34f640 1 -- 172.21.15.32:0/490558493 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fac9419f9f0 con 0x7fac94102280 2024-04-25T00:32:40.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.581+0000 7fac8a7fc640 1 -- 172.21.15.32:0/490558493 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fac90004e00 con 0x7fac94102280 2024-04-25T00:32:40.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.581+0000 7fac9b34f640 1 -- 172.21.15.32:0/490558493 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fac9419fec0 con 0x7fac94102280 2024-04-25T00:32:40.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.582+0000 7fac9b34f640 1 -- 172.21.15.32:0/490558493 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fac940fd700 con 0x7fac94102280 2024-04-25T00:32:40.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.583+0000 7fac8a7fc640 1 -- 172.21.15.32:0/490558493 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7fac90004fa0 con 0x7fac94102280 2024-04-25T00:32:40.586 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.583+0000 7fac8a7fc640 1 --2- 172.21.15.32:0/490558493 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fac800a2c20 0x7fac800a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:40.586 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.584+0000 7fac988ea640 1 --2- 172.21.15.32:0/490558493 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fac800a2c20 0x7fac800a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:40.586 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.584+0000 7fac8a7fc640 1 -- 172.21.15.32:0/490558493 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7fac900cf0c0 con 0x7fac94102280 2024-04-25T00:32:40.586 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.584+0000 7fac988ea640 1 --2- 172.21.15.32:0/490558493 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fac800a2c20 0x7fac800a50e0 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7fac84006010 tx=0x7fac84000bc0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:40.588 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.588+0000 7fac8a7fc640 1 -- 172.21.15.32:0/490558493 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fac9009bb00 con 0x7fac94102280 2024-04-25T00:32:40.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:40 smithi032 ceph-mon[29357]: pgmap v349: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 7 op/s 2024-04-25T00:32:40.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:40 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/586742952' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:32:40.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:40 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1024939214' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:32:40.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:40 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3540670235' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:32:40.777 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.776+0000 7fac9b34f640 1 -- 172.21.15.32:0/490558493 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7fac94062340 con 0x7fac94102280 2024-04-25T00:32:40.777 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.776+0000 7fac8a7fc640 1 -- 172.21.15.32:0/490558493 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v91) ==== 93+0+10 (secure 0 0 0) 0x7fac9009b400 con 0x7fac94102280 2024-04-25T00:32:40.778 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 1 2024-04-25T00:32:40.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.779+0000 7fac9b34f640 1 -- 172.21.15.32:0/490558493 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fac800a2c20 msgr2=0x7fac800a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:40.780 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.779+0000 7fac9b34f640 1 --2- 172.21.15.32:0/490558493 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fac800a2c20 0x7fac800a50e0 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7fac84006010 tx=0x7fac84000bc0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.780 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.779+0000 7fac9b34f640 1 -- 172.21.15.32:0/490558493 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fac94102280 msgr2=0x7fac94197ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:40.780 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.779+0000 7fac9b34f640 1 --2- 172.21.15.32:0/490558493 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fac94102280 0x7fac94197ae0 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 crypto rx=0x7fac9000d770 tx=0x7fac9000dc30 comp rx=0 tx=0).stop 2024-04-25T00:32:40.780 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.779+0000 7fac9b34f640 1 -- 172.21.15.32:0/490558493 shutdown_connections 2024-04-25T00:32:40.780 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.779+0000 7fac9b34f640 1 --2- 172.21.15.32:0/490558493 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fac800a2c20 0x7fac800a50e0 unknown :-1 s=CLOSED pgs=229 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.780 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.779+0000 7fac9b34f640 1 --2- 172.21.15.32:0/490558493 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fac941074e0 0x7fac9419f090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.780 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.779+0000 7fac9b34f640 1 --2- 172.21.15.32:0/490558493 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fac94104bb0 0x7fac94198020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.781 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.779+0000 7fac9b34f640 1 --2- 172.21.15.32:0/490558493 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fac94102280 0x7fac94197ae0 unknown :-1 s=CLOSED pgs=361 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.781 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.779+0000 7fac9b34f640 1 -- 172.21.15.32:0/490558493 >> 172.21.15.32:0/490558493 conn(0x7fac940fbc10 msgr2=0x7fac94102bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:40.781 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.779+0000 7fac9b34f640 1 -- 172.21.15.32:0/490558493 shutdown_connections 2024-04-25T00:32:40.781 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.779+0000 7fac9b34f640 1 -- 172.21.15.32:0/490558493 wait complete. 2024-04-25T00:32:40.791 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-04-25T00:32:40.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:40 smithi121 ceph-mon[32231]: pgmap v349: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 7 op/s 2024-04-25T00:32:40.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:40 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/586742952' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:32:40.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:40 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1024939214' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:32:40.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:40 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3540670235' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:32:40.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:40 smithi130 ceph-mon[32525]: pgmap v349: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 7 op/s 2024-04-25T00:32:40.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:40 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/586742952' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:32:40.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:40 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1024939214' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:32:40.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:40 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3540670235' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:32:40.897 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.896+0000 7f016635a640 1 -- 172.21.15.32:0/1818683353 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01601017a0 msgr2=0x7f0160101bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:40.897 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.896+0000 7f016635a640 1 --2- 172.21.15.32:0/1818683353 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01601017a0 0x7f0160101bc0 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 crypto rx=0x7f0148009900 tx=0x7f014802a410 comp rx=0 tx=0).stop 2024-04-25T00:32:40.897 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.896+0000 7f016635a640 1 -- 172.21.15.32:0/1818683353 shutdown_connections 2024-04-25T00:32:40.897 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.896+0000 7f016635a640 1 --2- 172.21.15.32:0/1818683353 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0160103350 0x7f0160109be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.897 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.896+0000 7f016635a640 1 --2- 172.21.15.32:0/1818683353 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0160102990 0x7f0160102e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.897 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.896+0000 7f016635a640 1 --2- 172.21.15.32:0/1818683353 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01601017a0 0x7f0160101bc0 unknown :-1 s=CLOSED pgs=362 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.897 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.896+0000 7f016635a640 1 -- 172.21.15.32:0/1818683353 >> 172.21.15.32:0/1818683353 conn(0x7f01600fcd10 msgr2=0x7f01600ff150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:40.897 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.896+0000 7f016635a640 1 -- 172.21.15.32:0/1818683353 shutdown_connections 2024-04-25T00:32:40.897 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.896+0000 7f016635a640 1 -- 172.21.15.32:0/1818683353 wait complete. 2024-04-25T00:32:40.899 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.897+0000 7f016635a640 1 Processor -- start 2024-04-25T00:32:40.899 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.897+0000 7f016635a640 1 -- start start 2024-04-25T00:32:40.899 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.897+0000 7f016635a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01601017a0 0x7f0160197ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:40.899 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.897+0000 7f016635a640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0160102990 0x7f0160197ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:40.899 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.897+0000 7f016635a640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0160103350 0x7f016019f060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:40.899 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.897+0000 7f016635a640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f016010d410 con 0x7f01601017a0 2024-04-25T00:32:40.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.897+0000 7f016635a640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f016010d290 con 0x7f0160103350 2024-04-25T00:32:40.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.897+0000 7f016635a640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f016010d590 con 0x7f0160102990 2024-04-25T00:32:40.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.897+0000 7f015ffff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01601017a0 0x7f0160197ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:40.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.897+0000 7f015ffff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01601017a0 0x7f0160197ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40928/0 (socket says 172.21.15.32:40928) 2024-04-25T00:32:40.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.897+0000 7f015ffff640 1 -- 172.21.15.32:0/721197679 learned_addr learned my addr 172.21.15.32:0/721197679 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:40.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.897+0000 7f01648f7640 1 --2- 172.21.15.32:0/721197679 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0160103350 0x7f016019f060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:40.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.897+0000 7f015f7fe640 1 --2- 172.21.15.32:0/721197679 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0160102990 0x7f0160197ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:40.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.898+0000 7f015ffff640 1 -- 172.21.15.32:0/721197679 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0160102990 msgr2=0x7f0160197ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:40.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.898+0000 7f015ffff640 1 --2- 172.21.15.32:0/721197679 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0160102990 0x7f0160197ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.898+0000 7f015ffff640 1 -- 172.21.15.32:0/721197679 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0160103350 msgr2=0x7f016019f060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:40.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.898+0000 7f015ffff640 1 --2- 172.21.15.32:0/721197679 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0160103350 0x7f016019f060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:40.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.898+0000 7f015ffff640 1 -- 172.21.15.32:0/721197679 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f016019f6d0 con 0x7f01601017a0 2024-04-25T00:32:40.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.898+0000 7f015ffff640 1 --2- 172.21.15.32:0/721197679 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01601017a0 0x7f0160197ab0 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7f0148031610 tx=0x7f01480043b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:40.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.898+0000 7f015d7fa640 1 -- 172.21.15.32:0/721197679 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f014803f030 con 0x7f01601017a0 2024-04-25T00:32:40.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.898+0000 7f015d7fa640 1 -- 172.21.15.32:0/721197679 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0148003ce0 con 0x7f01601017a0 2024-04-25T00:32:40.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.898+0000 7f016635a640 1 -- 172.21.15.32:0/721197679 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f016019f960 con 0x7f01601017a0 2024-04-25T00:32:40.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.898+0000 7f015d7fa640 1 -- 172.21.15.32:0/721197679 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0148035070 con 0x7f01601017a0 2024-04-25T00:32:40.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.898+0000 7f016635a640 1 -- 172.21.15.32:0/721197679 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f016019fe50 con 0x7f01601017a0 2024-04-25T00:32:40.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.899+0000 7f016635a640 1 -- 172.21.15.32:0/721197679 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0160108b40 con 0x7f01601017a0 2024-04-25T00:32:40.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.900+0000 7f015d7fa640 1 -- 172.21.15.32:0/721197679 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f0148042030 con 0x7f01601017a0 2024-04-25T00:32:40.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.901+0000 7f015d7fa640 1 --2- 172.21.15.32:0/721197679 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f014c0a2c20 0x7f014c0a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:40.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.901+0000 7f015d7fa640 1 -- 172.21.15.32:0/721197679 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f01480efb80 con 0x7f01601017a0 2024-04-25T00:32:40.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.901+0000 7f015f7fe640 1 --2- 172.21.15.32:0/721197679 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f014c0a2c20 0x7f014c0a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:40.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.901+0000 7f015f7fe640 1 --2- 172.21.15.32:0/721197679 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f014c0a2c20 0x7f014c0a50e0 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f0150005fd0 tx=0x7f0150005e20 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:40.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:40.905+0000 7f015d7fa640 1 -- 172.21.15.32:0/721197679 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f01480bc620 con 0x7f01601017a0 2024-04-25T00:32:41.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.089+0000 7f016635a640 1 -- 172.21.15.32:0/721197679 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) -- 0x7f016002d3c0 con 0x7f01601017a0 2024-04-25T00:32:41.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.089+0000 7f015d7fa640 1 -- 172.21.15.32:0/721197679 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v91) ==== 104+0+11 (secure 0 0 0) 0x7f01480bbf20 con 0x7f01601017a0 2024-04-25T00:32:41.090 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 32 2024-04-25T00:32:41.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.091+0000 7f016635a640 1 -- 172.21.15.32:0/721197679 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f014c0a2c20 msgr2=0x7f014c0a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:41.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.091+0000 7f016635a640 1 --2- 172.21.15.32:0/721197679 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f014c0a2c20 0x7f014c0a50e0 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f0150005fd0 tx=0x7f0150005e20 comp rx=0 tx=0).stop 2024-04-25T00:32:41.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.091+0000 7f016635a640 1 -- 172.21.15.32:0/721197679 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01601017a0 msgr2=0x7f0160197ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:41.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.091+0000 7f016635a640 1 --2- 172.21.15.32:0/721197679 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01601017a0 0x7f0160197ab0 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7f0148031610 tx=0x7f01480043b0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.091+0000 7f016635a640 1 -- 172.21.15.32:0/721197679 shutdown_connections 2024-04-25T00:32:41.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.091+0000 7f016635a640 1 --2- 172.21.15.32:0/721197679 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f014c0a2c20 0x7f014c0a50e0 unknown :-1 s=CLOSED pgs=230 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.091+0000 7f016635a640 1 --2- 172.21.15.32:0/721197679 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0160103350 0x7f016019f060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.091+0000 7f016635a640 1 --2- 172.21.15.32:0/721197679 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0160102990 0x7f0160197ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.091+0000 7f016635a640 1 --2- 172.21.15.32:0/721197679 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f01601017a0 0x7f0160197ab0 unknown :-1 s=CLOSED pgs=363 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.091+0000 7f016635a640 1 -- 172.21.15.32:0/721197679 >> 172.21.15.32:0/721197679 conn(0x7f01600fcd10 msgr2=0x7f0160105d10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:41.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.091+0000 7f016635a640 1 -- 172.21.15.32:0/721197679 shutdown_connections 2024-04-25T00:32:41.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.091+0000 7f016635a640 1 -- 172.21.15.32:0/721197679 wait complete. 2024-04-25T00:32:41.103 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-04-25T00:32:41.210 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.207+0000 7eff243d5640 1 -- 172.21.15.32:0/2413194021 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff1c1006a0 msgr2=0x7eff1c100ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:41.210 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.207+0000 7eff243d5640 1 --2- 172.21.15.32:0/2413194021 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff1c1006a0 0x7eff1c100ac0 secure :-1 s=READY pgs=364 cs=0 l=1 rev1=1 crypto rx=0x7eff18009900 tx=0x7eff1802a410 comp rx=0 tx=0).stop 2024-04-25T00:32:41.210 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.207+0000 7eff243d5640 1 -- 172.21.15.32:0/2413194021 shutdown_connections 2024-04-25T00:32:41.210 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.207+0000 7eff243d5640 1 --2- 172.21.15.32:0/2413194021 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7eff1c102250 0x7eff1c108ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.210 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.207+0000 7eff243d5640 1 --2- 172.21.15.32:0/2413194021 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7eff1c101890 0x7eff1c101d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.211 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.207+0000 7eff243d5640 1 --2- 172.21.15.32:0/2413194021 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff1c1006a0 0x7eff1c100ac0 unknown :-1 s=CLOSED pgs=364 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.211 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.207+0000 7eff243d5640 1 -- 172.21.15.32:0/2413194021 >> 172.21.15.32:0/2413194021 conn(0x7eff1c0fbc30 msgr2=0x7eff1c0fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:41.211 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.207+0000 7eff243d5640 1 -- 172.21.15.32:0/2413194021 shutdown_connections 2024-04-25T00:32:41.211 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.207+0000 7eff243d5640 1 -- 172.21.15.32:0/2413194021 wait complete. 2024-04-25T00:32:41.211 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.208+0000 7eff243d5640 1 Processor -- start 2024-04-25T00:32:41.211 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.208+0000 7eff243d5640 1 -- start start 2024-04-25T00:32:41.211 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.208+0000 7eff243d5640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff1c1006a0 0x7eff1c197ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:41.211 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.208+0000 7eff243d5640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7eff1c101890 0x7eff1c198000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:41.211 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.208+0000 7eff243d5640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7eff1c102250 0x7eff1c19f070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:41.211 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.208+0000 7eff243d5640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7eff1c10d440 con 0x7eff1c1006a0 2024-04-25T00:32:41.211 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.208+0000 7eff243d5640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7eff1c10d2c0 con 0x7eff1c102250 2024-04-25T00:32:41.211 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.208+0000 7eff243d5640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7eff1c10d5c0 con 0x7eff1c101890 2024-04-25T00:32:41.211 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.208+0000 7eff22171640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff1c1006a0 0x7eff1c197ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:41.211 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.208+0000 7eff22171640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff1c1006a0 0x7eff1c197ac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40938/0 (socket says 172.21.15.32:40938) 2024-04-25T00:32:41.211 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.208+0000 7eff22171640 1 -- 172.21.15.32:0/1432688625 learned_addr learned my addr 172.21.15.32:0/1432688625 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:41.211 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.208+0000 7eff21970640 1 --2- 172.21.15.32:0/1432688625 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7eff1c101890 0x7eff1c198000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:41.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.208+0000 7eff22171640 1 -- 172.21.15.32:0/1432688625 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7eff1c101890 msgr2=0x7eff1c198000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:41.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.208+0000 7eff22171640 1 --2- 172.21.15.32:0/1432688625 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7eff1c101890 0x7eff1c198000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.208+0000 7eff22972640 1 --2- 172.21.15.32:0/1432688625 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7eff1c102250 0x7eff1c19f070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:41.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.208+0000 7eff22171640 1 -- 172.21.15.32:0/1432688625 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7eff1c102250 msgr2=0x7eff1c19f070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:41.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.208+0000 7eff22171640 1 --2- 172.21.15.32:0/1432688625 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7eff1c102250 0x7eff1c19f070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.208+0000 7eff22171640 1 -- 172.21.15.32:0/1432688625 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7eff1c19f6e0 con 0x7eff1c1006a0 2024-04-25T00:32:41.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.209+0000 7eff22171640 1 --2- 172.21.15.32:0/1432688625 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff1c1006a0 0x7eff1c197ac0 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7eff18031610 tx=0x7eff180043b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:41.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.209+0000 7eff137fe640 1 -- 172.21.15.32:0/1432688625 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7eff1803f030 con 0x7eff1c1006a0 2024-04-25T00:32:41.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.209+0000 7eff243d5640 1 -- 172.21.15.32:0/1432688625 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7eff1c19f970 con 0x7eff1c1006a0 2024-04-25T00:32:41.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.209+0000 7eff137fe640 1 -- 172.21.15.32:0/1432688625 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7eff18003ce0 con 0x7eff1c1006a0 2024-04-25T00:32:41.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.209+0000 7eff137fe640 1 -- 172.21.15.32:0/1432688625 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7eff18035070 con 0x7eff1c1006a0 2024-04-25T00:32:41.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.209+0000 7eff243d5640 1 -- 172.21.15.32:0/1432688625 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7eff1c19feb0 con 0x7eff1c1006a0 2024-04-25T00:32:41.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.210+0000 7eff243d5640 1 -- 172.21.15.32:0/1432688625 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7eff1c1079c0 con 0x7eff1c1006a0 2024-04-25T00:32:41.213 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.211+0000 7eff137fe640 1 -- 172.21.15.32:0/1432688625 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7eff18042030 con 0x7eff1c1006a0 2024-04-25T00:32:41.213 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.212+0000 7eff137fe640 1 --2- 172.21.15.32:0/1432688625 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7eff080a2c20 0x7eff080a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:41.213 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.212+0000 7eff137fe640 1 -- 172.21.15.32:0/1432688625 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7eff180efb80 con 0x7eff1c1006a0 2024-04-25T00:32:41.213 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.212+0000 7eff21970640 1 --2- 172.21.15.32:0/1432688625 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7eff080a2c20 0x7eff080a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:41.213 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.212+0000 7eff21970640 1 --2- 172.21.15.32:0/1432688625 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7eff080a2c20 0x7eff080a50e0 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7eff0c005fd0 tx=0x7eff0c005850 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:41.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.216+0000 7eff137fe640 1 -- 172.21.15.32:0/1432688625 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7eff180bc620 con 0x7eff1c1006a0 2024-04-25T00:32:41.403 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.403+0000 7eff243d5640 1 -- 172.21.15.32:0/1432688625 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) -- 0x7eff1c02d3d0 con 0x7eff1c1006a0 2024-04-25T00:32:41.403 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.403+0000 7eff137fe640 1 -- 172.21.15.32:0/1432688625 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v91) ==== 100+0+11 (secure 0 0 0) 0x7eff180bbf20 con 0x7eff1c1006a0 2024-04-25T00:32:41.404 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 64 2024-04-25T00:32:41.406 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.405+0000 7eff243d5640 1 -- 172.21.15.32:0/1432688625 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7eff080a2c20 msgr2=0x7eff080a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:41.406 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.405+0000 7eff243d5640 1 --2- 172.21.15.32:0/1432688625 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7eff080a2c20 0x7eff080a50e0 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7eff0c005fd0 tx=0x7eff0c005850 comp rx=0 tx=0).stop 2024-04-25T00:32:41.406 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.405+0000 7eff243d5640 1 -- 172.21.15.32:0/1432688625 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff1c1006a0 msgr2=0x7eff1c197ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:41.406 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.405+0000 7eff243d5640 1 --2- 172.21.15.32:0/1432688625 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff1c1006a0 0x7eff1c197ac0 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7eff18031610 tx=0x7eff180043b0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.406 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.405+0000 7eff243d5640 1 -- 172.21.15.32:0/1432688625 shutdown_connections 2024-04-25T00:32:41.406 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.405+0000 7eff243d5640 1 --2- 172.21.15.32:0/1432688625 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7eff080a2c20 0x7eff080a50e0 unknown :-1 s=CLOSED pgs=231 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.406 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.405+0000 7eff243d5640 1 --2- 172.21.15.32:0/1432688625 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7eff1c102250 0x7eff1c19f070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.406 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.405+0000 7eff243d5640 1 --2- 172.21.15.32:0/1432688625 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7eff1c101890 0x7eff1c198000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.406 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.405+0000 7eff243d5640 1 --2- 172.21.15.32:0/1432688625 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff1c1006a0 0x7eff1c197ac0 unknown :-1 s=CLOSED pgs=365 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.407 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.405+0000 7eff243d5640 1 -- 172.21.15.32:0/1432688625 >> 172.21.15.32:0/1432688625 conn(0x7eff1c0fbc30 msgr2=0x7eff1c104b90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:41.407 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.405+0000 7eff243d5640 1 -- 172.21.15.32:0/1432688625 shutdown_connections 2024-04-25T00:32:41.407 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.405+0000 7eff243d5640 1 -- 172.21.15.32:0/1432688625 wait complete. 2024-04-25T00:32:41.417 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data_ec pg_num 2024-04-25T00:32:41.529 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.524+0000 7fe9c622c640 1 -- 172.21.15.32:0/4125987475 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe9c0102280 msgr2=0x7fe9c0104670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:41.530 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.524+0000 7fe9c622c640 1 --2- 172.21.15.32:0/4125987475 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe9c0102280 0x7fe9c0104670 secure :-1 s=READY pgs=366 cs=0 l=1 rev1=1 crypto rx=0x7fe9b4009900 tx=0x7fe9b402a410 comp rx=0 tx=0).stop 2024-04-25T00:32:41.530 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.524+0000 7fe9c622c640 1 -- 172.21.15.32:0/4125987475 shutdown_connections 2024-04-25T00:32:41.530 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.524+0000 7fe9c622c640 1 --2- 172.21.15.32:0/4125987475 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe9c01074e0 0x7fe9c01098d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.530 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.524+0000 7fe9c622c640 1 --2- 172.21.15.32:0/4125987475 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe9c0104bb0 0x7fe9c0106fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.530 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.524+0000 7fe9c622c640 1 --2- 172.21.15.32:0/4125987475 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe9c0102280 0x7fe9c0104670 unknown :-1 s=CLOSED pgs=366 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.530 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.524+0000 7fe9c622c640 1 -- 172.21.15.32:0/4125987475 >> 172.21.15.32:0/4125987475 conn(0x7fe9c00fbc30 msgr2=0x7fe9c00fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:41.530 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.524+0000 7fe9c622c640 1 -- 172.21.15.32:0/4125987475 shutdown_connections 2024-04-25T00:32:41.530 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.524+0000 7fe9c622c640 1 -- 172.21.15.32:0/4125987475 wait complete. 2024-04-25T00:32:41.530 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.524+0000 7fe9c622c640 1 Processor -- start 2024-04-25T00:32:41.530 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.525+0000 7fe9c622c640 1 -- start start 2024-04-25T00:32:41.530 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.525+0000 7fe9c622c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe9c0102280 0x7fe9c0197b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:41.530 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.525+0000 7fe9c622c640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe9c0104bb0 0x7fe9c01980a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:41.531 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.525+0000 7fe9c622c640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe9c01074e0 0x7fe9c019f110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:41.531 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.525+0000 7fe9c622c640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fe9c010d4b0 con 0x7fe9c0102280 2024-04-25T00:32:41.531 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.525+0000 7fe9c622c640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fe9c010d330 con 0x7fe9c0104bb0 2024-04-25T00:32:41.531 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.525+0000 7fe9c622c640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fe9c010d630 con 0x7fe9c01074e0 2024-04-25T00:32:41.531 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.525+0000 7fe9bf7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe9c0102280 0x7fe9c0197b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:41.531 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.525+0000 7fe9bf7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe9c0102280 0x7fe9c0197b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40946/0 (socket says 172.21.15.32:40946) 2024-04-25T00:32:41.531 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.525+0000 7fe9bf7fe640 1 -- 172.21.15.32:0/3001377517 learned_addr learned my addr 172.21.15.32:0/3001377517 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:41.531 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.525+0000 7fe9bffff640 1 --2- 172.21.15.32:0/3001377517 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe9c01074e0 0x7fe9c019f110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:41.531 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.525+0000 7fe9beffd640 1 --2- 172.21.15.32:0/3001377517 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe9c0104bb0 0x7fe9c01980a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:41.531 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.525+0000 7fe9bf7fe640 1 -- 172.21.15.32:0/3001377517 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe9c01074e0 msgr2=0x7fe9c019f110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:41.531 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.525+0000 7fe9bf7fe640 1 --2- 172.21.15.32:0/3001377517 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe9c01074e0 0x7fe9c019f110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.531 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.525+0000 7fe9bf7fe640 1 -- 172.21.15.32:0/3001377517 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe9c0104bb0 msgr2=0x7fe9c01980a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:41.531 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.525+0000 7fe9bf7fe640 1 --2- 172.21.15.32:0/3001377517 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe9c0104bb0 0x7fe9c01980a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.532 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.525+0000 7fe9bf7fe640 1 -- 172.21.15.32:0/3001377517 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe9c019f780 con 0x7fe9c0102280 2024-04-25T00:32:41.532 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.525+0000 7fe9bf7fe640 1 --2- 172.21.15.32:0/3001377517 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe9c0102280 0x7fe9c0197b60 secure :-1 s=READY pgs=367 cs=0 l=1 rev1=1 crypto rx=0x7fe9b4009230 tx=0x7fe9b4004360 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:41.532 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.525+0000 7fe9bcff9640 1 -- 172.21.15.32:0/3001377517 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe9b403f030 con 0x7fe9c0102280 2024-04-25T00:32:41.532 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.526+0000 7fe9c622c640 1 -- 172.21.15.32:0/3001377517 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe9c019fa10 con 0x7fe9c0102280 2024-04-25T00:32:41.532 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.526+0000 7fe9bcff9640 1 -- 172.21.15.32:0/3001377517 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe9b4003ce0 con 0x7fe9c0102280 2024-04-25T00:32:41.532 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.526+0000 7fe9bcff9640 1 -- 172.21.15.32:0/3001377517 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe9b4035070 con 0x7fe9c0102280 2024-04-25T00:32:41.532 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.526+0000 7fe9c622c640 1 -- 172.21.15.32:0/3001377517 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe9c019ffd0 con 0x7fe9c0102280 2024-04-25T00:32:41.532 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.527+0000 7fe9c622c640 1 -- 172.21.15.32:0/3001377517 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe9c00fd720 con 0x7fe9c0102280 2024-04-25T00:32:41.532 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.528+0000 7fe9bcff9640 1 -- 172.21.15.32:0/3001377517 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7fe9b4042030 con 0x7fe9c0102280 2024-04-25T00:32:41.532 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.528+0000 7fe9bcff9640 1 --2- 172.21.15.32:0/3001377517 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe9a80a2c20 0x7fe9a80a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:41.532 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.529+0000 7fe9bcff9640 1 -- 172.21.15.32:0/3001377517 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7fe9b40f0000 con 0x7fe9c0102280 2024-04-25T00:32:41.532 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.529+0000 7fe9beffd640 1 --2- 172.21.15.32:0/3001377517 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe9a80a2c20 0x7fe9a80a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:41.532 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.529+0000 7fe9beffd640 1 --2- 172.21.15.32:0/3001377517 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe9a80a2c20 0x7fe9a80a50e0 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7fe9c0199070 tx=0x7fe9b0008040 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:41.533 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.533+0000 7fe9bcff9640 1 -- 172.21.15.32:0/3001377517 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fe9b40bcaa0 con 0x7fe9c0102280 2024-04-25T00:32:41.719 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.718+0000 7fe9c622c640 1 -- 172.21.15.32:0/3001377517 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"} v 0) -- 0x7fe9c002d3c0 con 0x7fe9c0102280 2024-04-25T00:32:41.719 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.719+0000 7fe9bcff9640 1 -- 172.21.15.32:0/3001377517 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]=0 v91) ==== 103+0+11 (secure 0 0 0) 0x7fe9b40bc3a0 con 0x7fe9c0102280 2024-04-25T00:32:41.719 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 32 2024-04-25T00:32:41.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.721+0000 7fe9c622c640 1 -- 172.21.15.32:0/3001377517 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe9a80a2c20 msgr2=0x7fe9a80a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:41.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.721+0000 7fe9c622c640 1 --2- 172.21.15.32:0/3001377517 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe9a80a2c20 0x7fe9a80a50e0 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7fe9c0199070 tx=0x7fe9b0008040 comp rx=0 tx=0).stop 2024-04-25T00:32:41.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.721+0000 7fe9c622c640 1 -- 172.21.15.32:0/3001377517 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe9c0102280 msgr2=0x7fe9c0197b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:41.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.721+0000 7fe9c622c640 1 --2- 172.21.15.32:0/3001377517 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe9c0102280 0x7fe9c0197b60 secure :-1 s=READY pgs=367 cs=0 l=1 rev1=1 crypto rx=0x7fe9b4009230 tx=0x7fe9b4004360 comp rx=0 tx=0).stop 2024-04-25T00:32:41.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.721+0000 7fe9c622c640 1 -- 172.21.15.32:0/3001377517 shutdown_connections 2024-04-25T00:32:41.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.721+0000 7fe9c622c640 1 --2- 172.21.15.32:0/3001377517 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe9a80a2c20 0x7fe9a80a50e0 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.721+0000 7fe9c622c640 1 --2- 172.21.15.32:0/3001377517 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe9c01074e0 0x7fe9c019f110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.721+0000 7fe9c622c640 1 --2- 172.21.15.32:0/3001377517 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe9c0104bb0 0x7fe9c01980a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.721+0000 7fe9c622c640 1 --2- 172.21.15.32:0/3001377517 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe9c0102280 0x7fe9c0197b60 unknown :-1 s=CLOSED pgs=367 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.721+0000 7fe9c622c640 1 -- 172.21.15.32:0/3001377517 >> 172.21.15.32:0/3001377517 conn(0x7fe9c00fbc30 msgr2=0x7fe9c0102cc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:41.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.721+0000 7fe9c622c640 1 -- 172.21.15.32:0/3001377517 shutdown_connections 2024-04-25T00:32:41.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.721+0000 7fe9c622c640 1 -- 172.21.15.32:0/3001377517 wait complete. 2024-04-25T00:32:41.732 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:41 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3648278663' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:32:41.732 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:41 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/490558493' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:32:41.732 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:41 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/721197679' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:32:41.732 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:41 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1432688625' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:32:41.733 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:32:41.839 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.836+0000 7f3b17929640 1 -- 172.21.15.32:0/152107555 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3b10102250 msgr2=0x7f3b10108ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:41.840 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.836+0000 7f3b17929640 1 --2- 172.21.15.32:0/152107555 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3b10102250 0x7f3b10108ae0 secure :-1 s=READY pgs=368 cs=0 l=1 rev1=1 crypto rx=0x7f3b08009ca0 tx=0x7f3b0802a8c0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.840 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.836+0000 7f3b17929640 1 -- 172.21.15.32:0/152107555 shutdown_connections 2024-04-25T00:32:41.840 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.836+0000 7f3b17929640 1 --2- 172.21.15.32:0/152107555 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3b10102250 0x7f3b10108ae0 unknown :-1 s=CLOSED pgs=368 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.840 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.836+0000 7f3b17929640 1 --2- 172.21.15.32:0/152107555 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3b10101890 0x7f3b10101d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.840 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.836+0000 7f3b17929640 1 --2- 172.21.15.32:0/152107555 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3b101006a0 0x7f3b10100ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.840 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.836+0000 7f3b17929640 1 -- 172.21.15.32:0/152107555 >> 172.21.15.32:0/152107555 conn(0x7f3b100fbc30 msgr2=0x7f3b100fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:41.840 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.836+0000 7f3b17929640 1 -- 172.21.15.32:0/152107555 shutdown_connections 2024-04-25T00:32:41.840 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.836+0000 7f3b17929640 1 -- 172.21.15.32:0/152107555 wait complete. 2024-04-25T00:32:41.840 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.836+0000 7f3b17929640 1 Processor -- start 2024-04-25T00:32:41.840 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.837+0000 7f3b17929640 1 -- start start 2024-04-25T00:32:41.840 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.837+0000 7f3b17929640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3b101006a0 0x7f3b101979b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:41.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.837+0000 7f3b17929640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3b10101890 0x7f3b10197ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:41.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.837+0000 7f3b17929640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3b10102250 0x7f3b1019ef60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:41.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.837+0000 7f3b17929640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f3b1010d4a0 con 0x7f3b10101890 2024-04-25T00:32:41.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.837+0000 7f3b17929640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f3b1010d320 con 0x7f3b101006a0 2024-04-25T00:32:41.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.837+0000 7f3b17929640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f3b1010d620 con 0x7f3b10102250 2024-04-25T00:32:41.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.837+0000 7f3b14ec4640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3b10101890 0x7f3b10197ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:41.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.837+0000 7f3b14ec4640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3b10101890 0x7f3b10197ef0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:47750/0 (socket says 172.21.15.32:47750) 2024-04-25T00:32:41.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.837+0000 7f3b14ec4640 1 -- 172.21.15.32:0/1176865102 learned_addr learned my addr 172.21.15.32:0/1176865102 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:41.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.837+0000 7f3b156c5640 1 --2- 172.21.15.32:0/1176865102 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3b101006a0 0x7f3b101979b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:41.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.837+0000 7f3b15ec6640 1 --2- 172.21.15.32:0/1176865102 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3b10102250 0x7f3b1019ef60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:41.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.837+0000 7f3b14ec4640 1 -- 172.21.15.32:0/1176865102 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3b10102250 msgr2=0x7f3b1019ef60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:41.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.837+0000 7f3b14ec4640 1 --2- 172.21.15.32:0/1176865102 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3b10102250 0x7f3b1019ef60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.837+0000 7f3b14ec4640 1 -- 172.21.15.32:0/1176865102 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3b101006a0 msgr2=0x7f3b101979b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:41.842 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.837+0000 7f3b14ec4640 1 --2- 172.21.15.32:0/1176865102 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3b101006a0 0x7f3b101979b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:41.842 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.837+0000 7f3b14ec4640 1 -- 172.21.15.32:0/1176865102 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3b1019f5d0 con 0x7f3b10101890 2024-04-25T00:32:41.842 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.837+0000 7f3b14ec4640 1 --2- 172.21.15.32:0/1176865102 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3b10101890 0x7f3b10197ef0 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 crypto rx=0x7f3b0000c970 tx=0x7f3b0000ce30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:41.842 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.837+0000 7f3b067fc640 1 -- 172.21.15.32:0/1176865102 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3b00007bb0 con 0x7f3b10101890 2024-04-25T00:32:41.842 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.838+0000 7f3b17929640 1 -- 172.21.15.32:0/1176865102 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3b1019f8c0 con 0x7f3b10101890 2024-04-25T00:32:41.842 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.838+0000 7f3b067fc640 1 -- 172.21.15.32:0/1176865102 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3b00007d50 con 0x7f3b10101890 2024-04-25T00:32:41.842 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.838+0000 7f3b067fc640 1 -- 172.21.15.32:0/1176865102 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3b0000f450 con 0x7f3b10101890 2024-04-25T00:32:41.842 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.838+0000 7f3b17929640 1 -- 172.21.15.32:0/1176865102 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f3b1019fe10 con 0x7f3b10101890 2024-04-25T00:32:41.842 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.838+0000 7f3b17929640 1 -- 172.21.15.32:0/1176865102 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f3b101078f0 con 0x7f3b10101890 2024-04-25T00:32:41.843 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:41 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3648278663' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:32:41.843 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:41 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/490558493' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:32:41.843 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:41 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/721197679' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:32:41.843 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:41 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1432688625' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:32:41.843 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.840+0000 7f3b067fc640 1 -- 172.21.15.32:0/1176865102 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f3b00004fe0 con 0x7f3b10101890 2024-04-25T00:32:41.844 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.841+0000 7f3b067fc640 1 --2- 172.21.15.32:0/1176865102 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3afc0a2c20 0x7f3afc0a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:41.844 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.841+0000 7f3b156c5640 1 --2- 172.21.15.32:0/1176865102 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3afc0a2c20 0x7f3afc0a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:41.844 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.841+0000 7f3b067fc640 1 -- 172.21.15.32:0/1176865102 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f3b000cf650 con 0x7f3b10101890 2024-04-25T00:32:41.844 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.842+0000 7f3b156c5640 1 --2- 172.21.15.32:0/1176865102 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3afc0a2c20 0x7f3afc0a50e0 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f3b101016f0 tx=0x7f3b0c005f90 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:41.845 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:41.845+0000 7f3b067fc640 1 -- 172.21.15.32:0/1176865102 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f3b00014030 con 0x7f3b10101890 2024-04-25T00:32:41.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:41 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3648278663' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:32:41.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:41 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/490558493' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:32:41.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:41 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/721197679' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:32:41.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:41 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1432688625' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:32:42.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.068+0000 7f3b17929640 1 -- 172.21.15.32:0/1176865102 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f3b1002d330 con 0x7f3b10101890 2024-04-25T00:32:42.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.069+0000 7f3b067fc640 1 -- 172.21.15.32:0/1176865102 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) ==== 94+0+11876 (secure 0 0 0) 0x7f3b0009baa0 con 0x7f3b10101890 2024-04-25T00:32:42.070 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:32:42.070 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14643,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6839/187492674","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6838","nonce":187492674},{"type":"v1","addr":"172.21.15.32:6839","nonce":187492674}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14655,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6841/1948785545","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6840","nonce":1948785545},{"type":"v1","addr":"172.21.15.32:6841","nonce":1948785545}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-25T00:30:47.866875+0000","modified":"2024-04-25T00:31:14.972514+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24506,"mds_1":24481,"mds_2":24476},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24443":{"gid":24443,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24445":{"gid":24445,"name":"f","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24457":{"gid":24457,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24476":{"gid":24476,"name":"h","rank":2,"incarnation":22,"state":"up:active","state_seq":12,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24481":{"gid":24481,"name":"i","rank":1,"incarnation":20,"state":"up:active","state_seq":11,"addr":"172.21.15.130:6839/4267125809","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":4267125809},{"type":"v1","addr":"172.21.15.130:6839","nonce":4267125809}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24506":{"gid":24506,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.121:6839/258444853","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6838","nonce":258444853},{"type":"v1","addr":"172.21.15.121:6839","nonce":258444853}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24506,"qdb_cluster":[24476,24481,24506]},"id":1}]} 2024-04-25T00:32:42.070 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 24 2024-04-25T00:32:42.071 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.071+0000 7f3b17929640 1 -- 172.21.15.32:0/1176865102 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3afc0a2c20 msgr2=0x7f3afc0a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:42.072 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.071+0000 7f3b17929640 1 --2- 172.21.15.32:0/1176865102 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3afc0a2c20 0x7f3afc0a50e0 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f3b101016f0 tx=0x7f3b0c005f90 comp rx=0 tx=0).stop 2024-04-25T00:32:42.072 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.071+0000 7f3b17929640 1 -- 172.21.15.32:0/1176865102 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3b10101890 msgr2=0x7f3b10197ef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:42.072 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.071+0000 7f3b17929640 1 --2- 172.21.15.32:0/1176865102 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3b10101890 0x7f3b10197ef0 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 crypto rx=0x7f3b0000c970 tx=0x7f3b0000ce30 comp rx=0 tx=0).stop 2024-04-25T00:32:42.072 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.071+0000 7f3b17929640 1 -- 172.21.15.32:0/1176865102 shutdown_connections 2024-04-25T00:32:42.072 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.071+0000 7f3b17929640 1 --2- 172.21.15.32:0/1176865102 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3afc0a2c20 0x7f3afc0a50e0 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.072 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.071+0000 7f3b17929640 1 --2- 172.21.15.32:0/1176865102 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3b10102250 0x7f3b1019ef60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.072 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.071+0000 7f3b17929640 1 --2- 172.21.15.32:0/1176865102 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3b10101890 0x7f3b10197ef0 unknown :-1 s=CLOSED pgs=369 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.072 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.071+0000 7f3b17929640 1 --2- 172.21.15.32:0/1176865102 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3b101006a0 0x7f3b101979b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.072 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.071+0000 7f3b17929640 1 -- 172.21.15.32:0/1176865102 >> 172.21.15.32:0/1176865102 conn(0x7f3b100fbc30 msgr2=0x7f3b10104ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:42.072 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.071+0000 7f3b17929640 1 -- 172.21.15.32:0/1176865102 shutdown_connections 2024-04-25T00:32:42.072 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.071+0000 7f3b17929640 1 -- 172.21.15.32:0/1176865102 wait complete. 2024-04-25T00:32:42.083 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-25T00:32:42.191 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.185+0000 7fe1eada2640 1 -- 172.21.15.32:0/322034812 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe1e4103340 msgr2=0x7fe1e4109bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:42.191 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.185+0000 7fe1eada2640 1 --2- 172.21.15.32:0/322034812 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe1e4103340 0x7fe1e4109bd0 secure :-1 s=READY pgs=370 cs=0 l=1 rev1=1 crypto rx=0x7fe1dc009ca0 tx=0x7fe1dc02a8c0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.191 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.185+0000 7fe1eada2640 1 -- 172.21.15.32:0/322034812 shutdown_connections 2024-04-25T00:32:42.191 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.185+0000 7fe1eada2640 1 --2- 172.21.15.32:0/322034812 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe1e4103340 0x7fe1e4109bd0 unknown :-1 s=CLOSED pgs=370 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.185+0000 7fe1eada2640 1 --2- 172.21.15.32:0/322034812 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe1e4102980 0x7fe1e4102e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.185+0000 7fe1eada2640 1 --2- 172.21.15.32:0/322034812 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe1e4101790 0x7fe1e4101bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.185+0000 7fe1eada2640 1 -- 172.21.15.32:0/322034812 >> 172.21.15.32:0/322034812 conn(0x7fe1e40fcd20 msgr2=0x7fe1e40ff140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:42.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.185+0000 7fe1eada2640 1 -- 172.21.15.32:0/322034812 shutdown_connections 2024-04-25T00:32:42.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.185+0000 7fe1eada2640 1 -- 172.21.15.32:0/322034812 wait complete. 2024-04-25T00:32:42.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.186+0000 7fe1eada2640 1 Processor -- start 2024-04-25T00:32:42.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.186+0000 7fe1eada2640 1 -- start start 2024-04-25T00:32:42.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.186+0000 7fe1eada2640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe1e4101790 0x7fe1e4197a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.186+0000 7fe1eada2640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe1e4102980 0x7fe1e4197f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.186+0000 7fe1eada2640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe1e4103340 0x7fe1e419efb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.186+0000 7fe1eada2640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fe1e410d3f0 con 0x7fe1e4103340 2024-04-25T00:32:42.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.186+0000 7fe1eada2640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fe1e410d270 con 0x7fe1e4101790 2024-04-25T00:32:42.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.186+0000 7fe1eada2640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fe1e410d570 con 0x7fe1e4102980 2024-04-25T00:32:42.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.186+0000 7fe1e933f640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe1e4103340 0x7fe1e419efb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:42.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.186+0000 7fe1e933f640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe1e4103340 0x7fe1e419efb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:47770/0 (socket says 172.21.15.32:47770) 2024-04-25T00:32:42.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.186+0000 7fe1e933f640 1 -- 172.21.15.32:0/2413066454 learned_addr learned my addr 172.21.15.32:0/2413066454 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:42.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.186+0000 7fe1e8b3e640 1 --2- 172.21.15.32:0/2413066454 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe1e4101790 0x7fe1e4197a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:42.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.186+0000 7fe1e3fff640 1 --2- 172.21.15.32:0/2413066454 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe1e4102980 0x7fe1e4197f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:42.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.186+0000 7fe1e933f640 1 -- 172.21.15.32:0/2413066454 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe1e4102980 msgr2=0x7fe1e4197f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:42.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.186+0000 7fe1e933f640 1 --2- 172.21.15.32:0/2413066454 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe1e4102980 0x7fe1e4197f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.186+0000 7fe1e933f640 1 -- 172.21.15.32:0/2413066454 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe1e4101790 msgr2=0x7fe1e4197a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:42.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.186+0000 7fe1e933f640 1 --2- 172.21.15.32:0/2413066454 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe1e4101790 0x7fe1e4197a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.186+0000 7fe1e933f640 1 -- 172.21.15.32:0/2413066454 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe1e419f620 con 0x7fe1e4103340 2024-04-25T00:32:42.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.186+0000 7fe1e3fff640 1 --2- 172.21.15.32:0/2413066454 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe1e4102980 0x7fe1e4197f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:32:42.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.186+0000 7fe1e8b3e640 1 --2- 172.21.15.32:0/2413066454 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe1e4101790 0x7fe1e4197a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:32:42.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.187+0000 7fe1e933f640 1 --2- 172.21.15.32:0/2413066454 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe1e4103340 0x7fe1e419efb0 secure :-1 s=READY pgs=371 cs=0 l=1 rev1=1 crypto rx=0x7fe1dc005e30 tx=0x7fe1dc031ee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:42.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.187+0000 7fe1e1ffb640 1 -- 172.21.15.32:0/2413066454 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe1dc003c30 con 0x7fe1e4103340 2024-04-25T00:32:42.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.187+0000 7fe1eada2640 1 -- 172.21.15.32:0/2413066454 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe1e419f8b0 con 0x7fe1e4103340 2024-04-25T00:32:42.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.187+0000 7fe1e1ffb640 1 -- 172.21.15.32:0/2413066454 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe1dc003dd0 con 0x7fe1e4103340 2024-04-25T00:32:42.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.187+0000 7fe1e1ffb640 1 -- 172.21.15.32:0/2413066454 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe1dc02fde0 con 0x7fe1e4103340 2024-04-25T00:32:42.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.187+0000 7fe1eada2640 1 -- 172.21.15.32:0/2413066454 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe1e419fe00 con 0x7fe1e4103340 2024-04-25T00:32:42.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.188+0000 7fe1eada2640 1 -- 172.21.15.32:0/2413066454 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe1e4108a50 con 0x7fe1e4103340 2024-04-25T00:32:42.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.189+0000 7fe1e1ffb640 1 -- 172.21.15.32:0/2413066454 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7fe1dc004050 con 0x7fe1e4103340 2024-04-25T00:32:42.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.190+0000 7fe1e1ffb640 1 --2- 172.21.15.32:0/2413066454 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe1d40a2c20 0x7fe1d40a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.190+0000 7fe1e8b3e640 1 --2- 172.21.15.32:0/2413066454 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe1d40a2c20 0x7fe1d40a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:42.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.190+0000 7fe1e1ffb640 1 -- 172.21.15.32:0/2413066454 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7fe1dc0f0840 con 0x7fe1e4103340 2024-04-25T00:32:42.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.190+0000 7fe1e8b3e640 1 --2- 172.21.15.32:0/2413066454 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe1d40a2c20 0x7fe1d40a50e0 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7fe1d8005fd0 tx=0x7fe1d800b410 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:42.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.194+0000 7fe1e1ffb640 1 -- 172.21.15.32:0/2413066454 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fe1dc0bd3f0 con 0x7fe1e4103340 2024-04-25T00:32:42.383 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.383+0000 7fe1eada2640 1 -- 172.21.15.32:0/2413066454 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fe1e4062340 con 0x7fe1e4103340 2024-04-25T00:32:42.385 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.385+0000 7fe1e1ffb640 1 -- 172.21.15.32:0/2413066454 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v91) ==== 74+0+24600 (secure 0 0 0) 0x7fe1dc0bccf0 con 0x7fe1e4103340 2024-04-25T00:32:42.386 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:32:42.386 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":91,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:31:44.700076+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-25T00:30:46.239556+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-25T00:30:47.257182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2024-04-25T00:30:51.193975+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.6","mappings":[{"from":3,"to":6}]},{"pgid":"3.17","mappings":[{"from":4,"to":7}]},{"pgid":"3.1b","mappings":[{"from":5,"to":9}]},{"pgid":"3.20","mappings":[{"from":5,"to":9}]},{"pgid":"3.23","mappings":[{"from":4,"to":8}]},{"pgid":"3.2c","mappings":[{"from":4,"to":9}]},{"pgid":"4.c","mappings":[{"from":9,"to":10}]},{"pgid":"4.12","mappings":[{"from":9,"to":10}]},{"pgid":"4.1c","mappings":[{"from":9,"to":8}]},{"pgid":"4.1d","mappings":[{"from":5,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-num-failure-domains":"0","crush-osds-per-failure-domain":"0","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:32:42.388 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.387+0000 7fe1eada2640 1 -- 172.21.15.32:0/2413066454 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe1d40a2c20 msgr2=0x7fe1d40a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:42.389 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.387+0000 7fe1eada2640 1 --2- 172.21.15.32:0/2413066454 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe1d40a2c20 0x7fe1d40a50e0 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7fe1d8005fd0 tx=0x7fe1d800b410 comp rx=0 tx=0).stop 2024-04-25T00:32:42.389 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.387+0000 7fe1eada2640 1 -- 172.21.15.32:0/2413066454 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe1e4103340 msgr2=0x7fe1e419efb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:42.389 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.387+0000 7fe1eada2640 1 --2- 172.21.15.32:0/2413066454 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe1e4103340 0x7fe1e419efb0 secure :-1 s=READY pgs=371 cs=0 l=1 rev1=1 crypto rx=0x7fe1dc005e30 tx=0x7fe1dc031ee0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.389 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.387+0000 7fe1eada2640 1 -- 172.21.15.32:0/2413066454 shutdown_connections 2024-04-25T00:32:42.389 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.387+0000 7fe1eada2640 1 --2- 172.21.15.32:0/2413066454 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe1d40a2c20 0x7fe1d40a50e0 unknown :-1 s=CLOSED pgs=234 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.389 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.387+0000 7fe1eada2640 1 --2- 172.21.15.32:0/2413066454 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe1e4103340 0x7fe1e419efb0 unknown :-1 s=CLOSED pgs=371 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.389 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.387+0000 7fe1eada2640 1 --2- 172.21.15.32:0/2413066454 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe1e4102980 0x7fe1e4197f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.389 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.387+0000 7fe1eada2640 1 --2- 172.21.15.32:0/2413066454 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe1e4101790 0x7fe1e4197a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.389 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.387+0000 7fe1eada2640 1 -- 172.21.15.32:0/2413066454 >> 172.21.15.32:0/2413066454 conn(0x7fe1e40fcd20 msgr2=0x7fe1e4105c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:42.389 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.387+0000 7fe1eada2640 1 -- 172.21.15.32:0/2413066454 shutdown_connections 2024-04-25T00:32:42.389 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.387+0000 7fe1eada2640 1 -- 172.21.15.32:0/2413066454 wait complete. 2024-04-25T00:32:42.399 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:32:42.508 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.502+0000 7f0ecf916640 1 -- 172.21.15.32:0/952363112 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0ec8101790 msgr2=0x7f0ec8101bb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:42.508 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.502+0000 7f0ecf916640 1 --2- 172.21.15.32:0/952363112 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0ec8101790 0x7f0ec8101bb0 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7f0ec4009900 tx=0x7f0ec402a410 comp rx=0 tx=0).stop 2024-04-25T00:32:42.508 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.502+0000 7f0ecf916640 1 -- 172.21.15.32:0/952363112 shutdown_connections 2024-04-25T00:32:42.508 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.502+0000 7f0ecf916640 1 --2- 172.21.15.32:0/952363112 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0ec8103340 0x7f0ec8109bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.509 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.502+0000 7f0ecf916640 1 --2- 172.21.15.32:0/952363112 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0ec8102980 0x7f0ec8102e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.509 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.502+0000 7f0ecf916640 1 --2- 172.21.15.32:0/952363112 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0ec8101790 0x7f0ec8101bb0 unknown :-1 s=CLOSED pgs=372 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.509 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.502+0000 7f0ecf916640 1 -- 172.21.15.32:0/952363112 >> 172.21.15.32:0/952363112 conn(0x7f0ec80fcd40 msgr2=0x7f0ec80ff160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:42.509 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.502+0000 7f0ecf916640 1 -- 172.21.15.32:0/952363112 shutdown_connections 2024-04-25T00:32:42.509 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.502+0000 7f0ecf916640 1 -- 172.21.15.32:0/952363112 wait complete. 2024-04-25T00:32:42.510 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.502+0000 7f0ecf916640 1 Processor -- start 2024-04-25T00:32:42.510 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.502+0000 7f0ecf916640 1 -- start start 2024-04-25T00:32:42.510 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.503+0000 7f0ecf916640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0ec8101790 0x7f0ec80754f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.510 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.503+0000 7f0ecf916640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0ec8102980 0x7f0ec8073b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.510 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.503+0000 7f0ecf916640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0ec8103340 0x7f0ec8074070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.510 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.503+0000 7f0ecf916640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f0ec810d400 con 0x7f0ec8103340 2024-04-25T00:32:42.511 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.503+0000 7f0ecf916640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f0ec810d280 con 0x7f0ec8101790 2024-04-25T00:32:42.511 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.503+0000 7f0ecf916640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f0ec810d580 con 0x7f0ec8102980 2024-04-25T00:32:42.511 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.503+0000 7f0ecdeb3640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0ec8103340 0x7f0ec8074070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:42.511 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.503+0000 7f0ecdeb3640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0ec8103340 0x7f0ec8074070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:47782/0 (socket says 172.21.15.32:47782) 2024-04-25T00:32:42.511 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.503+0000 7f0ecdeb3640 1 -- 172.21.15.32:0/1241704843 learned_addr learned my addr 172.21.15.32:0/1241704843 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:42.511 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.503+0000 7f0ecceb1640 1 --2- 172.21.15.32:0/1241704843 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0ec8102980 0x7f0ec8073b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:42.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.503+0000 7f0ecd6b2640 1 --2- 172.21.15.32:0/1241704843 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0ec8101790 0x7f0ec80754f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:42.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.503+0000 7f0ecdeb3640 1 -- 172.21.15.32:0/1241704843 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0ec8102980 msgr2=0x7f0ec8073b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:42.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.503+0000 7f0ecdeb3640 1 --2- 172.21.15.32:0/1241704843 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0ec8102980 0x7f0ec8073b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.503+0000 7f0ecdeb3640 1 -- 172.21.15.32:0/1241704843 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0ec8101790 msgr2=0x7f0ec80754f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:42.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.503+0000 7f0ecdeb3640 1 --2- 172.21.15.32:0/1241704843 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0ec8101790 0x7f0ec80754f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.503+0000 7f0ecdeb3640 1 -- 172.21.15.32:0/1241704843 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0ec8074900 con 0x7f0ec8103340 2024-04-25T00:32:42.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.503+0000 7f0ecd6b2640 1 --2- 172.21.15.32:0/1241704843 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0ec8101790 0x7f0ec80754f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:32:42.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.503+0000 7f0ecdeb3640 1 --2- 172.21.15.32:0/1241704843 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0ec8103340 0x7f0ec8074070 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 crypto rx=0x7f0ec000cad0 tx=0x7f0ec000cf90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:42.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.503+0000 7f0ebe7fc640 1 -- 172.21.15.32:0/1241704843 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0ec000ae20 con 0x7f0ec8103340 2024-04-25T00:32:42.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.503+0000 7f0ecf916640 1 -- 172.21.15.32:0/1241704843 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0ec81a1c50 con 0x7f0ec8103340 2024-04-25T00:32:42.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.503+0000 7f0ebe7fc640 1 -- 172.21.15.32:0/1241704843 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0ec0005020 con 0x7f0ec8103340 2024-04-25T00:32:42.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.503+0000 7f0ebe7fc640 1 -- 172.21.15.32:0/1241704843 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0ec000f6f0 con 0x7f0ec8103340 2024-04-25T00:32:42.514 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.503+0000 7f0ecf916640 1 -- 172.21.15.32:0/1241704843 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0ec81a2150 con 0x7f0ec8103340 2024-04-25T00:32:42.514 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.504+0000 7f0ecf916640 1 -- 172.21.15.32:0/1241704843 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0ec8108b20 con 0x7f0ec8103340 2024-04-25T00:32:42.514 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.505+0000 7f0ebe7fc640 1 -- 172.21.15.32:0/1241704843 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f0ec00051d0 con 0x7f0ec8103340 2024-04-25T00:32:42.514 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.506+0000 7f0ebe7fc640 1 --2- 172.21.15.32:0/1241704843 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f0eb40a2c20 0x7f0eb40a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.514 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.506+0000 7f0ecd6b2640 1 --2- 172.21.15.32:0/1241704843 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f0eb40a2c20 0x7f0eb40a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:42.514 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.506+0000 7f0ebe7fc640 1 -- 172.21.15.32:0/1241704843 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f0ec00cf9e0 con 0x7f0ec8103340 2024-04-25T00:32:42.515 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.507+0000 7f0ecd6b2640 1 --2- 172.21.15.32:0/1241704843 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f0eb40a2c20 0x7f0eb40a50e0 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f0ec4006010 tx=0x7f0ec4032dd0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:42.515 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.510+0000 7f0ebe7fc640 1 -- 172.21.15.32:0/1241704843 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f0ec0014030 con 0x7f0ec8103340 2024-04-25T00:32:42.735 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.735+0000 7f0ecf916640 1 -- 172.21.15.32:0/1241704843 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f0ec802d170 con 0x7f0ec8103340 2024-04-25T00:32:42.737 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.736+0000 7f0ebe7fc640 1 -- 172.21.15.32:0/1241704843 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) ==== 94+0+11876 (secure 0 0 0) 0x7f0ec009c480 con 0x7f0ec8103340 2024-04-25T00:32:42.737 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:32:42.737 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14643,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6839/187492674","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6838","nonce":187492674},{"type":"v1","addr":"172.21.15.32:6839","nonce":187492674}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14655,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6841/1948785545","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6840","nonce":1948785545},{"type":"v1","addr":"172.21.15.32:6841","nonce":1948785545}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-25T00:30:47.866875+0000","modified":"2024-04-25T00:31:14.972514+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24506,"mds_1":24481,"mds_2":24476},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24443":{"gid":24443,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24445":{"gid":24445,"name":"f","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24457":{"gid":24457,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24476":{"gid":24476,"name":"h","rank":2,"incarnation":22,"state":"up:active","state_seq":12,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24481":{"gid":24481,"name":"i","rank":1,"incarnation":20,"state":"up:active","state_seq":11,"addr":"172.21.15.130:6839/4267125809","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":4267125809},{"type":"v1","addr":"172.21.15.130:6839","nonce":4267125809}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24506":{"gid":24506,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.121:6839/258444853","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6838","nonce":258444853},{"type":"v1","addr":"172.21.15.121:6839","nonce":258444853}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24506,"qdb_cluster":[24476,24481,24506]},"id":1}]} 2024-04-25T00:32:42.738 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 24 2024-04-25T00:32:42.738 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.738+0000 7f0ecf916640 1 -- 172.21.15.32:0/1241704843 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f0eb40a2c20 msgr2=0x7f0eb40a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:42.738 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.738+0000 7f0ecf916640 1 --2- 172.21.15.32:0/1241704843 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f0eb40a2c20 0x7f0eb40a50e0 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f0ec4006010 tx=0x7f0ec4032dd0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.739 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.738+0000 7f0ecf916640 1 -- 172.21.15.32:0/1241704843 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0ec8103340 msgr2=0x7f0ec8074070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:42.739 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.738+0000 7f0ecf916640 1 --2- 172.21.15.32:0/1241704843 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0ec8103340 0x7f0ec8074070 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 crypto rx=0x7f0ec000cad0 tx=0x7f0ec000cf90 comp rx=0 tx=0).stop 2024-04-25T00:32:42.739 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.738+0000 7f0ecf916640 1 -- 172.21.15.32:0/1241704843 shutdown_connections 2024-04-25T00:32:42.739 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.738+0000 7f0ecf916640 1 --2- 172.21.15.32:0/1241704843 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f0eb40a2c20 0x7f0eb40a50e0 unknown :-1 s=CLOSED pgs=235 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.739 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.738+0000 7f0ecf916640 1 --2- 172.21.15.32:0/1241704843 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0ec8103340 0x7f0ec8074070 unknown :-1 s=CLOSED pgs=373 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.739 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.738+0000 7f0ecf916640 1 --2- 172.21.15.32:0/1241704843 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0ec8102980 0x7f0ec8073b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.739 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.738+0000 7f0ecf916640 1 --2- 172.21.15.32:0/1241704843 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0ec8101790 0x7f0ec80754f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.739 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.738+0000 7f0ecf916640 1 -- 172.21.15.32:0/1241704843 >> 172.21.15.32:0/1241704843 conn(0x7f0ec80fcd40 msgr2=0x7f0ec8105cf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:42.739 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.738+0000 7f0ecf916640 1 -- 172.21.15.32:0/1241704843 shutdown_connections 2024-04-25T00:32:42.739 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.738+0000 7f0ecf916640 1 -- 172.21.15.32:0/1241704843 wait complete. 2024-04-25T00:32:42.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:42 smithi032 ceph-mon[29357]: pgmap v350: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:32:42.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:42 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3001377517' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:32:42.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:42 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1176865102' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:32:42.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:42 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2413066454' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:32:42.750 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 24, 'flags': 50, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': True, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 32, 'explicitly_allowed_features': 32, 'created': '2024-04-25T00:30:47.866875+0000', 'modified': '2024-04-25T00:31:14.972514+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 300, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}, 'max_mds': 3, 'in': [0, 1, 2], 'up': {'mds_0': 24506, 'mds_1': 24481, 'mds_2': 24476}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_24443': {'gid': 24443, 'name': 'e', 'rank': 2, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.121:6835/271690252', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.121:6834', 'nonce': 271690252}, {'type': 'v1', 'addr': '172.21.15.121:6835', 'nonce': 271690252}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24445': {'gid': 24445, 'name': 'f', 'rank': 1, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.130:6835/2892666533', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.130:6834', 'nonce': 2892666533}, {'type': 'v1', 'addr': '172.21.15.130:6835', 'nonce': 2892666533}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24457': {'gid': 24457, 'name': 'c', 'rank': 0, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.130:6837/1920876316', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.130:6836', 'nonce': 1920876316}, {'type': 'v1', 'addr': '172.21.15.130:6837', 'nonce': 1920876316}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24476': {'gid': 24476, 'name': 'h', 'rank': 2, 'incarnation': 22, 'state': 'up:active', 'state_seq': 12, 'addr': '172.21.15.121:6837/3456022860', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.121:6836', 'nonce': 3456022860}, {'type': 'v1', 'addr': '172.21.15.121:6837', 'nonce': 3456022860}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24481': {'gid': 24481, 'name': 'i', 'rank': 1, 'incarnation': 20, 'state': 'up:active', 'state_seq': 11, 'addr': '172.21.15.130:6839/4267125809', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.130:6838', 'nonce': 4267125809}, {'type': 'v1', 'addr': '172.21.15.130:6839', 'nonce': 4267125809}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24506': {'gid': 24506, 'name': 'b', 'rank': 0, 'incarnation': 14, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.121:6839/258444853', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.121:6838', 'nonce': 258444853}, {'type': 'v1', 'addr': '172.21.15.121:6839', 'nonce': 258444853}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}}, 'data_pools': [3, 4], 'metadata_pool': 2, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 24506, 'qdb_cluster': [24476, 24481, 24506]} 2024-04-25T00:32:42.750 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 3/3 2024-04-25T00:32:42.750 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.h status 2024-04-25T00:32:42.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:42 smithi121 ceph-mon[32231]: pgmap v350: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:32:42.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:42 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3001377517' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:32:42.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:42 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1176865102' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:32:42.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:42 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2413066454' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:32:42.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.861+0000 7ff07baba640 1 -- 172.21.15.32:0/1987899271 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff07410ffc0 msgr2=0x7ff0741123b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:42.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.861+0000 7ff07baba640 1 --2- 172.21.15.32:0/1987899271 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff07410ffc0 0x7ff0741123b0 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7ff06c009ca0 tx=0x7ff06c02a8f0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.867+0000 7ff07baba640 1 -- 172.21.15.32:0/1987899271 shutdown_connections 2024-04-25T00:32:42.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.867+0000 7ff07baba640 1 --2- 172.21.15.32:0/1987899271 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff07410ffc0 0x7ff0741123b0 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.867+0000 7ff07baba640 1 --2- 172.21.15.32:0/1987899271 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff0741090a0 0x7ff07410fa80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.867+0000 7ff07baba640 1 --2- 172.21.15.32:0/1987899271 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff074108740 0x7ff074108b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.867+0000 7ff07baba640 1 -- 172.21.15.32:0/1987899271 >> 172.21.15.32:0/1987899271 conn(0x7ff07406e5d0 msgr2=0x7ff074070a10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:42.872 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.871+0000 7ff07baba640 1 -- 172.21.15.32:0/1987899271 shutdown_connections 2024-04-25T00:32:42.874 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.873+0000 7ff07baba640 1 -- 172.21.15.32:0/1987899271 wait complete. 2024-04-25T00:32:42.874 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.874+0000 7ff07baba640 1 Processor -- start 2024-04-25T00:32:42.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.874+0000 7ff07baba640 1 -- start start 2024-04-25T00:32:42.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.874+0000 7ff07baba640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff074108740 0x7ff07419e150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.874+0000 7ff07baba640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff0741090a0 0x7ff07419e690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.874+0000 7ff07baba640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff07410ffc0 0x7ff0741a56b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.874+0000 7ff07baba640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7ff074117f50 con 0x7ff0741090a0 2024-04-25T00:32:42.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.874+0000 7ff07baba640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7ff074117dd0 con 0x7ff07410ffc0 2024-04-25T00:32:42.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.874+0000 7ff07baba640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7ff0741180d0 con 0x7ff074108740 2024-04-25T00:32:42.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.875+0000 7ff07a057640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff07410ffc0 0x7ff0741a56b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:42.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.875+0000 7ff07a057640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff07410ffc0 0x7ff0741a56b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:39592/0 (socket says 172.21.15.32:39592) 2024-04-25T00:32:42.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.875+0000 7ff07a057640 1 -- 172.21.15.32:0/3386049497 learned_addr learned my addr 172.21.15.32:0/3386049497 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:42.878 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.877+0000 7ff07a057640 1 -- 172.21.15.32:0/3386049497 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff074108740 msgr2=0x7ff07419e150 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-04-25T00:32:42.878 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.877+0000 7ff07a057640 1 --2- 172.21.15.32:0/3386049497 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff074108740 0x7ff07419e150 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.878 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.877+0000 7ff07a057640 1 -- 172.21.15.32:0/3386049497 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff0741090a0 msgr2=0x7ff07419e690 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:32:42.878 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.877+0000 7ff07a057640 1 --2- 172.21.15.32:0/3386049497 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff0741090a0 0x7ff07419e690 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.879 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.877+0000 7ff07a057640 1 -- 172.21.15.32:0/3386049497 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff0741b6e50 con 0x7ff07410ffc0 2024-04-25T00:32:42.880 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.880+0000 7ff07a057640 1 --2- 172.21.15.32:0/3386049497 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff07410ffc0 0x7ff0741a56b0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7ff06c031a80 tx=0x7ff06c003f10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:42.884 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.880+0000 7ff06affd640 1 -- 172.21.15.32:0/3386049497 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff06c0042c0 con 0x7ff07410ffc0 2024-04-25T00:32:42.885 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.880+0000 7ff07baba640 1 -- 172.21.15.32:0/3386049497 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff0741b7020 con 0x7ff07410ffc0 2024-04-25T00:32:42.885 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.880+0000 7ff07baba640 1 -- 172.21.15.32:0/3386049497 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff0741b75e0 con 0x7ff07410ffc0 2024-04-25T00:32:42.885 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.880+0000 7ff06affd640 1 -- 172.21.15.32:0/3386049497 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff06c02d070 con 0x7ff07410ffc0 2024-04-25T00:32:42.885 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.880+0000 7ff06affd640 1 -- 172.21.15.32:0/3386049497 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff06c02fed0 con 0x7ff07410ffc0 2024-04-25T00:32:42.885 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.884+0000 7ff07e7c8740 1 Processor -- start 2024-04-25T00:32:42.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.885+0000 7ff06affd640 1 -- 172.21.15.32:0/3386049497 <== mon.2 v2:172.21.15.121:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7ff06c004460 con 0x7ff07410ffc0 2024-04-25T00:32:42.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.886+0000 7ff06affd640 1 --2- 172.21.15.32:0/3386049497 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff0600a2e60 0x7ff0600a5320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.887+0000 7ff06affd640 1 -- 172.21.15.32:0/3386049497 <== mon.2 v2:172.21.15.121:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7ff06c0305f0 con 0x7ff07410ffc0 2024-04-25T00:32:42.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.887+0000 7ff079055640 1 --2- 172.21.15.32:0/3386049497 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff0600a2e60 0x7ff0600a5320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:42.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.887+0000 7ff07e7c8740 1 -- start start 2024-04-25T00:32:42.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.887+0000 7ff07e7c8740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575c3e48440 0x5575c3e48860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.887+0000 7ff07e7c8740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e48e30 0x5575c3e4d930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.887+0000 7ff07e7c8740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575c3e4dfa0 0x5575c3e50430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.887+0000 7ff07e7c8740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5575c3dabf00 con 0x5575c3e48e30 2024-04-25T00:32:42.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.887+0000 7ff07e7c8740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5575c3d54750 con 0x5575c3e4dfa0 2024-04-25T00:32:42.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.887+0000 7ff07e7c8740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5575c3da7ec0 con 0x5575c3e48440 2024-04-25T00:32:42.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.888+0000 7ff07a057640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575c3e4dfa0 0x5575c3e50430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:42.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.888+0000 7ff07a057640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575c3e4dfa0 0x5575c3e50430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:39608/0 (socket says 172.21.15.32:39608) 2024-04-25T00:32:42.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.888+0000 7ff07a057640 1 -- 172.21.15.32:0/3510899844 learned_addr learned my addr 172.21.15.32:0/3510899844 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:42.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.888+0000 7ff079055640 1 --2- 172.21.15.32:0/3510899844 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e48e30 0x5575c3e4d930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:42.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.888+0000 7ff079055640 1 --2- 172.21.15.32:0/3386049497 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff0600a2e60 0x7ff0600a5320 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7ff07419f610 tx=0x7ff064006bd0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:42.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.888+0000 7ff07a057640 1 -- 172.21.15.32:0/3510899844 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575c3e48440 msgr2=0x5575c3e48860 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-25T00:32:42.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.888+0000 7ff07a057640 1 --2- 172.21.15.32:0/3510899844 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575c3e48440 0x5575c3e48860 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.888+0000 7ff07a057640 1 -- 172.21.15.32:0/3510899844 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e48e30 msgr2=0x5575c3e4d930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:32:42.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.888+0000 7ff07a057640 1 --2- 172.21.15.32:0/3510899844 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e48e30 0x5575c3e4d930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.888+0000 7ff07a057640 1 -- 172.21.15.32:0/3510899844 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5575c3ac02d0 con 0x5575c3e4dfa0 2024-04-25T00:32:42.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.890+0000 7ff079055640 1 --2- 172.21.15.32:0/3510899844 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e48e30 0x5575c3e4d930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:32:42.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:42 smithi130 ceph-mon[32525]: pgmap v350: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:32:42.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:42 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3001377517' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:32:42.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:42 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1176865102' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:32:42.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:42 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2413066454' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:32:42.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.892+0000 7ff07a057640 1 --2- 172.21.15.32:0/3510899844 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575c3e4dfa0 0x5575c3e50430 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7ff06c0bd6d0 tx=0x7ff06c0c2c10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=984a143fedc90241 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:42.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.893+0000 7ff05ffff640 1 -- 172.21.15.32:0/3510899844 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff06c0ba170 con 0x5575c3e4dfa0 2024-04-25T00:32:42.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.893+0000 7ff05ffff640 1 -- 172.21.15.32:0/3510899844 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff06c03be90 con 0x5575c3e4dfa0 2024-04-25T00:32:42.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.893+0000 7ff05ffff640 1 -- 172.21.15.32:0/3510899844 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff06c0ba310 con 0x5575c3e4dfa0 2024-04-25T00:32:42.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.894+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3510899844 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575c3e4dfa0 msgr2=0x5575c3e50430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:42.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.894+0000 7ff07e7c8740 1 --2- 172.21.15.32:0/3510899844 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575c3e4dfa0 0x5575c3e50430 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7ff06c0bd6d0 tx=0x7ff06c0c2c10 comp rx=0 tx=0).stop 2024-04-25T00:32:42.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.894+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3510899844 shutdown_connections 2024-04-25T00:32:42.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.894+0000 7ff07e7c8740 1 --2- 172.21.15.32:0/3510899844 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575c3e4dfa0 0x5575c3e50430 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.894+0000 7ff07e7c8740 1 --2- 172.21.15.32:0/3510899844 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e48e30 0x5575c3e4d930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.894+0000 7ff07e7c8740 1 --2- 172.21.15.32:0/3510899844 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575c3e48440 0x5575c3e48860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.896 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.894+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3510899844 >> 172.21.15.32:0/3510899844 conn(0x5575c3e43ab0 msgr2=0x5575c3e43eb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:42.896 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.895+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3510899844 shutdown_connections 2024-04-25T00:32:42.897 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.897+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3510899844 wait complete. 2024-04-25T00:32:42.899 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.897+0000 7ff07e7c8740 1 -- start start 2024-04-25T00:32:42.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.897+0000 7ff07e7c8740 1 Processor -- start 2024-04-25T00:32:42.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.898+0000 7ff07e7c8740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575c3e48440 0x5575c3e52500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.898+0000 7ff07e7c8740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e48e30 0x5575c3e52a40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.898+0000 7ff07e7c8740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575c3e4dfa0 0x5575c3e59a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.898+0000 7ff07e7c8740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5575c3d54750 con 0x5575c3e48e30 2024-04-25T00:32:42.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.898+0000 7ff07e7c8740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5575c3d67cf0 con 0x5575c3e48440 2024-04-25T00:32:42.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.898+0000 7ff07e7c8740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5575c3e50d10 con 0x5575c3e4dfa0 2024-04-25T00:32:42.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.898+0000 7ff07e7c8740 10 client.0 _ll_register_callbacks cb 0x5575c3d676e0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:32:42.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.898+0000 7ff07a057640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575c3e4dfa0 0x5575c3e59a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:42.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.898+0000 7ff07a057640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575c3e4dfa0 0x5575c3e59a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:44686/0 (socket says 172.21.15.32:44686) 2024-04-25T00:32:42.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.898+0000 7ff07a057640 1 -- 172.21.15.32:0/3709106538 learned_addr learned my addr 172.21.15.32:0/3709106538 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:42.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.898+0000 7ff079856640 1 --2- 172.21.15.32:0/3709106538 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575c3e48440 0x5575c3e52500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:42.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.899+0000 7ff079055640 1 --2- 172.21.15.32:0/3709106538 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e48e30 0x5575c3e52a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:42.904 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.899+0000 7ff079856640 1 -- 172.21.15.32:0/3709106538 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575c3e4dfa0 msgr2=0x5575c3e59a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:32:42.904 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.899+0000 7ff079856640 1 --2- 172.21.15.32:0/3709106538 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575c3e4dfa0 0x5575c3e59a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.899+0000 7ff079856640 1 -- 172.21.15.32:0/3709106538 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e48e30 msgr2=0x5575c3e52a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:32:42.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.899+0000 7ff079856640 1 --2- 172.21.15.32:0/3709106538 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e48e30 0x5575c3e52a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.899+0000 7ff079856640 1 -- 172.21.15.32:0/3709106538 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5575c3b8a6b0 con 0x5575c3e48440 2024-04-25T00:32:42.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.899+0000 7ff079856640 1 --2- 172.21.15.32:0/3709106538 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575c3e48440 0x5575c3e52500 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x5575c3e49690 tx=0x7ff07000b4b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=4b5a29b5eb698480 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:42.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.899+0000 7ff05d7fa640 10 client.0 ms_handle_connect on v2:172.21.15.121:3300/0 2024-04-25T00:32:42.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.900+0000 7ff05d7fa640 1 -- 172.21.15.32:0/3709106538 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff070011420 con 0x5575c3e48440 2024-04-25T00:32:42.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.900+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3709106538 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ff06c03be90 con 0x5575c3e48440 2024-04-25T00:32:42.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.900+0000 7ff05d7fa640 1 -- 172.21.15.32:0/3709106538 <== mon.2 v2:172.21.15.121:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7ff0700115c0 con 0x5575c3e48440 2024-04-25T00:32:42.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.900+0000 7ff05d7fa640 1 -- 172.21.15.32:0/3709106538 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) ==== 24+0+0 (secure 0 0 0) 0x7ff070011800 con 0x5575c3e48440 2024-04-25T00:32:42.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.900+0000 7ff07e7c8740 10 client.24932 fetch_fsmap learned FSMap version 24 2024-04-25T00:32:42.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.900+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3709106538 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) -- 0x5575c3e5cb50 con 0x5575c3e48440 2024-04-25T00:32:42.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.900+0000 7ff05d7fa640 1 -- 172.21.15.32:0/3709106538 <== mon.2 v2:172.21.15.121:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff06c03be90 con 0x5575c3e48440 2024-04-25T00:32:42.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.901+0000 7ff05d7fa640 1 -- 172.21.15.32:0/3709106538 <== mon.2 v2:172.21.15.121:3300/0 5 ==== fsmap(e 24) ==== 8346+0+0 (secure 0 0 0) 0x7ff070010b40 con 0x5575c3e48440 2024-04-25T00:32:42.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.901+0000 7ff07e7c8740 10 client.24932 fetch_fsmap finished waiting for FSMap version 24 2024-04-25T00:32:42.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.901+0000 7ff07e7c8740 10 client.24932 resolve_mds: resolved name 'h' to daemon mds.h 2024-04-25T00:32:42.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.901+0000 7ff07e7c8740 20 client.24932 populate_metadata read hostname 'smithi032' 2024-04-25T00:32:42.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.901+0000 7ff07e7c8740 1 --2- 172.21.15.32:0/3709106538 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x5575c3e5f290 0x5575c3e60700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.901+0000 7ff07e7c8740 4 client.24932 mds_command: new command op to 24476 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:32:42.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.901+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3709106538 --> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5575c3db9440 con 0x5575c3e5f290 2024-04-25T00:32:42.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.901+0000 7ff079055640 1 --2- 172.21.15.32:0/3709106538 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x5575c3e5f290 0x5575c3e60700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:42.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.902+0000 7ff05d7fa640 10 client.24932 ms_handle_connect on v2:172.21.15.121:6836/3456022860 2024-04-25T00:32:42.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.902+0000 7ff079055640 1 --2- 172.21.15.32:0/3709106538 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x5575c3e5f290 0x5575c3e60700 crc :-1 s=READY pgs=16 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=8bd6291902f20735 server_cookie=e8189d522fa22fe9 in_seq=0 out_seq=0 2024-04-25T00:32:42.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.903+0000 7ff05d7fa640 1 -- 172.21.15.32:0/3709106538 <== mds.2 v2:172.21.15.121:6836/3456022860 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x5575c3db9440 con 0x5575c3e5f290 2024-04-25T00:32:42.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.903+0000 7ff05d7fa640 10 client.24932 handle_command_reply: tid=0 2024-04-25T00:32:42.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.903+0000 7ff07e7c8740 1 client.24932 shutdown 2024-04-25T00:32:42.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.903+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3709106538 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575c3e48440 msgr2=0x5575c3e52500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:42.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.903+0000 7ff07e7c8740 1 --2- 172.21.15.32:0/3709106538 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575c3e48440 0x5575c3e52500 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x5575c3e49690 tx=0x7ff07000b4b0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.903+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3709106538 shutdown_connections 2024-04-25T00:32:42.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.903+0000 7ff07e7c8740 1 --2- 172.21.15.32:0/3709106538 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x5575c3e5f290 0x5575c3e60700 crc :-1 s=READY pgs=16 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.903+0000 7ff07e7c8740 1 --2- 172.21.15.32:0/3709106538 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575c3e4dfa0 0x5575c3e59a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.903+0000 7ff07e7c8740 1 --2- 172.21.15.32:0/3709106538 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e48e30 0x5575c3e52a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.903+0000 7ff05d7fa640 0 client.24932 ms_handle_reset on v2:172.21.15.121:6836/3456022860 2024-04-25T00:32:42.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.903+0000 7ff07e7c8740 1 --2- 172.21.15.32:0/3709106538 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575c3e48440 0x5575c3e52500 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.903+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3709106538 >> 172.21.15.32:0/3709106538 conn(0x5575c3e43ab0 msgr2=0x5575c3e42ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:42.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.904+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3709106538 shutdown_connections 2024-04-25T00:32:42.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.904+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3709106538 wait complete. 2024-04-25T00:32:42.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.904+0000 7ff07e7c8740 20 client.24932 trim_cache size 0 max 16384 2024-04-25T00:32:42.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.922+0000 7ff07e7c8740 1 Processor -- start 2024-04-25T00:32:42.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.922+0000 7ff07e7c8740 1 -- start start 2024-04-25T00:32:42.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.922+0000 7ff07e7c8740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575c3e48440 0x5575c3e46790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.922+0000 7ff07e7c8740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e48e30 0x5575c3e599a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.922+0000 7ff07e7c8740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575c3e4dfa0 0x5575c3e46cd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.922+0000 7ff07e7c8740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5575c3e5ea40 con 0x5575c3e48e30 2024-04-25T00:32:42.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.922+0000 7ff07e7c8740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5575c3e5f110 con 0x5575c3e48440 2024-04-25T00:32:42.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.922+0000 7ff07e7c8740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5575c3e5f7e0 con 0x5575c3e4dfa0 2024-04-25T00:32:42.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.922+0000 7ff079055640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e48e30 0x5575c3e599a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:42.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.922+0000 7ff079055640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e48e30 0x5575c3e599a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:47800/0 (socket says 172.21.15.32:47800) 2024-04-25T00:32:42.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.922+0000 7ff079055640 1 -- 172.21.15.32:0/3277417594 learned_addr learned my addr 172.21.15.32:0/3277417594 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:42.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.923+0000 7ff07a057640 1 --2- 172.21.15.32:0/3277417594 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575c3e4dfa0 0x5575c3e46cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:42.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.923+0000 7ff079856640 1 --2- 172.21.15.32:0/3277417594 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575c3e48440 0x5575c3e46790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:42.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.923+0000 7ff079055640 1 -- 172.21.15.32:0/3277417594 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575c3e4dfa0 msgr2=0x5575c3e46cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:32:42.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.923+0000 7ff079055640 1 --2- 172.21.15.32:0/3277417594 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575c3e4dfa0 0x5575c3e46cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.923+0000 7ff079055640 1 -- 172.21.15.32:0/3277417594 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575c3e48440 msgr2=0x5575c3e46790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:32:42.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.923+0000 7ff079055640 1 --2- 172.21.15.32:0/3277417594 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575c3e48440 0x5575c3e46790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.923+0000 7ff079055640 1 -- 172.21.15.32:0/3277417594 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5575c3e60a30 con 0x5575c3e48e30 2024-04-25T00:32:42.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.923+0000 7ff079856640 1 --2- 172.21.15.32:0/3277417594 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575c3e48440 0x5575c3e46790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:32:42.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.923+0000 7ff079055640 1 --2- 172.21.15.32:0/3277417594 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e48e30 0x5575c3e599a0 secure :-1 s=READY pgs=374 cs=0 l=1 rev1=1 crypto rx=0x7ff064006fb0 tx=0x7ff06400e000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=519bb94cc6d75191 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:42.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.923+0000 7ff05f7fe640 1 -- 172.21.15.32:0/3277417594 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff06402a030 con 0x5575c3e48e30 2024-04-25T00:32:42.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.923+0000 7ff05f7fe640 1 -- 172.21.15.32:0/3277417594 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff064014e40 con 0x5575c3e48e30 2024-04-25T00:32:42.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.923+0000 7ff05f7fe640 1 -- 172.21.15.32:0/3277417594 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff064018040 con 0x5575c3e48e30 2024-04-25T00:32:42.929 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.924+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3277417594 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e48e30 msgr2=0x5575c3e599a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:42.929 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.924+0000 7ff07e7c8740 1 --2- 172.21.15.32:0/3277417594 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e48e30 0x5575c3e599a0 secure :-1 s=READY pgs=374 cs=0 l=1 rev1=1 crypto rx=0x7ff064006fb0 tx=0x7ff06400e000 comp rx=0 tx=0).stop 2024-04-25T00:32:42.930 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.924+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3277417594 shutdown_connections 2024-04-25T00:32:42.930 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.924+0000 7ff07e7c8740 1 --2- 172.21.15.32:0/3277417594 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575c3e4dfa0 0x5575c3e46cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.930 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.924+0000 7ff07e7c8740 1 --2- 172.21.15.32:0/3277417594 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e48e30 0x5575c3e599a0 unknown :-1 s=CLOSED pgs=374 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.930 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.924+0000 7ff07e7c8740 1 --2- 172.21.15.32:0/3277417594 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575c3e48440 0x5575c3e46790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.930 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.924+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3277417594 >> 172.21.15.32:0/3277417594 conn(0x5575c3e43ab0 msgr2=0x5575c3e3e690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:42.930 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.924+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3277417594 shutdown_connections 2024-04-25T00:32:42.930 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.924+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3277417594 wait complete. 2024-04-25T00:32:42.930 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.924+0000 7ff07e7c8740 1 -- start start 2024-04-25T00:32:42.930 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.924+0000 7ff07e7c8740 1 Processor -- start 2024-04-25T00:32:42.930 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.924+0000 7ff07e7c8740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575c3e48440 0x5575c3e55200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.930 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.924+0000 7ff07e7c8740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575c3e48e30 0x5575c3e55740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.930 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.924+0000 7ff07e7c8740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e4dfa0 0x5575c3e58760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.930 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.924+0000 7ff07e7c8740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5575c3e60580 con 0x5575c3e4dfa0 2024-04-25T00:32:42.931 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.924+0000 7ff07e7c8740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5575c3e5feb0 con 0x5575c3e48e30 2024-04-25T00:32:42.931 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.924+0000 7ff07e7c8740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5575c3e60c50 con 0x5575c3e48440 2024-04-25T00:32:42.931 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.925+0000 7ff07e7c8740 10 client.0 _ll_register_callbacks cb 0x7ff070009910 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:32:42.931 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.925+0000 7ff07a057640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e4dfa0 0x5575c3e58760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:42.931 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.925+0000 7ff07a057640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e4dfa0 0x5575c3e58760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:47802/0 (socket says 172.21.15.32:47802) 2024-04-25T00:32:42.931 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.925+0000 7ff07a057640 1 -- 172.21.15.32:0/3373331642 learned_addr learned my addr 172.21.15.32:0/3373331642 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:42.931 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.925+0000 7ff079856640 1 --2- 172.21.15.32:0/3373331642 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575c3e48440 0x5575c3e55200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:42.931 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.925+0000 7ff079055640 1 --2- 172.21.15.32:0/3373331642 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575c3e48e30 0x5575c3e55740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:42.931 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.925+0000 7ff07a057640 1 -- 172.21.15.32:0/3373331642 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575c3e48440 msgr2=0x5575c3e55200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:32:42.931 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.925+0000 7ff07a057640 1 --2- 172.21.15.32:0/3373331642 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575c3e48440 0x5575c3e55200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.931 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.925+0000 7ff07a057640 1 -- 172.21.15.32:0/3373331642 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575c3e48e30 msgr2=0x5575c3e55740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:32:42.931 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.925+0000 7ff07a057640 1 --2- 172.21.15.32:0/3373331642 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575c3e48e30 0x5575c3e55740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.931 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.925+0000 7ff07a057640 1 -- 172.21.15.32:0/3373331642 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5575c3e58d50 con 0x5575c3e4dfa0 2024-04-25T00:32:42.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.925+0000 7ff07a057640 1 --2- 172.21.15.32:0/3373331642 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e4dfa0 0x5575c3e58760 secure :-1 s=READY pgs=375 cs=0 l=1 rev1=1 crypto rx=0x7ff06c0096a0 tx=0x7ff06c005d10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c5a14c9faad91fd0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:42.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.925+0000 7ff05d7fa640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:32:42.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.925+0000 7ff05d7fa640 1 -- 172.21.15.32:0/3373331642 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff06c0d5040 con 0x5575c3e4dfa0 2024-04-25T00:32:42.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.925+0000 7ff05d7fa640 1 -- 172.21.15.32:0/3373331642 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7ff06c0b9db0 con 0x5575c3e4dfa0 2024-04-25T00:32:42.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.925+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3373331642 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x5575c3db3790 con 0x5575c3e4dfa0 2024-04-25T00:32:42.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.925+0000 7ff05d7fa640 1 -- 172.21.15.32:0/3373331642 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) ==== 24+0+0 (secure 0 0 0) 0x7ff06c0a5050 con 0x5575c3e4dfa0 2024-04-25T00:32:42.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.925+0000 7ff07e7c8740 10 client.15249 fetch_fsmap learned FSMap version 24 2024-04-25T00:32:42.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.925+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3373331642 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) -- 0x5575c3ec5330 con 0x5575c3e4dfa0 2024-04-25T00:32:42.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.926+0000 7ff05d7fa640 1 -- 172.21.15.32:0/3373331642 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x5575c3db3790 con 0x5575c3e4dfa0 2024-04-25T00:32:42.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.926+0000 7ff05d7fa640 1 -- 172.21.15.32:0/3373331642 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 24) ==== 8346+0+0 (secure 0 0 0) 0x7ff06c0c5070 con 0x5575c3e4dfa0 2024-04-25T00:32:42.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.926+0000 7ff07e7c8740 10 client.15249 fetch_fsmap finished waiting for FSMap version 24 2024-04-25T00:32:42.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.926+0000 7ff07e7c8740 10 client.15249 resolve_mds: resolved name 'h' to daemon mds.h 2024-04-25T00:32:42.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.926+0000 7ff07e7c8740 20 client.15249 populate_metadata read hostname 'smithi032' 2024-04-25T00:32:42.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.926+0000 7ff07e7c8740 1 --2- 172.21.15.32:0/3373331642 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x5575c3ec9d00 0x5575c3ec8a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:42.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.926+0000 7ff07e7c8740 4 client.15249 mds_command: new command op to 24476 tid=0 multi_id=0 [{"prefix": "status"}] 2024-04-25T00:32:42.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.926+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3373331642 --> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] -- command(tid 0: {"prefix": "status"}) -- 0x5575c3d31b80 con 0x5575c3ec9d00 2024-04-25T00:32:42.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.926+0000 7ff079856640 1 --2- 172.21.15.32:0/3373331642 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x5575c3ec9d00 0x5575c3ec8a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:42.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.926+0000 7ff05d7fa640 10 client.15249 ms_handle_connect on v2:172.21.15.121:6836/3456022860 2024-04-25T00:32:42.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.926+0000 7ff079856640 1 --2- 172.21.15.32:0/3373331642 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x5575c3ec9d00 0x5575c3ec8a10 crc :-1 s=READY pgs=17 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=9b13cd2a75b87866 server_cookie=ff802bc2717505f0 in_seq=0 out_seq=0 2024-04-25T00:32:42.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.927+0000 7ff05d7fa640 1 -- 172.21.15.32:0/3373331642 <== mds.2 v2:172.21.15.121:6836/3456022860 1 ==== command_reply(tid 0: 0 ) ==== 8+0+332 (crc 0 0 0) 0x5575c3d31b80 con 0x5575c3ec9d00 2024-04-25T00:32:42.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.927+0000 7ff05d7fa640 10 client.15249 handle_command_reply: tid=0 2024-04-25T00:32:42.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.927+0000 7ff07e7c8740 1 client.15249 shutdown 2024-04-25T00:32:42.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.927+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3373331642 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e4dfa0 msgr2=0x5575c3e58760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:42.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.927+0000 7ff07e7c8740 1 --2- 172.21.15.32:0/3373331642 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e4dfa0 0x5575c3e58760 secure :-1 s=READY pgs=375 cs=0 l=1 rev1=1 crypto rx=0x7ff06c0096a0 tx=0x7ff06c005d10 comp rx=0 tx=0).stop 2024-04-25T00:32:42.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.927+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3373331642 shutdown_connections 2024-04-25T00:32:42.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.927+0000 7ff07e7c8740 1 --2- 172.21.15.32:0/3373331642 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x5575c3ec9d00 0x5575c3ec8a10 crc :-1 s=READY pgs=17 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.927+0000 7ff07e7c8740 1 --2- 172.21.15.32:0/3373331642 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575c3e4dfa0 0x5575c3e58760 unknown :-1 s=CLOSED pgs=375 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.927+0000 7ff07e7c8740 1 --2- 172.21.15.32:0/3373331642 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575c3e48e30 0x5575c3e55740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.927+0000 7ff07e7c8740 1 --2- 172.21.15.32:0/3373331642 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575c3e48440 0x5575c3e55200 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.934 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.927+0000 7ff05d7fa640 0 client.15249 ms_handle_reset on v2:172.21.15.121:6836/3456022860 2024-04-25T00:32:42.934 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.927+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3373331642 >> 172.21.15.32:0/3373331642 conn(0x5575c3e43ab0 msgr2=0x5575c3e42250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:42.934 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.927+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3373331642 shutdown_connections 2024-04-25T00:32:42.934 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.927+0000 7ff07e7c8740 1 -- 172.21.15.32:0/3373331642 wait complete. 2024-04-25T00:32:42.934 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.927+0000 7ff07e7c8740 20 client.15249 trim_cache size 0 max 16384 2024-04-25T00:32:42.934 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:32:42.934 INFO:teuthology.orchestra.run.smithi032.stdout: "cluster_fsid": "cb997156-0299-11ef-bc93-c7b262605968", 2024-04-25T00:32:42.934 INFO:teuthology.orchestra.run.smithi032.stdout: "whoami": 2, 2024-04-25T00:32:42.934 INFO:teuthology.orchestra.run.smithi032.stdout: "id": 24476, 2024-04-25T00:32:42.934 INFO:teuthology.orchestra.run.smithi032.stdout: "want_state": "up:active", 2024-04-25T00:32:42.935 INFO:teuthology.orchestra.run.smithi032.stdout: "state": "up:active", 2024-04-25T00:32:42.935 INFO:teuthology.orchestra.run.smithi032.stdout: "fs_name": "cephfs", 2024-04-25T00:32:42.935 INFO:teuthology.orchestra.run.smithi032.stdout: "rank_uptime": 88.962570088000007, 2024-04-25T00:32:42.935 INFO:teuthology.orchestra.run.smithi032.stdout: "mdsmap_epoch": 24, 2024-04-25T00:32:42.935 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_epoch": 91, 2024-04-25T00:32:42.935 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_epoch_barrier": 89, 2024-04-25T00:32:42.935 INFO:teuthology.orchestra.run.smithi032.stdout: "uptime": 132.31961282200001 2024-04-25T00:32:42.935 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:32:42.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.927+0000 7ff07baba640 1 -- 172.21.15.32:0/3386049497 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff0600a2e60 msgr2=0x7ff0600a5320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:42.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.927+0000 7ff07baba640 1 --2- 172.21.15.32:0/3386049497 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff0600a2e60 0x7ff0600a5320 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7ff07419f610 tx=0x7ff064006bd0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.928+0000 7ff07baba640 1 -- 172.21.15.32:0/3386049497 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff07410ffc0 msgr2=0x7ff0741a56b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:42.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.928+0000 7ff07baba640 1 --2- 172.21.15.32:0/3386049497 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff07410ffc0 0x7ff0741a56b0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7ff06c031a80 tx=0x7ff06c003f10 comp rx=0 tx=0).stop 2024-04-25T00:32:42.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.928+0000 7ff07baba640 1 -- 172.21.15.32:0/3386049497 shutdown_connections 2024-04-25T00:32:42.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.928+0000 7ff07baba640 1 --2- 172.21.15.32:0/3386049497 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7ff0600a2e60 0x7ff0600a5320 unknown :-1 s=CLOSED pgs=236 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.928+0000 7ff07baba640 1 --2- 172.21.15.32:0/3386049497 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff07410ffc0 0x7ff0741a56b0 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.928+0000 7ff07baba640 1 --2- 172.21.15.32:0/3386049497 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff0741090a0 0x7ff07419e690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.928+0000 7ff07baba640 1 --2- 172.21.15.32:0/3386049497 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff074108740 0x7ff07419e150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:42.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.928+0000 7ff07baba640 1 -- 172.21.15.32:0/3386049497 >> 172.21.15.32:0/3386049497 conn(0x7ff07406e5d0 msgr2=0x7ff0740701a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:42.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.928+0000 7ff07baba640 1 -- 172.21.15.32:0/3386049497 shutdown_connections 2024-04-25T00:32:42.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:42.928+0000 7ff07baba640 1 -- 172.21.15.32:0/3386049497 wait complete. 2024-04-25T00:32:42.939 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.i status 2024-04-25T00:32:43.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.041+0000 7f32a4329640 1 -- 172.21.15.32:0/1711565825 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f329c104bd0 msgr2=0x7f329c106fc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:43.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.041+0000 7f32a4329640 1 --2- 172.21.15.32:0/1711565825 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f329c104bd0 0x7f329c106fc0 secure :-1 s=READY pgs=376 cs=0 l=1 rev1=1 crypto rx=0x7f3290009900 tx=0x7f329002a410 comp rx=0 tx=0).stop 2024-04-25T00:32:43.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.041+0000 7f32a4329640 1 -- 172.21.15.32:0/1711565825 shutdown_connections 2024-04-25T00:32:43.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.041+0000 7f32a4329640 1 --2- 172.21.15.32:0/1711565825 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f329c107500 0x7f329c1098f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.041+0000 7f32a4329640 1 --2- 172.21.15.32:0/1711565825 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f329c104bd0 0x7f329c106fc0 unknown :-1 s=CLOSED pgs=376 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.041+0000 7f32a4329640 1 --2- 172.21.15.32:0/1711565825 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f329c1022a0 0x7f329c104690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.041+0000 7f32a4329640 1 -- 172.21.15.32:0/1711565825 >> 172.21.15.32:0/1711565825 conn(0x7f329c0fbbf0 msgr2=0x7f329c0fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:43.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.042+0000 7f32a4329640 1 -- 172.21.15.32:0/1711565825 shutdown_connections 2024-04-25T00:32:43.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.042+0000 7f32a4329640 1 -- 172.21.15.32:0/1711565825 wait complete. 2024-04-25T00:32:43.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.042+0000 7f32a4329640 1 Processor -- start 2024-04-25T00:32:43.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.042+0000 7f32a4329640 1 -- start start 2024-04-25T00:32:43.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.042+0000 7f32a4329640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f329c1022a0 0x7f329c197ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.042+0000 7f32a4329640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f329c104bd0 0x7f329c198010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.042+0000 7f32a4329640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f329c107500 0x7f329c19f080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.042+0000 7f32a4329640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f329c10d400 con 0x7f329c1022a0 2024-04-25T00:32:43.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.042+0000 7f32a4329640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f329c10d280 con 0x7f329c104bd0 2024-04-25T00:32:43.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.042+0000 7f32a4329640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f329c10d580 con 0x7f329c107500 2024-04-25T00:32:43.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.042+0000 7f32a20c5640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f329c1022a0 0x7f329c197ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.042+0000 7f32a20c5640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f329c1022a0 0x7f329c197ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:47824/0 (socket says 172.21.15.32:47824) 2024-04-25T00:32:43.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.042+0000 7f32a20c5640 1 -- 172.21.15.32:0/2623841642 learned_addr learned my addr 172.21.15.32:0/2623841642 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:43.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.043+0000 7f32a28c6640 1 --2- 172.21.15.32:0/2623841642 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f329c107500 0x7f329c19f080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.043+0000 7f32a18c4640 1 --2- 172.21.15.32:0/2623841642 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f329c104bd0 0x7f329c198010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.043+0000 7f32a20c5640 1 -- 172.21.15.32:0/2623841642 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f329c107500 msgr2=0x7f329c19f080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:43.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.043+0000 7f32a20c5640 1 --2- 172.21.15.32:0/2623841642 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f329c107500 0x7f329c19f080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.043+0000 7f32a20c5640 1 -- 172.21.15.32:0/2623841642 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f329c104bd0 msgr2=0x7f329c198010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:43.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.043+0000 7f32a20c5640 1 --2- 172.21.15.32:0/2623841642 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f329c104bd0 0x7f329c198010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.043+0000 7f32a20c5640 1 -- 172.21.15.32:0/2623841642 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f329c19f6f0 con 0x7f329c1022a0 2024-04-25T00:32:43.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.043+0000 7f32a18c4640 1 --2- 172.21.15.32:0/2623841642 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f329c104bd0 0x7f329c198010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:32:43.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.043+0000 7f32a20c5640 1 --2- 172.21.15.32:0/2623841642 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f329c1022a0 0x7f329c197ad0 secure :-1 s=READY pgs=377 cs=0 l=1 rev1=1 crypto rx=0x7f328c00db80 tx=0x7f328c00b630 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:43.047 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.043+0000 7f329b7fe640 1 -- 172.21.15.32:0/2623841642 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f328c017070 con 0x7f329c1022a0 2024-04-25T00:32:43.047 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.043+0000 7f329b7fe640 1 -- 172.21.15.32:0/2623841642 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f328c003b70 con 0x7f329c1022a0 2024-04-25T00:32:43.047 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.043+0000 7f32a4329640 1 -- 172.21.15.32:0/2623841642 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f329c19f9e0 con 0x7f329c1022a0 2024-04-25T00:32:43.047 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.043+0000 7f329b7fe640 1 -- 172.21.15.32:0/2623841642 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f328c0106c0 con 0x7f329c1022a0 2024-04-25T00:32:43.047 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.043+0000 7f32a4329640 1 -- 172.21.15.32:0/2623841642 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f329c19ff00 con 0x7f329c1022a0 2024-04-25T00:32:43.054 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.046+0000 7f32a703c740 1 Processor -- start 2024-04-25T00:32:43.054 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.046+0000 7f329b7fe640 1 -- 172.21.15.32:0/2623841642 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7f328c004020 con 0x7f329c1022a0 2024-04-25T00:32:43.054 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.046+0000 7f32a703c740 1 -- start start 2024-04-25T00:32:43.054 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.047+0000 7f32a703c740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55759ba09440 0x55759ba09860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.054 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.047+0000 7f32a703c740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba09e30 0x55759ba0e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.054 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.047+0000 7f32a703c740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55759ba0efa0 0x55759ba11430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.047+0000 7f32a703c740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55759b96cf00 con 0x55759ba09e30 2024-04-25T00:32:43.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.047+0000 7f32a703c740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55759b915750 con 0x55759ba0efa0 2024-04-25T00:32:43.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.047+0000 7f32a703c740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55759b968ec0 con 0x55759ba09440 2024-04-25T00:32:43.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.047+0000 7f32a20c5640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba09e30 0x55759ba0e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.047+0000 7f32a20c5640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba09e30 0x55759ba0e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:47832/0 (socket says 172.21.15.32:47832) 2024-04-25T00:32:43.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.047+0000 7f32a20c5640 1 -- 172.21.15.32:0/3879263910 learned_addr learned my addr 172.21.15.32:0/3879263910 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:43.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.047+0000 7f32a28c6640 1 --2- 172.21.15.32:0/3879263910 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55759ba09440 0x55759ba09860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.047+0000 7f32a18c4640 1 --2- 172.21.15.32:0/3879263910 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55759ba0efa0 0x55759ba11430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.047+0000 7f329b7fe640 1 --2- 172.21.15.32:0/2623841642 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f32840a2c20 0x7f32840a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.047+0000 7f32a20c5640 1 -- 172.21.15.32:0/3879263910 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55759ba09440 msgr2=0x55759ba09860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:32:43.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.047+0000 7f32a20c5640 1 --2- 172.21.15.32:0/3879263910 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55759ba09440 0x55759ba09860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.047+0000 7f32a20c5640 1 -- 172.21.15.32:0/3879263910 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55759ba0efa0 msgr2=0x55759ba11430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:32:43.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.047+0000 7f32a20c5640 1 --2- 172.21.15.32:0/3879263910 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55759ba0efa0 0x55759ba11430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.047+0000 7f32a20c5640 1 -- 172.21.15.32:0/3879263910 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55759b6812d0 con 0x55759ba09e30 2024-04-25T00:32:43.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.047+0000 7f32a18c4640 1 --2- 172.21.15.32:0/3879263910 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55759ba0efa0 0x55759ba11430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:32:43.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.047+0000 7f32a28c6640 1 --2- 172.21.15.32:0/2623841642 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f32840a2c20 0x7f32840a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.047+0000 7f329b7fe640 1 -- 172.21.15.32:0/2623841642 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f328c0cf380 con 0x7f329c1022a0 2024-04-25T00:32:43.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.047+0000 7f32a20c5640 1 --2- 172.21.15.32:0/3879263910 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba09e30 0x55759ba0e930 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7f328c0cc910 tx=0x7f328c090c00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1bc2bfeb2e815ab4 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:43.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.047+0000 7f3298bfe640 1 -- 172.21.15.32:0/3879263910 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f328c09b260 con 0x55759ba09e30 2024-04-25T00:32:43.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.048+0000 7f3298bfe640 1 -- 172.21.15.32:0/3879263910 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f328c066800 con 0x55759ba09e30 2024-04-25T00:32:43.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.048+0000 7f32a28c6640 1 --2- 172.21.15.32:0/2623841642 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f32840a2c20 0x7f32840a50e0 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x55759ba0a690 tx=0x7f32940046c0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:43.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.048+0000 7f3298bfe640 1 -- 172.21.15.32:0/3879263910 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f328c09b080 con 0x55759ba09e30 2024-04-25T00:32:43.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.048+0000 7f32a703c740 1 -- 172.21.15.32:0/3879263910 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba09e30 msgr2=0x55759ba0e930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:43.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.048+0000 7f32a703c740 1 --2- 172.21.15.32:0/3879263910 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba09e30 0x55759ba0e930 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7f328c0cc910 tx=0x7f328c090c00 comp rx=0 tx=0).stop 2024-04-25T00:32:43.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.048+0000 7f32a703c740 1 -- 172.21.15.32:0/3879263910 shutdown_connections 2024-04-25T00:32:43.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.048+0000 7f32a703c740 1 --2- 172.21.15.32:0/3879263910 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55759ba0efa0 0x55759ba11430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.048+0000 7f32a703c740 1 --2- 172.21.15.32:0/3879263910 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba09e30 0x55759ba0e930 unknown :-1 s=CLOSED pgs=378 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.048+0000 7f32a703c740 1 --2- 172.21.15.32:0/3879263910 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55759ba09440 0x55759ba09860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.048+0000 7f32a703c740 1 -- 172.21.15.32:0/3879263910 >> 172.21.15.32:0/3879263910 conn(0x55759ba04ab0 msgr2=0x55759ba04eb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:43.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.048+0000 7f32a703c740 1 -- 172.21.15.32:0/3879263910 shutdown_connections 2024-04-25T00:32:43.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.048+0000 7f32a703c740 1 -- 172.21.15.32:0/3879263910 wait complete. 2024-04-25T00:32:43.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.049+0000 7f32a703c740 1 -- start start 2024-04-25T00:32:43.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.049+0000 7f32a703c740 1 Processor -- start 2024-04-25T00:32:43.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.049+0000 7f32a703c740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55759ba09440 0x55759ba13810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.049+0000 7f32a703c740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55759ba09e30 0x55759ba13d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.049+0000 7f32a703c740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba0efa0 0x55759ba1ad90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.049+0000 7f32a703c740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55759ba11ef0 con 0x55759ba0efa0 2024-04-25T00:32:43.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.049+0000 7f32a703c740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55759b928cf0 con 0x55759ba09440 2024-04-25T00:32:43.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.049+0000 7f32a703c740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55759ba12070 con 0x55759ba09e30 2024-04-25T00:32:43.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.049+0000 7f32a703c740 10 client.0 _ll_register_callbacks cb 0x55759b9286e0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:32:43.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.049+0000 7f32a28c6640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba0efa0 0x55759ba1ad90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.049+0000 7f32a28c6640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba0efa0 0x55759ba1ad90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:47836/0 (socket says 172.21.15.32:47836) 2024-04-25T00:32:43.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.049+0000 7f32a28c6640 1 -- 172.21.15.32:0/279651775 learned_addr learned my addr 172.21.15.32:0/279651775 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:43.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.049+0000 7f32a20c5640 1 --2- 172.21.15.32:0/279651775 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55759ba09440 0x55759ba13810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.049+0000 7f32a18c4640 1 --2- 172.21.15.32:0/279651775 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55759ba09e30 0x55759ba13d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.049+0000 7f32a28c6640 1 -- 172.21.15.32:0/279651775 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55759ba09e30 msgr2=0x55759ba13d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:32:43.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.049+0000 7f32a28c6640 1 --2- 172.21.15.32:0/279651775 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55759ba09e30 0x55759ba13d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.049+0000 7f32a28c6640 1 -- 172.21.15.32:0/279651775 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55759ba09440 msgr2=0x55759ba13810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:32:43.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.049+0000 7f32a28c6640 1 --2- 172.21.15.32:0/279651775 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55759ba09440 0x55759ba13810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.049+0000 7f32a28c6640 1 -- 172.21.15.32:0/279651775 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55759b74b6b0 con 0x55759ba0efa0 2024-04-25T00:32:43.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.049+0000 7f32a18c4640 1 --2- 172.21.15.32:0/279651775 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55759ba09e30 0x55759ba13d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:32:43.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.049+0000 7f32a20c5640 1 --2- 172.21.15.32:0/279651775 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55759ba09440 0x55759ba13810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:32:43.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.050+0000 7f32a28c6640 1 --2- 172.21.15.32:0/279651775 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba0efa0 0x55759ba1ad90 secure :-1 s=READY pgs=379 cs=0 l=1 rev1=1 crypto rx=0x7f3294003d00 tx=0x7f3294004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=451bd87439d13b04 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:43.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.050+0000 7f3281ffb640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:32:43.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.050+0000 7f3281ffb640 1 -- 172.21.15.32:0/279651775 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f329401c620 con 0x55759ba0efa0 2024-04-25T00:32:43.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.050+0000 7f3281ffb640 1 -- 172.21.15.32:0/279651775 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f329401c7c0 con 0x55759ba0efa0 2024-04-25T00:32:43.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.050+0000 7f32a703c740 1 -- 172.21.15.32:0/279651775 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f328c066800 con 0x55759ba0efa0 2024-04-25T00:32:43.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.050+0000 7f3281ffb640 1 -- 172.21.15.32:0/279651775 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) ==== 24+0+0 (secure 0 0 0) 0x7f329401c9c0 con 0x55759ba0efa0 2024-04-25T00:32:43.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.050+0000 7f32a703c740 10 client.15261 fetch_fsmap learned FSMap version 24 2024-04-25T00:32:43.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.050+0000 7f32a703c740 1 -- 172.21.15.32:0/279651775 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) -- 0x55759ba1ddb0 con 0x55759ba0efa0 2024-04-25T00:32:43.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.050+0000 7f3281ffb640 1 -- 172.21.15.32:0/279651775 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f328c066800 con 0x55759ba0efa0 2024-04-25T00:32:43.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.050+0000 7f3281ffb640 1 -- 172.21.15.32:0/279651775 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 24) ==== 8346+0+0 (secure 0 0 0) 0x7f329401d420 con 0x55759ba0efa0 2024-04-25T00:32:43.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.050+0000 7f32a703c740 10 client.15261 fetch_fsmap finished waiting for FSMap version 24 2024-04-25T00:32:43.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.050+0000 7f32a703c740 10 client.15261 resolve_mds: resolved name 'i' to daemon mds.i 2024-04-25T00:32:43.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.050+0000 7f32a703c740 20 client.15261 populate_metadata read hostname 'smithi032' 2024-04-25T00:32:43.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.050+0000 7f32a703c740 1 --2- 172.21.15.32:0/279651775 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55759ba204d0 0x55759ba21940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.050+0000 7f32a703c740 4 client.15261 mds_command: new command op to 24481 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:32:43.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.050+0000 7f32a703c740 1 -- 172.21.15.32:0/279651775 --> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55759b97a440 con 0x55759ba204d0 2024-04-25T00:32:43.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.051+0000 7f32a20c5640 1 --2- 172.21.15.32:0/279651775 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55759ba204d0 0x55759ba21940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.051+0000 7f32a20c5640 1 --2- 172.21.15.32:0/279651775 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55759ba204d0 0x55759ba21940 crc :-1 s=READY pgs=15 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=8b0d68230e2c5d4b server_cookie=a3a652ed6a8da7 in_seq=0 out_seq=0 2024-04-25T00:32:43.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.051+0000 7f3281ffb640 10 client.15261 ms_handle_connect on v2:172.21.15.130:6838/4267125809 2024-04-25T00:32:43.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.052+0000 7f3281ffb640 1 -- 172.21.15.32:0/279651775 <== mds.1 v2:172.21.15.130:6838/4267125809 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x55759b97a440 con 0x55759ba204d0 2024-04-25T00:32:43.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.052+0000 7f3281ffb640 10 client.15261 handle_command_reply: tid=0 2024-04-25T00:32:43.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.052+0000 7f32a703c740 1 client.15261 shutdown 2024-04-25T00:32:43.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.052+0000 7f32a703c740 1 -- 172.21.15.32:0/279651775 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba0efa0 msgr2=0x55759ba1ad90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:43.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.052+0000 7f32a703c740 1 --2- 172.21.15.32:0/279651775 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba0efa0 0x55759ba1ad90 secure :-1 s=READY pgs=379 cs=0 l=1 rev1=1 crypto rx=0x7f3294003d00 tx=0x7f3294004730 comp rx=0 tx=0).stop 2024-04-25T00:32:43.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.052+0000 7f32a703c740 1 -- 172.21.15.32:0/279651775 shutdown_connections 2024-04-25T00:32:43.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.052+0000 7f32a703c740 1 --2- 172.21.15.32:0/279651775 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55759ba204d0 0x55759ba21940 crc :-1 s=READY pgs=15 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.052+0000 7f32a703c740 1 --2- 172.21.15.32:0/279651775 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba0efa0 0x55759ba1ad90 unknown :-1 s=CLOSED pgs=379 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.052+0000 7f32a703c740 1 --2- 172.21.15.32:0/279651775 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55759ba09e30 0x55759ba13d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.052+0000 7f3281ffb640 0 client.15261 ms_handle_reset on v2:172.21.15.130:6838/4267125809 2024-04-25T00:32:43.062 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.052+0000 7f32a703c740 1 --2- 172.21.15.32:0/279651775 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55759ba09440 0x55759ba13810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.062 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.052+0000 7f32a703c740 1 -- 172.21.15.32:0/279651775 >> 172.21.15.32:0/279651775 conn(0x55759ba04ab0 msgr2=0x55759ba03ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:43.062 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.052+0000 7f32a703c740 1 -- 172.21.15.32:0/279651775 shutdown_connections 2024-04-25T00:32:43.062 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.052+0000 7f32a703c740 1 -- 172.21.15.32:0/279651775 wait complete. 2024-04-25T00:32:43.062 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.053+0000 7f32a703c740 20 client.15261 trim_cache size 0 max 16384 2024-04-25T00:32:43.077 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.071+0000 7f32a703c740 1 Processor -- start 2024-04-25T00:32:43.077 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.071+0000 7f32a703c740 1 -- start start 2024-04-25T00:32:43.077 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.071+0000 7f32a703c740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba09440 0x55759b9ff690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.077 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.071+0000 7f32a703c740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55759ba09e30 0x55759ba0a6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.077 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.071+0000 7f32a703c740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55759ba0efa0 0x55759ba0b1f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.077 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.071+0000 7f32a703c740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55759ba1fc80 con 0x55759ba09440 2024-04-25T00:32:43.077 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.071+0000 7f32a703c740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55759ba20350 con 0x55759ba09e30 2024-04-25T00:32:43.077 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.071+0000 7f32a703c740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55759ba20a20 con 0x55759ba0efa0 2024-04-25T00:32:43.077 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.071+0000 7f32a20c5640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba09440 0x55759b9ff690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.077 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.071+0000 7f32a20c5640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba09440 0x55759b9ff690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:47848/0 (socket says 172.21.15.32:47848) 2024-04-25T00:32:43.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.071+0000 7f32a20c5640 1 -- 172.21.15.32:0/1660566069 learned_addr learned my addr 172.21.15.32:0/1660566069 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:43.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.071+0000 7f32a18c4640 1 --2- 172.21.15.32:0/1660566069 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55759ba09e30 0x55759ba0a6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.071+0000 7f32a20c5640 1 -- 172.21.15.32:0/1660566069 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55759ba0efa0 msgr2=0x55759ba0b1f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:32:43.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.071+0000 7f32a28c6640 1 --2- 172.21.15.32:0/1660566069 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55759ba0efa0 0x55759ba0b1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.071+0000 7f32a20c5640 1 --2- 172.21.15.32:0/1660566069 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55759ba0efa0 0x55759ba0b1f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.071+0000 7f32a20c5640 1 -- 172.21.15.32:0/1660566069 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55759ba09e30 msgr2=0x55759ba0a6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:32:43.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.071+0000 7f32a20c5640 1 --2- 172.21.15.32:0/1660566069 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55759ba09e30 0x55759ba0a6c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.071+0000 7f32a20c5640 1 -- 172.21.15.32:0/1660566069 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55759b9fec10 con 0x55759ba09440 2024-04-25T00:32:43.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.071+0000 7f32a18c4640 1 --2- 172.21.15.32:0/1660566069 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55759ba09e30 0x55759ba0a6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:32:43.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.071+0000 7f32a20c5640 1 --2- 172.21.15.32:0/1660566069 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba09440 0x55759b9ff690 secure :-1 s=READY pgs=380 cs=0 l=1 rev1=1 crypto rx=0x7f328c097b80 tx=0x7f328c000f40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=28cb924176b07894 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:43.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.071+0000 7f3283fff640 1 -- 172.21.15.32:0/1660566069 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f328c09be20 con 0x55759ba09440 2024-04-25T00:32:43.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.071+0000 7f3283fff640 1 -- 172.21.15.32:0/1660566069 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f328c09b720 con 0x55759ba09440 2024-04-25T00:32:43.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.071+0000 7f3283fff640 1 -- 172.21.15.32:0/1660566069 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f328c0860b0 con 0x55759ba09440 2024-04-25T00:32:43.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.072+0000 7f32a703c740 1 -- 172.21.15.32:0/1660566069 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba09440 msgr2=0x55759b9ff690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:43.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.072+0000 7f32a703c740 1 --2- 172.21.15.32:0/1660566069 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba09440 0x55759b9ff690 secure :-1 s=READY pgs=380 cs=0 l=1 rev1=1 crypto rx=0x7f328c097b80 tx=0x7f328c000f40 comp rx=0 tx=0).stop 2024-04-25T00:32:43.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.072+0000 7f32a703c740 1 -- 172.21.15.32:0/1660566069 shutdown_connections 2024-04-25T00:32:43.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.072+0000 7f32a703c740 1 --2- 172.21.15.32:0/1660566069 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55759ba0efa0 0x55759ba0b1f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.072+0000 7f32a703c740 1 --2- 172.21.15.32:0/1660566069 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55759ba09e30 0x55759ba0a6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.072+0000 7f32a703c740 1 --2- 172.21.15.32:0/1660566069 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba09440 0x55759b9ff690 unknown :-1 s=CLOSED pgs=380 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.072+0000 7f32a703c740 1 -- 172.21.15.32:0/1660566069 >> 172.21.15.32:0/1660566069 conn(0x55759ba04ab0 msgr2=0x55759ba04e90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:43.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.072+0000 7f32a703c740 1 -- 172.21.15.32:0/1660566069 shutdown_connections 2024-04-25T00:32:43.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.072+0000 7f32a703c740 1 -- 172.21.15.32:0/1660566069 wait complete. 2024-04-25T00:32:43.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.072+0000 7f32a703c740 1 -- start start 2024-04-25T00:32:43.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.072+0000 7f32a703c740 1 Processor -- start 2024-04-25T00:32:43.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f32a703c740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55759ba09440 0x55759ba1a4d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f32a703c740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba09e30 0x55759ba1aa10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f32a703c740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55759ba0efa0 0x55759ba88ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f32a703c740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55759ba217c0 con 0x55759ba09e30 2024-04-25T00:32:43.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f32a703c740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55759ba210f0 con 0x55759ba09440 2024-04-25T00:32:43.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f32a703c740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55759ba21e90 con 0x55759ba0efa0 2024-04-25T00:32:43.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f32a703c740 10 client.0 _ll_register_callbacks cb 0x7f32940039b0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:32:43.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f32a18c4640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba09e30 0x55759ba1aa10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f32a18c4640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba09e30 0x55759ba1aa10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:47862/0 (socket says 172.21.15.32:47862) 2024-04-25T00:32:43.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f32a18c4640 1 -- 172.21.15.32:0/2178429473 learned_addr learned my addr 172.21.15.32:0/2178429473 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:43.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f32a28c6640 1 --2- 172.21.15.32:0/2178429473 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55759ba0efa0 0x55759ba88ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f32a20c5640 1 --2- 172.21.15.32:0/2178429473 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55759ba09440 0x55759ba1a4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f32a18c4640 1 -- 172.21.15.32:0/2178429473 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55759ba0efa0 msgr2=0x55759ba88ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:32:43.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f32a18c4640 1 --2- 172.21.15.32:0/2178429473 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55759ba0efa0 0x55759ba88ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f32a18c4640 1 -- 172.21.15.32:0/2178429473 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55759ba09440 msgr2=0x55759ba1a4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:32:43.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f32a18c4640 1 --2- 172.21.15.32:0/2178429473 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55759ba09440 0x55759ba1a4d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f32a18c4640 1 -- 172.21.15.32:0/2178429473 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55759ba890d0 con 0x55759ba09e30 2024-04-25T00:32:43.081 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f32a20c5640 1 --2- 172.21.15.32:0/2178429473 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55759ba09440 0x55759ba1a4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:32:43.081 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f32a18c4640 1 --2- 172.21.15.32:0/2178429473 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba09e30 0x55759ba1aa10 secure :-1 s=READY pgs=381 cs=0 l=1 rev1=1 crypto rx=0x7f3290031440 tx=0x7f3290031a20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d5b472c49ce725ff server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:43.081 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f3281ffb640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:32:43.081 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f3281ffb640 1 -- 172.21.15.32:0/2178429473 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f329003f070 con 0x55759ba09e30 2024-04-25T00:32:43.081 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f3281ffb640 1 -- 172.21.15.32:0/2178429473 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7f3290004450 con 0x55759ba09e30 2024-04-25T00:32:43.081 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f32a703c740 1 -- 172.21.15.32:0/2178429473 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f328c09b720 con 0x55759ba09e30 2024-04-25T00:32:43.081 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f3281ffb640 1 -- 172.21.15.32:0/2178429473 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) ==== 24+0+0 (secure 0 0 0) 0x7f329004c050 con 0x55759ba09e30 2024-04-25T00:32:43.081 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f32a703c740 10 client.15267 fetch_fsmap learned FSMap version 24 2024-04-25T00:32:43.081 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.073+0000 7f32a703c740 1 -- 172.21.15.32:0/2178429473 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) -- 0x55759ba8a440 con 0x55759ba09e30 2024-04-25T00:32:43.081 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.074+0000 7f3281ffb640 1 -- 172.21.15.32:0/2178429473 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f328c09b720 con 0x55759ba09e30 2024-04-25T00:32:43.081 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.074+0000 7f3281ffb640 1 -- 172.21.15.32:0/2178429473 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 24) ==== 8346+0+0 (secure 0 0 0) 0x7f3290004cd0 con 0x55759ba09e30 2024-04-25T00:32:43.081 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.074+0000 7f32a703c740 10 client.15267 fetch_fsmap finished waiting for FSMap version 24 2024-04-25T00:32:43.081 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.074+0000 7f32a703c740 10 client.15267 resolve_mds: resolved name 'i' to daemon mds.i 2024-04-25T00:32:43.081 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.074+0000 7f32a703c740 20 client.15267 populate_metadata read hostname 'smithi032' 2024-04-25T00:32:43.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.074+0000 7f32a703c740 1 --2- 172.21.15.32:0/2178429473 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55759ba8f090 0x55759ba8dda0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.074+0000 7f32a703c740 4 client.15267 mds_command: new command op to 24481 tid=0 multi_id=0 [{"prefix": "status"}] 2024-04-25T00:32:43.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.074+0000 7f32a703c740 1 -- 172.21.15.32:0/2178429473 --> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] -- command(tid 0: {"prefix": "status"}) -- 0x55759b974790 con 0x55759ba8f090 2024-04-25T00:32:43.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.074+0000 7f32a20c5640 1 --2- 172.21.15.32:0/2178429473 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55759ba8f090 0x55759ba8dda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.074+0000 7f32a20c5640 1 --2- 172.21.15.32:0/2178429473 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55759ba8f090 0x55759ba8dda0 crc :-1 s=READY pgs=16 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=3b644d617e9802e7 server_cookie=c44867e7ca1cd93e in_seq=0 out_seq=0 2024-04-25T00:32:43.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.074+0000 7f3281ffb640 10 client.15267 ms_handle_connect on v2:172.21.15.130:6838/4267125809 2024-04-25T00:32:43.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.075+0000 7f3281ffb640 1 -- 172.21.15.32:0/2178429473 <== mds.1 v2:172.21.15.130:6838/4267125809 1 ==== command_reply(tid 0: 0 ) ==== 8+0+332 (crc 0 0 0) 0x55759b974790 con 0x55759ba8f090 2024-04-25T00:32:43.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.075+0000 7f3281ffb640 10 client.15267 handle_command_reply: tid=0 2024-04-25T00:32:43.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.075+0000 7f32a703c740 1 client.15267 shutdown 2024-04-25T00:32:43.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.075+0000 7f32a703c740 1 -- 172.21.15.32:0/2178429473 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba09e30 msgr2=0x55759ba1aa10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:43.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.075+0000 7f32a703c740 1 --2- 172.21.15.32:0/2178429473 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba09e30 0x55759ba1aa10 secure :-1 s=READY pgs=381 cs=0 l=1 rev1=1 crypto rx=0x7f3290031440 tx=0x7f3290031a20 comp rx=0 tx=0).stop 2024-04-25T00:32:43.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.075+0000 7f32a703c740 1 -- 172.21.15.32:0/2178429473 shutdown_connections 2024-04-25T00:32:43.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.075+0000 7f32a703c740 1 --2- 172.21.15.32:0/2178429473 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55759ba0efa0 0x55759ba88ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.075+0000 7f32a703c740 1 --2- 172.21.15.32:0/2178429473 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55759ba8f090 0x55759ba8dda0 crc :-1 s=READY pgs=16 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.075+0000 7f32a703c740 1 --2- 172.21.15.32:0/2178429473 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55759ba09e30 0x55759ba1aa10 unknown :-1 s=CLOSED pgs=381 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.075+0000 7f32a703c740 1 --2- 172.21.15.32:0/2178429473 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55759ba09440 0x55759ba1a4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.075+0000 7f3281ffb640 0 client.15267 ms_handle_reset on v2:172.21.15.130:6838/4267125809 2024-04-25T00:32:43.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.075+0000 7f32a703c740 1 -- 172.21.15.32:0/2178429473 >> 172.21.15.32:0/2178429473 conn(0x55759ba04ab0 msgr2=0x55759b9ff690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:43.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.075+0000 7f32a703c740 1 -- 172.21.15.32:0/2178429473 shutdown_connections 2024-04-25T00:32:43.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.075+0000 7f32a703c740 1 -- 172.21.15.32:0/2178429473 wait complete. 2024-04-25T00:32:43.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.075+0000 7f32a703c740 20 client.15267 trim_cache size 0 max 16384 2024-04-25T00:32:43.084 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:32:43.084 INFO:teuthology.orchestra.run.smithi032.stdout: "cluster_fsid": "cb997156-0299-11ef-bc93-c7b262605968", 2024-04-25T00:32:43.084 INFO:teuthology.orchestra.run.smithi032.stdout: "whoami": 1, 2024-04-25T00:32:43.084 INFO:teuthology.orchestra.run.smithi032.stdout: "id": 24481, 2024-04-25T00:32:43.084 INFO:teuthology.orchestra.run.smithi032.stdout: "want_state": "up:active", 2024-04-25T00:32:43.084 INFO:teuthology.orchestra.run.smithi032.stdout: "state": "up:active", 2024-04-25T00:32:43.085 INFO:teuthology.orchestra.run.smithi032.stdout: "fs_name": "cephfs", 2024-04-25T00:32:43.085 INFO:teuthology.orchestra.run.smithi032.stdout: "rank_uptime": 90.122969928000003, 2024-04-25T00:32:43.085 INFO:teuthology.orchestra.run.smithi032.stdout: "mdsmap_epoch": 24, 2024-04-25T00:32:43.085 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_epoch": 91, 2024-04-25T00:32:43.085 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_epoch_barrier": 89, 2024-04-25T00:32:43.085 INFO:teuthology.orchestra.run.smithi032.stdout: "uptime": 129.73336154699999 2024-04-25T00:32:43.085 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:32:43.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.075+0000 7f32a4329640 1 -- 172.21.15.32:0/2623841642 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f32840a2c20 msgr2=0x7f32840a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:43.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.075+0000 7f32a4329640 1 --2- 172.21.15.32:0/2623841642 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f32840a2c20 0x7f32840a50e0 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x55759ba0a690 tx=0x7f32940046c0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.075+0000 7f32a4329640 1 -- 172.21.15.32:0/2623841642 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f329c1022a0 msgr2=0x7f329c197ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:43.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.075+0000 7f32a4329640 1 --2- 172.21.15.32:0/2623841642 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f329c1022a0 0x7f329c197ad0 secure :-1 s=READY pgs=377 cs=0 l=1 rev1=1 crypto rx=0x7f328c00db80 tx=0x7f328c00b630 comp rx=0 tx=0).stop 2024-04-25T00:32:43.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.075+0000 7f32a4329640 1 -- 172.21.15.32:0/2623841642 shutdown_connections 2024-04-25T00:32:43.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.075+0000 7f32a4329640 1 --2- 172.21.15.32:0/2623841642 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f32840a2c20 0x7f32840a50e0 unknown :-1 s=CLOSED pgs=237 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.076+0000 7f32a4329640 1 --2- 172.21.15.32:0/2623841642 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f329c107500 0x7f329c19f080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.076+0000 7f32a4329640 1 --2- 172.21.15.32:0/2623841642 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f329c104bd0 0x7f329c198010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.076+0000 7f32a4329640 1 --2- 172.21.15.32:0/2623841642 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f329c1022a0 0x7f329c197ad0 unknown :-1 s=CLOSED pgs=377 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.076+0000 7f32a4329640 1 -- 172.21.15.32:0/2623841642 >> 172.21.15.32:0/2623841642 conn(0x7f329c0fbbf0 msgr2=0x7f329c102d00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:43.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.076+0000 7f32a4329640 1 -- 172.21.15.32:0/2623841642 shutdown_connections 2024-04-25T00:32:43.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.076+0000 7f32a4329640 1 -- 172.21.15.32:0/2623841642 wait complete. 2024-04-25T00:32:43.088 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.b status 2024-04-25T00:32:43.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.190+0000 7fe1a31ed640 1 -- 172.21.15.32:0/1957654527 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe19c102230 msgr2=0x7fe19c108ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:43.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.190+0000 7fe1a31ed640 1 --2- 172.21.15.32:0/1957654527 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe19c102230 0x7fe19c108ac0 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 crypto rx=0x7fe194009ca0 tx=0x7fe19402a8c0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.190+0000 7fe1a31ed640 1 -- 172.21.15.32:0/1957654527 shutdown_connections 2024-04-25T00:32:43.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.190+0000 7fe1a31ed640 1 --2- 172.21.15.32:0/1957654527 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe19c102230 0x7fe19c108ac0 unknown :-1 s=CLOSED pgs=382 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.190+0000 7fe1a31ed640 1 --2- 172.21.15.32:0/1957654527 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe19c101870 0x7fe19c101cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.190+0000 7fe1a31ed640 1 --2- 172.21.15.32:0/1957654527 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe19c100680 0x7fe19c100aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.190+0000 7fe1a31ed640 1 -- 172.21.15.32:0/1957654527 >> 172.21.15.32:0/1957654527 conn(0x7fe19c0fbbf0 msgr2=0x7fe19c0fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:43.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.190+0000 7fe1a31ed640 1 -- 172.21.15.32:0/1957654527 shutdown_connections 2024-04-25T00:32:43.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.190+0000 7fe1a31ed640 1 -- 172.21.15.32:0/1957654527 wait complete. 2024-04-25T00:32:43.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.191+0000 7fe1a31ed640 1 Processor -- start 2024-04-25T00:32:43.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.191+0000 7fe1a31ed640 1 -- start start 2024-04-25T00:32:43.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.191+0000 7fe1a31ed640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe19c100680 0x7fe19c197a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.191+0000 7fe1a31ed640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe19c101870 0x7fe19c197fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.191+0000 7fe1a31ed640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe19c102230 0x7fe19c19f010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.191+0000 7fe1a31ed640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fe19c10d3e0 con 0x7fe19c102230 2024-04-25T00:32:43.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.191+0000 7fe1a31ed640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fe19c10d260 con 0x7fe19c101870 2024-04-25T00:32:43.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.191+0000 7fe1a31ed640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fe19c10d560 con 0x7fe19c100680 2024-04-25T00:32:43.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.191+0000 7fe1a178a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe19c102230 0x7fe19c19f010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.191+0000 7fe1a178a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe19c102230 0x7fe19c19f010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:47884/0 (socket says 172.21.15.32:47884) 2024-04-25T00:32:43.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.191+0000 7fe1a178a640 1 -- 172.21.15.32:0/663018042 learned_addr learned my addr 172.21.15.32:0/663018042 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:43.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.191+0000 7fe1a0f89640 1 --2- 172.21.15.32:0/663018042 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe19c100680 0x7fe19c197a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.191+0000 7fe19bfff640 1 --2- 172.21.15.32:0/663018042 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe19c101870 0x7fe19c197fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.191+0000 7fe1a178a640 1 -- 172.21.15.32:0/663018042 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe19c100680 msgr2=0x7fe19c197a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:43.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.191+0000 7fe1a178a640 1 --2- 172.21.15.32:0/663018042 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe19c100680 0x7fe19c197a60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.191+0000 7fe1a178a640 1 -- 172.21.15.32:0/663018042 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe19c101870 msgr2=0x7fe19c197fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:43.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.191+0000 7fe1a178a640 1 --2- 172.21.15.32:0/663018042 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe19c101870 0x7fe19c197fa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.191+0000 7fe1a178a640 1 -- 172.21.15.32:0/663018042 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe19c19f680 con 0x7fe19c102230 2024-04-25T00:32:43.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.192+0000 7fe1a178a640 1 --2- 172.21.15.32:0/663018042 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe19c102230 0x7fe19c19f010 secure :-1 s=READY pgs=383 cs=0 l=1 rev1=1 crypto rx=0x7fe194031ae0 tx=0x7fe194031ee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:43.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.192+0000 7fe199ffb640 1 -- 172.21.15.32:0/663018042 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe194003c30 con 0x7fe19c102230 2024-04-25T00:32:43.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.192+0000 7fe199ffb640 1 -- 172.21.15.32:0/663018042 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe194003dd0 con 0x7fe19c102230 2024-04-25T00:32:43.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.192+0000 7fe1a31ed640 1 -- 172.21.15.32:0/663018042 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe19c19f910 con 0x7fe19c102230 2024-04-25T00:32:43.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.192+0000 7fe199ffb640 1 -- 172.21.15.32:0/663018042 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe19402fcb0 con 0x7fe19c102230 2024-04-25T00:32:43.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.192+0000 7fe1a31ed640 1 -- 172.21.15.32:0/663018042 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe19c19fe60 con 0x7fe19c102230 2024-04-25T00:32:43.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.195+0000 7fe199ffb640 1 -- 172.21.15.32:0/663018042 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 17) ==== 137884+0+0 (secure 0 0 0) 0x7fe194004050 con 0x7fe19c102230 2024-04-25T00:32:43.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.195+0000 7fe1a5fcf740 1 Processor -- start 2024-04-25T00:32:43.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.195+0000 7fe1a5fcf740 1 -- start start 2024-04-25T00:32:43.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.195+0000 7fe1a5fcf740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd317850 0x55ebcd317c70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.195+0000 7fe1a5fcf740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ebcd318240 0x55ebcd31cd40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.195+0000 7fe1a5fcf740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ebcd31d3b0 0x55ebcd31f840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.195+0000 7fe1a5fcf740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55ebcd276f00 con 0x55ebcd317850 2024-04-25T00:32:43.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.195+0000 7fe1a5fcf740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55ebcd21f750 con 0x55ebcd31d3b0 2024-04-25T00:32:43.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.195+0000 7fe1a5fcf740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55ebcd272ec0 con 0x55ebcd318240 2024-04-25T00:32:43.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.195+0000 7fe19bfff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd317850 0x55ebcd317c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.195+0000 7fe19bfff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd317850 0x55ebcd317c70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:47888/0 (socket says 172.21.15.32:47888) 2024-04-25T00:32:43.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.195+0000 7fe19bfff640 1 -- 172.21.15.32:0/255143767 learned_addr learned my addr 172.21.15.32:0/255143767 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:43.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.195+0000 7fe199ffb640 1 --2- 172.21.15.32:0/663018042 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe1840a2c20 0x7fe1840a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.196+0000 7fe1a0f89640 1 --2- 172.21.15.32:0/255143767 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ebcd318240 0x55ebcd31cd40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.196+0000 7fe19bfff640 1 --2- 172.21.15.32:0/255143767 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ebcd31d3b0 0x55ebcd31f840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.196+0000 7fe1a178a640 1 --2- 172.21.15.32:0/663018042 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe1840a2c20 0x7fe1840a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.196+0000 7fe19bfff640 1 -- 172.21.15.32:0/255143767 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ebcd318240 msgr2=0x55ebcd31cd40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:32:43.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.196+0000 7fe19bfff640 1 --2- 172.21.15.32:0/255143767 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ebcd318240 0x55ebcd31cd40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.196+0000 7fe19bfff640 1 -- 172.21.15.32:0/255143767 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ebcd31d3b0 msgr2=0x55ebcd31f840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:32:43.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.196+0000 7fe19bfff640 1 --2- 172.21.15.32:0/255143767 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ebcd31d3b0 0x55ebcd31f840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.196+0000 7fe19bfff640 1 -- 172.21.15.32:0/255143767 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55ebccf8b2d0 con 0x55ebcd317850 2024-04-25T00:32:43.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.196+0000 7fe1a0f89640 1 --2- 172.21.15.32:0/255143767 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ebcd318240 0x55ebcd31cd40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:32:43.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.196+0000 7fe199ffb640 1 -- 172.21.15.32:0/663018042 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7fe1940f07b0 con 0x7fe19c102230 2024-04-25T00:32:43.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.196+0000 7fe19bfff640 1 --2- 172.21.15.32:0/255143767 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd317850 0x55ebcd317c70 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7fe190012390 tx=0x7fe190009720 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7d8731c9f7908b65 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:43.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.196+0000 7fe182ffd640 1 -- 172.21.15.32:0/255143767 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe19001a030 con 0x55ebcd317850 2024-04-25T00:32:43.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.196+0000 7fe1a178a640 1 --2- 172.21.15.32:0/663018042 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe1840a2c20 0x7fe1840a50e0 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7fe1940bcbf0 tx=0x7fe1940bdf90 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:43.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.196+0000 7fe182ffd640 1 -- 172.21.15.32:0/255143767 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe190012070 con 0x55ebcd317850 2024-04-25T00:32:43.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.196+0000 7fe182ffd640 1 -- 172.21.15.32:0/255143767 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe190003cd0 con 0x55ebcd317850 2024-04-25T00:32:43.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.197+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/255143767 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd317850 msgr2=0x55ebcd317c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:43.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.197+0000 7fe1a5fcf740 1 --2- 172.21.15.32:0/255143767 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd317850 0x55ebcd317c70 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7fe190012390 tx=0x7fe190009720 comp rx=0 tx=0).stop 2024-04-25T00:32:43.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.197+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/255143767 shutdown_connections 2024-04-25T00:32:43.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.197+0000 7fe1a5fcf740 1 --2- 172.21.15.32:0/255143767 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ebcd31d3b0 0x55ebcd31f840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.197+0000 7fe1a5fcf740 1 --2- 172.21.15.32:0/255143767 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ebcd318240 0x55ebcd31cd40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.197+0000 7fe1a5fcf740 1 --2- 172.21.15.32:0/255143767 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd317850 0x55ebcd317c70 unknown :-1 s=CLOSED pgs=384 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.197+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/255143767 >> 172.21.15.32:0/255143767 conn(0x55ebcd30eab0 msgr2=0x55ebcd30eeb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:43.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.197+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/255143767 shutdown_connections 2024-04-25T00:32:43.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.197+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/255143767 wait complete. 2024-04-25T00:32:43.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.197+0000 7fe1a5fcf740 1 -- start start 2024-04-25T00:32:43.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.197+0000 7fe1a5fcf740 1 Processor -- start 2024-04-25T00:32:43.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.198+0000 7fe1a5fcf740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ebcd317850 0x55ebcd321c70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.198+0000 7fe1a5fcf740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd318240 0x55ebcd3221b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.198+0000 7fe1a5fcf740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ebcd31d3b0 0x55ebcd3291f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.198+0000 7fe1a5fcf740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55ebcd320360 con 0x55ebcd318240 2024-04-25T00:32:43.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.198+0000 7fe1a5fcf740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55ebcd232cf0 con 0x55ebcd317850 2024-04-25T00:32:43.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.198+0000 7fe1a5fcf740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55ebcd3204e0 con 0x55ebcd31d3b0 2024-04-25T00:32:43.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.198+0000 7fe1a5fcf740 10 client.0 _ll_register_callbacks cb 0x55ebcd2326e0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:32:43.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.198+0000 7fe1a0f89640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd318240 0x55ebcd3221b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.198+0000 7fe1a0f89640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd318240 0x55ebcd3221b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:47900/0 (socket says 172.21.15.32:47900) 2024-04-25T00:32:43.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.198+0000 7fe1a0f89640 1 -- 172.21.15.32:0/1483741041 learned_addr learned my addr 172.21.15.32:0/1483741041 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:43.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.198+0000 7fe19bfff640 1 --2- 172.21.15.32:0/1483741041 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ebcd317850 0x55ebcd321c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.198+0000 7fe19bfff640 1 --2- 172.21.15.32:0/1483741041 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ebcd31d3b0 0x55ebcd3291f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.198+0000 7fe1a0f89640 1 -- 172.21.15.32:0/1483741041 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ebcd31d3b0 msgr2=0x55ebcd3291f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:32:43.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.198+0000 7fe1a0f89640 1 --2- 172.21.15.32:0/1483741041 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ebcd31d3b0 0x55ebcd3291f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.198+0000 7fe1a0f89640 1 -- 172.21.15.32:0/1483741041 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ebcd317850 msgr2=0x55ebcd321c70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:32:43.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.198+0000 7fe1a0f89640 1 --2- 172.21.15.32:0/1483741041 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ebcd317850 0x55ebcd321c70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.198+0000 7fe1a0f89640 1 -- 172.21.15.32:0/1483741041 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55ebcd0556b0 con 0x55ebcd318240 2024-04-25T00:32:43.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.198+0000 7fe1a0f89640 1 --2- 172.21.15.32:0/1483741041 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd318240 0x55ebcd3221b0 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 crypto rx=0x7fe18c010440 tx=0x7fe18c010900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=35167bc8d9d85cdb server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:43.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.198+0000 7fe17bfff640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:32:43.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.199+0000 7fe17bfff640 1 -- 172.21.15.32:0/1483741041 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe18c005220 con 0x55ebcd318240 2024-04-25T00:32:43.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.199+0000 7fe17bfff640 1 -- 172.21.15.32:0/1483741041 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7fe18c0053c0 con 0x55ebcd318240 2024-04-25T00:32:43.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.199+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/1483741041 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe190012070 con 0x55ebcd318240 2024-04-25T00:32:43.209 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.199+0000 7fe17bfff640 1 -- 172.21.15.32:0/1483741041 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) ==== 24+0+0 (secure 0 0 0) 0x7fe18c024050 con 0x55ebcd318240 2024-04-25T00:32:43.209 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.199+0000 7fe1a5fcf740 10 client.15279 fetch_fsmap learned FSMap version 24 2024-04-25T00:32:43.209 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.199+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/1483741041 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) -- 0x55ebcd32c280 con 0x55ebcd318240 2024-04-25T00:32:43.209 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.199+0000 7fe17bfff640 1 -- 172.21.15.32:0/1483741041 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe190012070 con 0x55ebcd318240 2024-04-25T00:32:43.209 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.199+0000 7fe17bfff640 1 -- 172.21.15.32:0/1483741041 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 24) ==== 8346+0+0 (secure 0 0 0) 0x7fe18c00f040 con 0x55ebcd318240 2024-04-25T00:32:43.209 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.199+0000 7fe1a5fcf740 10 client.15279 fetch_fsmap finished waiting for FSMap version 24 2024-04-25T00:32:43.209 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.199+0000 7fe1a5fcf740 10 client.15279 resolve_mds: resolved name 'b' to daemon mds.b 2024-04-25T00:32:43.209 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.199+0000 7fe1a5fcf740 20 client.15279 populate_metadata read hostname 'smithi032' 2024-04-25T00:32:43.209 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.199+0000 7fe1a5fcf740 1 --2- 172.21.15.32:0/1483741041 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55ebcd32e9a0 0x55ebcd32fe10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.210 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.199+0000 7fe1a5fcf740 4 client.15279 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:32:43.210 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.199+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/1483741041 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55ebcd284440 con 0x55ebcd32e9a0 2024-04-25T00:32:43.210 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.200+0000 7fe19bfff640 1 --2- 172.21.15.32:0/1483741041 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55ebcd32e9a0 0x55ebcd32fe10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.210 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.200+0000 7fe17bfff640 10 client.15279 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:32:43.210 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.200+0000 7fe19bfff640 1 --2- 172.21.15.32:0/1483741041 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55ebcd32e9a0 0x55ebcd32fe10 crc :-1 s=READY pgs=25 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b9e96677ace9d294 server_cookie=642b40a3969608c1 in_seq=0 out_seq=0 2024-04-25T00:32:43.210 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.201+0000 7fe17bfff640 1 -- 172.21.15.32:0/1483741041 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x55ebcd284440 con 0x55ebcd32e9a0 2024-04-25T00:32:43.211 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.201+0000 7fe17bfff640 10 client.15279 handle_command_reply: tid=0 2024-04-25T00:32:43.211 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.201+0000 7fe1a5fcf740 1 client.15279 shutdown 2024-04-25T00:32:43.211 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.201+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/1483741041 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd318240 msgr2=0x55ebcd3221b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:43.211 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.201+0000 7fe1a5fcf740 1 --2- 172.21.15.32:0/1483741041 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd318240 0x55ebcd3221b0 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 crypto rx=0x7fe18c010440 tx=0x7fe18c010900 comp rx=0 tx=0).stop 2024-04-25T00:32:43.211 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.201+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/1483741041 shutdown_connections 2024-04-25T00:32:43.211 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.201+0000 7fe1a5fcf740 1 --2- 172.21.15.32:0/1483741041 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55ebcd32e9a0 0x55ebcd32fe10 crc :-1 s=READY pgs=25 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.201+0000 7fe1a5fcf740 1 --2- 172.21.15.32:0/1483741041 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ebcd31d3b0 0x55ebcd3291f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.201+0000 7fe17bfff640 0 client.15279 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:32:43.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.201+0000 7fe1a5fcf740 1 --2- 172.21.15.32:0/1483741041 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd318240 0x55ebcd3221b0 unknown :-1 s=CLOSED pgs=385 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.201+0000 7fe1a5fcf740 1 --2- 172.21.15.32:0/1483741041 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ebcd317850 0x55ebcd321c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.201+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/1483741041 >> 172.21.15.32:0/1483741041 conn(0x55ebcd30eab0 msgr2=0x55ebcd30dae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:43.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.201+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/1483741041 shutdown_connections 2024-04-25T00:32:43.213 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.201+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/1483741041 wait complete. 2024-04-25T00:32:43.213 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.201+0000 7fe1a5fcf740 20 client.15279 trim_cache size 0 max 16384 2024-04-25T00:32:43.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.219+0000 7fe1a5fcf740 1 Processor -- start 2024-04-25T00:32:43.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.219+0000 7fe1a5fcf740 1 -- start start 2024-04-25T00:32:43.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.219+0000 7fe1a5fcf740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ebcd317850 0x55ebcd32fe10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.219+0000 7fe1a5fcf740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd318240 0x55ebcd3291b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.219+0000 7fe1a5fcf740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ebcd31d3b0 0x55ebcd321880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.219+0000 7fe1a5fcf740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55ebcd32e150 con 0x55ebcd318240 2024-04-25T00:32:43.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.219+0000 7fe1a5fcf740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55ebcd32e820 con 0x55ebcd31d3b0 2024-04-25T00:32:43.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.219+0000 7fe1a5fcf740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55ebcd32eef0 con 0x55ebcd317850 2024-04-25T00:32:43.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.219+0000 7fe1a0f89640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd318240 0x55ebcd3291b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.219+0000 7fe1a0f89640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd318240 0x55ebcd3291b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:47912/0 (socket says 172.21.15.32:47912) 2024-04-25T00:32:43.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.219+0000 7fe1a0f89640 1 -- 172.21.15.32:0/154582401 learned_addr learned my addr 172.21.15.32:0/154582401 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:43.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.220+0000 7fe19bfff640 1 --2- 172.21.15.32:0/154582401 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ebcd31d3b0 0x55ebcd321880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.220+0000 7fe1a0f89640 1 -- 172.21.15.32:0/154582401 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ebcd317850 msgr2=0x55ebcd32fe10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:32:43.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.220+0000 7fe1a0f89640 1 --2- 172.21.15.32:0/154582401 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ebcd317850 0x55ebcd32fe10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.220+0000 7fe1a0f89640 1 -- 172.21.15.32:0/154582401 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ebcd31d3b0 msgr2=0x55ebcd321880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:32:43.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.220+0000 7fe1a0f89640 1 --2- 172.21.15.32:0/154582401 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ebcd31d3b0 0x55ebcd321880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.220+0000 7fe1a0f89640 1 -- 172.21.15.32:0/154582401 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55ebcd32fa70 con 0x55ebcd318240 2024-04-25T00:32:43.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.220+0000 7fe1a0f89640 1 --2- 172.21.15.32:0/154582401 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd318240 0x55ebcd3291b0 secure :-1 s=READY pgs=386 cs=0 l=1 rev1=1 crypto rx=0x7fe18c012bc0 tx=0x7fe18c011500 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f26505eec340cedc server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:43.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.220+0000 7fe1827fc640 1 -- 172.21.15.32:0/154582401 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe18c012720 con 0x55ebcd318240 2024-04-25T00:32:43.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.220+0000 7fe1827fc640 1 -- 172.21.15.32:0/154582401 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe18c0128c0 con 0x55ebcd318240 2024-04-25T00:32:43.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.220+0000 7fe1827fc640 1 -- 172.21.15.32:0/154582401 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe18c0048a0 con 0x55ebcd318240 2024-04-25T00:32:43.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.221+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/154582401 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd318240 msgr2=0x55ebcd3291b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:43.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.221+0000 7fe1a5fcf740 1 --2- 172.21.15.32:0/154582401 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd318240 0x55ebcd3291b0 secure :-1 s=READY pgs=386 cs=0 l=1 rev1=1 crypto rx=0x7fe18c012bc0 tx=0x7fe18c011500 comp rx=0 tx=0).stop 2024-04-25T00:32:43.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.221+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/154582401 shutdown_connections 2024-04-25T00:32:43.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.221+0000 7fe1a5fcf740 1 --2- 172.21.15.32:0/154582401 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ebcd31d3b0 0x55ebcd321880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.221+0000 7fe1a5fcf740 1 --2- 172.21.15.32:0/154582401 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd318240 0x55ebcd3291b0 unknown :-1 s=CLOSED pgs=386 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.221+0000 7fe1a5fcf740 1 --2- 172.21.15.32:0/154582401 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ebcd317850 0x55ebcd32fe10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.221+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/154582401 >> 172.21.15.32:0/154582401 conn(0x55ebcd30eab0 msgr2=0x55ebcd309690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:43.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.221+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/154582401 shutdown_connections 2024-04-25T00:32:43.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.221+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/154582401 wait complete. 2024-04-25T00:32:43.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.221+0000 7fe1a5fcf740 1 -- start start 2024-04-25T00:32:43.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.221+0000 7fe1a5fcf740 1 Processor -- start 2024-04-25T00:32:43.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.221+0000 7fe1a5fcf740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd317850 0x55ebcd328ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.221+0000 7fe1a5fcf740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ebcd318240 0x55ebcd329010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.221+0000 7fe1a5fcf740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ebcd31d3b0 0x55ebcd325750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.221+0000 7fe1a5fcf740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55ebcd32fc90 con 0x55ebcd317850 2024-04-25T00:32:43.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.221+0000 7fe1a5fcf740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55ebcd32f5c0 con 0x55ebcd31d3b0 2024-04-25T00:32:43.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.221+0000 7fe1a5fcf740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55ebcd330360 con 0x55ebcd318240 2024-04-25T00:32:43.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.222+0000 7fe1a5fcf740 10 client.0 _ll_register_callbacks cb 0x7fe18c00ec70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:32:43.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.222+0000 7fe19bfff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd317850 0x55ebcd328ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.222+0000 7fe19bfff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd317850 0x55ebcd328ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:47926/0 (socket says 172.21.15.32:47926) 2024-04-25T00:32:43.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.222+0000 7fe19bfff640 1 -- 172.21.15.32:0/863452039 learned_addr learned my addr 172.21.15.32:0/863452039 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:43.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.222+0000 7fe1a0f89640 1 --2- 172.21.15.32:0/863452039 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ebcd318240 0x55ebcd329010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.222+0000 7fe19bfff640 1 --2- 172.21.15.32:0/863452039 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ebcd31d3b0 0x55ebcd325750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.222+0000 7fe19bfff640 1 -- 172.21.15.32:0/863452039 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ebcd318240 msgr2=0x55ebcd329010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:32:43.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.222+0000 7fe19bfff640 1 --2- 172.21.15.32:0/863452039 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ebcd318240 0x55ebcd329010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.222+0000 7fe19bfff640 1 -- 172.21.15.32:0/863452039 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ebcd31d3b0 msgr2=0x55ebcd325750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:32:43.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.222+0000 7fe19bfff640 1 --2- 172.21.15.32:0/863452039 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ebcd31d3b0 0x55ebcd325750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.222+0000 7fe19bfff640 1 -- 172.21.15.32:0/863452039 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55ebcd325ec0 con 0x55ebcd317850 2024-04-25T00:32:43.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.222+0000 7fe19bfff640 1 --2- 172.21.15.32:0/863452039 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd317850 0x55ebcd328ad0 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 crypto rx=0x7fe190012390 tx=0x7fe190005f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=962a49b30bf3115 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:43.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.222+0000 7fe17bfff640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:32:43.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.222+0000 7fe17bfff640 1 -- 172.21.15.32:0/863452039 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe190005230 con 0x55ebcd317850 2024-04-25T00:32:43.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.222+0000 7fe17bfff640 1 -- 172.21.15.32:0/863452039 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 9092+0+0 (secure 0 0 0) 0x7fe1900053d0 con 0x55ebcd317850 2024-04-25T00:32:43.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.222+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/863452039 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe18c0128c0 con 0x55ebcd317850 2024-04-25T00:32:43.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.222+0000 7fe17bfff640 1 -- 172.21.15.32:0/863452039 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) ==== 24+0+0 (secure 0 0 0) 0x7fe190003fa0 con 0x55ebcd317850 2024-04-25T00:32:43.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.222+0000 7fe1a5fcf740 10 client.15285 fetch_fsmap learned FSMap version 24 2024-04-25T00:32:43.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.222+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/863452039 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) -- 0x55ebcd395660 con 0x55ebcd317850 2024-04-25T00:32:43.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.223+0000 7fe17bfff640 1 -- 172.21.15.32:0/863452039 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe18c0128c0 con 0x55ebcd317850 2024-04-25T00:32:43.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.223+0000 7fe17bfff640 1 -- 172.21.15.32:0/863452039 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 24) ==== 8346+0+0 (secure 0 0 0) 0x7fe19000f040 con 0x55ebcd317850 2024-04-25T00:32:43.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.223+0000 7fe1a5fcf740 10 client.15285 fetch_fsmap finished waiting for FSMap version 24 2024-04-25T00:32:43.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.223+0000 7fe1a5fcf740 10 client.15285 resolve_mds: resolved name 'b' to daemon mds.b 2024-04-25T00:32:43.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.223+0000 7fe1a5fcf740 20 client.15285 populate_metadata read hostname 'smithi032' 2024-04-25T00:32:43.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.223+0000 7fe1a5fcf740 1 --2- 172.21.15.32:0/863452039 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55ebcd32e9a0 0x55ebcd3964c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:43.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.223+0000 7fe1a5fcf740 4 client.15285 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "status"}] 2024-04-25T00:32:43.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.223+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/863452039 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "status"}) -- 0x55ebcd27e790 con 0x55ebcd32e9a0 2024-04-25T00:32:43.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.223+0000 7fe1a0f89640 1 --2- 172.21.15.32:0/863452039 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55ebcd32e9a0 0x55ebcd3964c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:43.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.223+0000 7fe17bfff640 10 client.15285 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:32:43.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.223+0000 7fe1a0f89640 1 --2- 172.21.15.32:0/863452039 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55ebcd32e9a0 0x55ebcd3964c0 crc :-1 s=READY pgs=26 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e8b59cba33b63c9a server_cookie=7a73824a01114d3d in_seq=0 out_seq=0 2024-04-25T00:32:43.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.224+0000 7fe17bfff640 1 -- 172.21.15.32:0/863452039 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+321 (crc 0 0 0) 0x55ebcd27e790 con 0x55ebcd32e9a0 2024-04-25T00:32:43.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.224+0000 7fe17bfff640 10 client.15285 handle_command_reply: tid=0 2024-04-25T00:32:43.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.224+0000 7fe1a5fcf740 1 client.15285 shutdown 2024-04-25T00:32:43.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.224+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/863452039 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd317850 msgr2=0x55ebcd328ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:43.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.224+0000 7fe1a5fcf740 1 --2- 172.21.15.32:0/863452039 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd317850 0x55ebcd328ad0 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 crypto rx=0x7fe190012390 tx=0x7fe190005f90 comp rx=0 tx=0).stop 2024-04-25T00:32:43.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.224+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/863452039 shutdown_connections 2024-04-25T00:32:43.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.224+0000 7fe1a5fcf740 1 --2- 172.21.15.32:0/863452039 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55ebcd32e9a0 0x55ebcd3964c0 crc :-1 s=READY pgs=26 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.224+0000 7fe1a5fcf740 1 --2- 172.21.15.32:0/863452039 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ebcd31d3b0 0x55ebcd325750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.224+0000 7fe1a5fcf740 1 --2- 172.21.15.32:0/863452039 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ebcd318240 0x55ebcd329010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.224+0000 7fe17bfff640 0 client.15285 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:32:43.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.224+0000 7fe1a5fcf740 1 --2- 172.21.15.32:0/863452039 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ebcd317850 0x55ebcd328ad0 unknown :-1 s=CLOSED pgs=387 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.224+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/863452039 >> 172.21.15.32:0/863452039 conn(0x55ebcd30eab0 msgr2=0x55ebcd321880 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:43.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.224+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/863452039 shutdown_connections 2024-04-25T00:32:43.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.224+0000 7fe1a5fcf740 1 -- 172.21.15.32:0/863452039 wait complete. 2024-04-25T00:32:43.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.224+0000 7fe1a5fcf740 20 client.15285 trim_cache size 0 max 16384 2024-04-25T00:32:43.231 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:32:43.232 INFO:teuthology.orchestra.run.smithi032.stdout: "cluster_fsid": "cb997156-0299-11ef-bc93-c7b262605968", 2024-04-25T00:32:43.232 INFO:teuthology.orchestra.run.smithi032.stdout: "whoami": 0, 2024-04-25T00:32:43.232 INFO:teuthology.orchestra.run.smithi032.stdout: "id": 24506, 2024-04-25T00:32:43.232 INFO:teuthology.orchestra.run.smithi032.stdout: "want_state": "up:active", 2024-04-25T00:32:43.232 INFO:teuthology.orchestra.run.smithi032.stdout: "state": "up:active", 2024-04-25T00:32:43.232 INFO:teuthology.orchestra.run.smithi032.stdout: "fs_name": "cephfs", 2024-04-25T00:32:43.232 INFO:teuthology.orchestra.run.smithi032.stdout: "rank_uptime": 114.488623088, 2024-04-25T00:32:43.232 INFO:teuthology.orchestra.run.smithi032.stdout: "mdsmap_epoch": 24, 2024-04-25T00:32:43.232 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_epoch": 91, 2024-04-25T00:32:43.232 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_epoch_barrier": 89, 2024-04-25T00:32:43.232 INFO:teuthology.orchestra.run.smithi032.stdout: "uptime": 120.89516384 2024-04-25T00:32:43.232 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:32:43.233 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.224+0000 7fe1a31ed640 1 -- 172.21.15.32:0/663018042 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe1840a2c20 msgr2=0x7fe1840a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:43.233 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.224+0000 7fe1a31ed640 1 --2- 172.21.15.32:0/663018042 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe1840a2c20 0x7fe1840a50e0 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7fe1940bcbf0 tx=0x7fe1940bdf90 comp rx=0 tx=0).stop 2024-04-25T00:32:43.233 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.225+0000 7fe1a31ed640 1 -- 172.21.15.32:0/663018042 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe19c102230 msgr2=0x7fe19c19f010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:43.233 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.225+0000 7fe1a31ed640 1 --2- 172.21.15.32:0/663018042 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe19c102230 0x7fe19c19f010 secure :-1 s=READY pgs=383 cs=0 l=1 rev1=1 crypto rx=0x7fe194031ae0 tx=0x7fe194031ee0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.233 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.225+0000 7fe1a31ed640 1 -- 172.21.15.32:0/663018042 shutdown_connections 2024-04-25T00:32:43.233 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.225+0000 7fe1a31ed640 1 --2- 172.21.15.32:0/663018042 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe1840a2c20 0x7fe1840a50e0 unknown :-1 s=CLOSED pgs=238 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.225+0000 7fe1a31ed640 1 --2- 172.21.15.32:0/663018042 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe19c102230 0x7fe19c19f010 unknown :-1 s=CLOSED pgs=383 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.225+0000 7fe1a31ed640 1 --2- 172.21.15.32:0/663018042 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe19c101870 0x7fe19c197fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.225+0000 7fe1a31ed640 1 --2- 172.21.15.32:0/663018042 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe19c100680 0x7fe19c197a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:43.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.225+0000 7fe1a31ed640 1 -- 172.21.15.32:0/663018042 >> 172.21.15.32:0/663018042 conn(0x7fe19c0fbbf0 msgr2=0x7fe19c104b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:43.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.225+0000 7fe1a31ed640 1 -- 172.21.15.32:0/663018042 shutdown_connections 2024-04-25T00:32:43.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:43.225+0000 7fe1a31ed640 1 -- 172.21.15.32:0/663018042 wait complete. 2024-04-25T00:32:43.236 INFO:tasks.cephadm:Setup complete, yielding 2024-04-25T00:32:43.236 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-25T00:32:43.250 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi032.front.sepia.ceph.com 2024-04-25T00:32:43.250 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- bash -c 'ceph orch status' 2024-04-25T00:32:43.500 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:32:43.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:43 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1241704843' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:32:43.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:43 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1241704843' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:32:43.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:43 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1241704843' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:32:44.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:44 smithi032 ceph-mon[29357]: pgmap v351: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 7 op/s 2024-04-25T00:32:44.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:44 smithi032 ceph-mon[29357]: mgrmap e18: x(active, since 9m), standbys: z, y 2024-04-25T00:32:44.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [8, 11]}]: dispatch 2024-04-25T00:32:44.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [2, 1]}]: dispatch 2024-04-25T00:32:44.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [8, 11]}]: dispatch 2024-04-25T00:32:44.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:44 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.10", "id": [2, 4]}]: dispatch 2024-04-25T00:32:44.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:44 smithi121 ceph-mon[32231]: pgmap v351: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 7 op/s 2024-04-25T00:32:44.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:44 smithi121 ceph-mon[32231]: mgrmap e18: x(active, since 9m), standbys: z, y 2024-04-25T00:32:44.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [8, 11]}]: dispatch 2024-04-25T00:32:44.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [2, 1]}]: dispatch 2024-04-25T00:32:44.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [8, 11]}]: dispatch 2024-04-25T00:32:44.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:44 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.10", "id": [2, 4]}]: dispatch 2024-04-25T00:32:44.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:44 smithi130 ceph-mon[32525]: pgmap v351: 129 pgs: 129 active+clean; 641 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 7 op/s 2024-04-25T00:32:44.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:44 smithi130 ceph-mon[32525]: mgrmap e18: x(active, since 9m), standbys: z, y 2024-04-25T00:32:44.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [8, 11]}]: dispatch 2024-04-25T00:32:44.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [2, 1]}]: dispatch 2024-04-25T00:32:44.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [8, 11]}]: dispatch 2024-04-25T00:32:44.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:44 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.10", "id": [2, 4]}]: dispatch 2024-04-25T00:32:45.448 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.448+0000 7fdf72b2c700 1 -- 172.21.15.32:0/996208653 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdf6c102d20 msgr2=0x7fdf6c1095d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:45.448 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.448+0000 7fdf72b2c700 1 --2- 172.21.15.32:0/996208653 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdf6c102d20 0x7fdf6c1095d0 secure :-1 s=READY pgs=388 cs=0 l=1 rev1=1 crypto rx=0x7fdf6400b0a0 tx=0x7fdf6400b3b0 comp rx=0 tx=0).stop 2024-04-25T00:32:45.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.448+0000 7fdf72b2c700 1 -- 172.21.15.32:0/996208653 shutdown_connections 2024-04-25T00:32:45.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.448+0000 7fdf72b2c700 1 --2- 172.21.15.32:0/996208653 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdf6c102d20 0x7fdf6c1095d0 unknown :-1 s=CLOSED pgs=388 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:45.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.448+0000 7fdf72b2c700 1 --2- 172.21.15.32:0/996208653 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdf6c101180 0x7fdf6c1015a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:45.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.448+0000 7fdf72b2c700 1 --2- 172.21.15.32:0/996208653 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdf6c102380 0x7fdf6c1027e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:45.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.448+0000 7fdf72b2c700 1 -- 172.21.15.32:0/996208653 >> 172.21.15.32:0/996208653 conn(0x7fdf6c0fc720 msgr2=0x7fdf6c0feb60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:45.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.448+0000 7fdf72b2c700 1 -- 172.21.15.32:0/996208653 shutdown_connections 2024-04-25T00:32:45.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.448+0000 7fdf72b2c700 1 -- 172.21.15.32:0/996208653 wait complete. 2024-04-25T00:32:45.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.449+0000 7fdf72b2c700 1 Processor -- start 2024-04-25T00:32:45.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.449+0000 7fdf72b2c700 1 -- start start 2024-04-25T00:32:45.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.449+0000 7fdf72b2c700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdf6c101180 0x7fdf6c1983e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:45.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.449+0000 7fdf72b2c700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdf6c102380 0x7fdf6c198920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:45.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.449+0000 7fdf72b2c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdf6c102d20 0x7fdf6c19f9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:45.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.449+0000 7fdf72b2c700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fdf6c10dea0 con 0x7fdf6c101180 2024-04-25T00:32:45.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.449+0000 7fdf72b2c700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fdf6c10dd20 con 0x7fdf6c102380 2024-04-25T00:32:45.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.449+0000 7fdf72b2c700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fdf6c10e020 con 0x7fdf6c102d20 2024-04-25T00:32:45.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.449+0000 7fdf708c8700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdf6c101180 0x7fdf6c1983e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:45.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.449+0000 7fdf708c8700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdf6c101180 0x7fdf6c1983e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:47952/0 (socket says 172.21.15.32:47952) 2024-04-25T00:32:45.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.449+0000 7fdf708c8700 1 -- 172.21.15.32:0/4202043729 learned_addr learned my addr 172.21.15.32:0/4202043729 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:45.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.449+0000 7fdf710c9700 1 --2- 172.21.15.32:0/4202043729 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdf6c102d20 0x7fdf6c19f9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:45.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.449+0000 7fdf6bfff700 1 --2- 172.21.15.32:0/4202043729 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdf6c102380 0x7fdf6c198920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:45.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.449+0000 7fdf708c8700 1 -- 172.21.15.32:0/4202043729 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdf6c102d20 msgr2=0x7fdf6c19f9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:45.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.449+0000 7fdf708c8700 1 --2- 172.21.15.32:0/4202043729 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdf6c102d20 0x7fdf6c19f9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:45.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.449+0000 7fdf708c8700 1 -- 172.21.15.32:0/4202043729 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdf6c102380 msgr2=0x7fdf6c198920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:45.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.449+0000 7fdf708c8700 1 --2- 172.21.15.32:0/4202043729 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdf6c102380 0x7fdf6c198920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:45.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.449+0000 7fdf708c8700 1 -- 172.21.15.32:0/4202043729 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fdf6c1a0040 con 0x7fdf6c101180 2024-04-25T00:32:45.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.449+0000 7fdf710c9700 1 --2- 172.21.15.32:0/4202043729 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdf6c102d20 0x7fdf6c19f9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:32:45.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.450+0000 7fdf708c8700 1 --2- 172.21.15.32:0/4202043729 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdf6c101180 0x7fdf6c1983e0 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 crypto rx=0x7fdf6000d9b0 tx=0x7fdf6000dd70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:45.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.450+0000 7fdf69ffb700 1 -- 172.21.15.32:0/4202043729 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdf60016070 con 0x7fdf6c101180 2024-04-25T00:32:45.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.450+0000 7fdf69ffb700 1 -- 172.21.15.32:0/4202043729 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fdf6000bb60 con 0x7fdf6c101180 2024-04-25T00:32:45.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.450+0000 7fdf72b2c700 1 -- 172.21.15.32:0/4202043729 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fdf6c1a0330 con 0x7fdf6c101180 2024-04-25T00:32:45.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.450+0000 7fdf69ffb700 1 -- 172.21.15.32:0/4202043729 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdf6000f3c0 con 0x7fdf6c101180 2024-04-25T00:32:45.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.450+0000 7fdf72b2c700 1 -- 172.21.15.32:0/4202043729 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fdf6c1a08c0 con 0x7fdf6c101180 2024-04-25T00:32:45.453 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.451+0000 7fdf72b2c700 1 -- 172.21.15.32:0/4202043729 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fdf6c068f20 con 0x7fdf6c101180 2024-04-25T00:32:45.453 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.452+0000 7fdf69ffb700 1 -- 172.21.15.32:0/4202043729 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 18) ==== 137833+0+0 (secure 0 0 0) 0x7fdf60019070 con 0x7fdf6c101180 2024-04-25T00:32:45.453 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.453+0000 7fdf69ffb700 1 --2- 172.21.15.32:0/4202043729 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdf5c0a2ca0 0x7fdf5c0a5160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:45.453 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.453+0000 7fdf6bfff700 1 --2- 172.21.15.32:0/4202043729 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdf5c0a2ca0 0x7fdf5c0a5160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:45.453 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.453+0000 7fdf69ffb700 1 -- 172.21.15.32:0/4202043729 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 9208+0+0 (secure 0 0 0) 0x7fdf600cefd0 con 0x7fdf6c101180 2024-04-25T00:32:45.454 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.453+0000 7fdf6bfff700 1 --2- 172.21.15.32:0/4202043729 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdf5c0a2ca0 0x7fdf5c0a5160 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7fdf5800bbb0 tx=0x7fdf5800b590 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:45.457 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.457+0000 7fdf69ffb700 1 -- 172.21.15.32:0/4202043729 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fdf60020030 con 0x7fdf6c101180 2024-04-25T00:32:45.691 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.691+0000 7fdf72b2c700 1 -- 172.21.15.32:0/4202043729 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch status", "target": ["mon-mgr", ""]}) -- 0x7fdf6c062670 con 0x7fdf5c0a2ca0 2024-04-25T00:32:45.692 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [8, 11]}]': finished 2024-04-25T00:32:45.692 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [2, 1]}]': finished 2024-04-25T00:32:45.692 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [8, 11]}]': finished 2024-04-25T00:32:45.692 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:45 smithi032 ceph-mon[29357]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.10", "id": [2, 4]}]': finished 2024-04-25T00:32:45.692 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:45 smithi032 ceph-mon[29357]: osdmap e92: 12 total, 12 up, 12 in 2024-04-25T00:32:45.692 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.692+0000 7fdf69ffb700 1 -- 172.21.15.32:0/4202043729 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+43 (secure 0 0 0) 0x7fdf6c062670 con 0x7fdf5c0a2ca0 2024-04-25T00:32:45.692 INFO:teuthology.orchestra.run.smithi032.stdout:Backend: cephadm 2024-04-25T00:32:45.693 INFO:teuthology.orchestra.run.smithi032.stdout:Available: Yes 2024-04-25T00:32:45.693 INFO:teuthology.orchestra.run.smithi032.stdout:Paused: No 2024-04-25T00:32:45.695 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.694+0000 7fdf72b2c700 1 -- 172.21.15.32:0/4202043729 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdf5c0a2ca0 msgr2=0x7fdf5c0a5160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:45.695 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.694+0000 7fdf72b2c700 1 --2- 172.21.15.32:0/4202043729 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdf5c0a2ca0 0x7fdf5c0a5160 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7fdf5800bbb0 tx=0x7fdf5800b590 comp rx=0 tx=0).stop 2024-04-25T00:32:45.695 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.694+0000 7fdf72b2c700 1 -- 172.21.15.32:0/4202043729 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdf6c101180 msgr2=0x7fdf6c1983e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:45.695 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.694+0000 7fdf72b2c700 1 --2- 172.21.15.32:0/4202043729 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdf6c101180 0x7fdf6c1983e0 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 crypto rx=0x7fdf6000d9b0 tx=0x7fdf6000dd70 comp rx=0 tx=0).stop 2024-04-25T00:32:45.695 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.694+0000 7fdf72b2c700 1 -- 172.21.15.32:0/4202043729 shutdown_connections 2024-04-25T00:32:45.695 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.694+0000 7fdf72b2c700 1 --2- 172.21.15.32:0/4202043729 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdf5c0a2ca0 0x7fdf5c0a5160 unknown :-1 s=CLOSED pgs=239 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:45.695 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.694+0000 7fdf72b2c700 1 --2- 172.21.15.32:0/4202043729 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdf6c102d20 0x7fdf6c19f9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:45.695 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.694+0000 7fdf72b2c700 1 --2- 172.21.15.32:0/4202043729 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdf6c101180 0x7fdf6c1983e0 unknown :-1 s=CLOSED pgs=389 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:45.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.694+0000 7fdf72b2c700 1 --2- 172.21.15.32:0/4202043729 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdf6c102380 0x7fdf6c198920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:45.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.694+0000 7fdf72b2c700 1 -- 172.21.15.32:0/4202043729 >> 172.21.15.32:0/4202043729 conn(0x7fdf6c0fc720 msgr2=0x7fdf6c105620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:45.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.694+0000 7fdf72b2c700 1 -- 172.21.15.32:0/4202043729 shutdown_connections 2024-04-25T00:32:45.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:45.694+0000 7fdf72b2c700 1 -- 172.21.15.32:0/4202043729 wait complete. 2024-04-25T00:32:45.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [8, 11]}]': finished 2024-04-25T00:32:45.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [2, 1]}]': finished 2024-04-25T00:32:45.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [8, 11]}]': finished 2024-04-25T00:32:45.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:45 smithi121 ceph-mon[32231]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.10", "id": [2, 4]}]': finished 2024-04-25T00:32:45.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:45 smithi121 ceph-mon[32231]: osdmap e92: 12 total, 12 up, 12 in 2024-04-25T00:32:45.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [8, 11]}]': finished 2024-04-25T00:32:45.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [2, 1]}]': finished 2024-04-25T00:32:45.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [8, 11]}]': finished 2024-04-25T00:32:45.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:45 smithi130 ceph-mon[32525]: from='mgr.14148 172.21.15.32:0/1018635330' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.10", "id": [2, 4]}]': finished 2024-04-25T00:32:45.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:45 smithi130 ceph-mon[32525]: osdmap e92: 12 total, 12 up, 12 in 2024-04-25T00:32:46.459 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- bash -c 'ceph orch ps' 2024-04-25T00:32:46.710 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:32:46.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:46 smithi032 ceph-mon[29357]: pgmap v353: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:32:46.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:46 smithi032 ceph-mon[29357]: osdmap e93: 12 total, 12 up, 12 in 2024-04-25T00:32:46.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:46 smithi032 ceph-mon[29357]: from='client.15291 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:32:46.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:46 smithi121 ceph-mon[32231]: pgmap v353: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:32:46.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:46 smithi121 ceph-mon[32231]: osdmap e93: 12 total, 12 up, 12 in 2024-04-25T00:32:46.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:46 smithi121 ceph-mon[32231]: from='client.15291 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:32:46.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:46 smithi130 ceph-mon[32525]: pgmap v353: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:32:46.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:46 smithi130 ceph-mon[32525]: osdmap e93: 12 total, 12 up, 12 in 2024-04-25T00:32:46.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:46 smithi130 ceph-mon[32525]: from='client.15291 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:32:48.618 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.616+0000 7fe98652f700 1 -- 172.21.15.32:0/2746735118 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe980102d30 msgr2=0x7fe9801095e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:48.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.616+0000 7fe98652f700 1 --2- 172.21.15.32:0/2746735118 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe980102d30 0x7fe9801095e0 secure :-1 s=READY pgs=390 cs=0 l=1 rev1=1 crypto rx=0x7fe97800b0d0 tx=0x7fe97800b3e0 comp rx=0 tx=0).stop 2024-04-25T00:32:48.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.616+0000 7fe98652f700 1 -- 172.21.15.32:0/2746735118 shutdown_connections 2024-04-25T00:32:48.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.616+0000 7fe98652f700 1 --2- 172.21.15.32:0/2746735118 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe980102d30 0x7fe9801095e0 unknown :-1 s=CLOSED pgs=390 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:48.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.616+0000 7fe98652f700 1 --2- 172.21.15.32:0/2746735118 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe980101190 0x7fe9801015b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:48.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.616+0000 7fe98652f700 1 --2- 172.21.15.32:0/2746735118 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe980102390 0x7fe9801027f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:48.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.616+0000 7fe98652f700 1 -- 172.21.15.32:0/2746735118 >> 172.21.15.32:0/2746735118 conn(0x7fe9800fc730 msgr2=0x7fe9800feb70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:48.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.616+0000 7fe98652f700 1 -- 172.21.15.32:0/2746735118 shutdown_connections 2024-04-25T00:32:48.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.616+0000 7fe98652f700 1 -- 172.21.15.32:0/2746735118 wait complete. 2024-04-25T00:32:48.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.616+0000 7fe98652f700 1 Processor -- start 2024-04-25T00:32:48.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.616+0000 7fe98652f700 1 -- start start 2024-04-25T00:32:48.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.617+0000 7fe98652f700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe980101190 0x7fe98019a690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:48.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.617+0000 7fe98652f700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe980102390 0x7fe98019abd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:48.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.617+0000 7fe98652f700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe980102d30 0x7fe9801a1c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:48.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.617+0000 7fe98652f700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fe98010cdf0 con 0x7fe980102390 2024-04-25T00:32:48.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.617+0000 7fe98652f700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fe98010cc70 con 0x7fe980101190 2024-04-25T00:32:48.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.617+0000 7fe98652f700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fe98010cf70 con 0x7fe980102d30 2024-04-25T00:32:48.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.617+0000 7fe97f7fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe980102390 0x7fe98019abd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:48.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.617+0000 7fe97f7fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe980102390 0x7fe98019abd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:47966/0 (socket says 172.21.15.32:47966) 2024-04-25T00:32:48.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.617+0000 7fe97f7fe700 1 -- 172.21.15.32:0/3552612624 learned_addr learned my addr 172.21.15.32:0/3552612624 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:48.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.617+0000 7fe97ffff700 1 --2- 172.21.15.32:0/3552612624 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe980101190 0x7fe98019a690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:48.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.617+0000 7fe984acc700 1 --2- 172.21.15.32:0/3552612624 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe980102d30 0x7fe9801a1c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:48.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.617+0000 7fe97f7fe700 1 -- 172.21.15.32:0/3552612624 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe980102d30 msgr2=0x7fe9801a1c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:48.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.617+0000 7fe97f7fe700 1 --2- 172.21.15.32:0/3552612624 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe980102d30 0x7fe9801a1c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:48.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.617+0000 7fe97f7fe700 1 -- 172.21.15.32:0/3552612624 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe980101190 msgr2=0x7fe98019a690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:48.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.617+0000 7fe97f7fe700 1 --2- 172.21.15.32:0/3552612624 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe980101190 0x7fe98019a690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:48.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.617+0000 7fe97f7fe700 1 -- 172.21.15.32:0/3552612624 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe9801a22f0 con 0x7fe980102390 2024-04-25T00:32:48.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.617+0000 7fe97ffff700 1 --2- 172.21.15.32:0/3552612624 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe980101190 0x7fe98019a690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:32:48.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.618+0000 7fe97f7fe700 1 --2- 172.21.15.32:0/3552612624 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe980102390 0x7fe98019abd0 secure :-1 s=READY pgs=391 cs=0 l=1 rev1=1 crypto rx=0x7fe97400c9e0 tx=0x7fe97400ccf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:48.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.618+0000 7fe97d7fa700 1 -- 172.21.15.32:0/3552612624 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe974007ac0 con 0x7fe980102390 2024-04-25T00:32:48.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.618+0000 7fe98652f700 1 -- 172.21.15.32:0/3552612624 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe9801a25e0 con 0x7fe980102390 2024-04-25T00:32:48.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.618+0000 7fe97d7fa700 1 -- 172.21.15.32:0/3552612624 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe974007c60 con 0x7fe980102390 2024-04-25T00:32:48.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.618+0000 7fe98652f700 1 -- 172.21.15.32:0/3552612624 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe9801a2bf0 con 0x7fe980102390 2024-04-25T00:32:48.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.618+0000 7fe97d7fa700 1 -- 172.21.15.32:0/3552612624 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe974018730 con 0x7fe980102390 2024-04-25T00:32:48.622 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.619+0000 7fe98652f700 1 -- 172.21.15.32:0/3552612624 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe980068f20 con 0x7fe980102390 2024-04-25T00:32:48.622 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.621+0000 7fe97d7fa700 1 -- 172.21.15.32:0/3552612624 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 18) ==== 137833+0+0 (secure 0 0 0) 0x7fe9740188d0 con 0x7fe980102390 2024-04-25T00:32:48.622 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.621+0000 7fe97d7fa700 1 --2- 172.21.15.32:0/3552612624 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe9680a2c20 0x7fe9680a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:48.622 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.621+0000 7fe97ffff700 1 --2- 172.21.15.32:0/3552612624 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe9680a2c20 0x7fe9680a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:48.622 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.621+0000 7fe97d7fa700 1 -- 172.21.15.32:0/3552612624 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7fe9740cf690 con 0x7fe980102390 2024-04-25T00:32:48.623 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.622+0000 7fe97ffff700 1 --2- 172.21.15.32:0/3552612624 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe9680a2c20 0x7fe9680a50e0 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7fe9801021f0 tx=0x7fe970008040 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:48.625 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.625+0000 7fe97d7fa700 1 -- 172.21.15.32:0/3552612624 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fe974014030 con 0x7fe980102390 2024-04-25T00:32:48.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:48 smithi032 ceph-mon[29357]: pgmap v355: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 1.9 KiB/s rd, 127 B/s wr, 4 op/s 2024-04-25T00:32:48.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:48 smithi121 ceph-mon[32231]: pgmap v355: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 1.9 KiB/s rd, 127 B/s wr, 4 op/s 2024-04-25T00:32:48.843 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.843+0000 7fe98652f700 1 -- 172.21.15.32:0/3552612624 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch ps", "target": ["mon-mgr", ""]}) -- 0x7fe98006a620 con 0x7fe9680a2c20 2024-04-25T00:32:48.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.857+0000 7fe97d7fa700 1 -- 172.21.15.32:0/3552612624 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+4092 (secure 0 0 0) 0x7fe98006a620 con 0x7fe9680a2c20 2024-04-25T00:32:48.858 INFO:teuthology.orchestra.run.smithi032.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T00:32:48.858 INFO:teuthology.orchestra.run.smithi032.stdout:mds.a smithi032 running (2m) 21s ago 2m 13.9M - 19.0.0-2445-gd2c45d97 f68e37b807ef cdcac5998a3b 2024-04-25T00:32:48.858 INFO:teuthology.orchestra.run.smithi032.stdout:mds.b smithi121 running (2m) 60s ago 2m 28.1M - 19.0.0-2445-gd2c45d97 f68e37b807ef 43cd35e25c64 2024-04-25T00:32:48.859 INFO:teuthology.orchestra.run.smithi032.stdout:mds.c smithi130 running (2m) 60s ago 2m 16.9M - 19.0.0-2445-gd2c45d97 f68e37b807ef 73fc15c8c094 2024-04-25T00:32:48.859 INFO:teuthology.orchestra.run.smithi032.stdout:mds.d smithi032 running (2m) 21s ago 2m 16.5M - 19.0.0-2445-gd2c45d97 f68e37b807ef 0003cea6ab58 2024-04-25T00:32:48.859 INFO:teuthology.orchestra.run.smithi032.stdout:mds.e smithi121 running (2m) 60s ago 2m 18.4M - 19.0.0-2445-gd2c45d97 f68e37b807ef 489b0aa7cac6 2024-04-25T00:32:48.859 INFO:teuthology.orchestra.run.smithi032.stdout:mds.f smithi130 running (2m) 60s ago 2m 18.1M - 19.0.0-2445-gd2c45d97 f68e37b807ef 39daa494086d 2024-04-25T00:32:48.859 INFO:teuthology.orchestra.run.smithi032.stdout:mds.g smithi032 running (2m) 21s ago 2m 13.5M - 19.0.0-2445-gd2c45d97 f68e37b807ef 6f2ee1447833 2024-04-25T00:32:48.859 INFO:teuthology.orchestra.run.smithi032.stdout:mds.h smithi121 running (2m) 60s ago 2m 16.9M - 19.0.0-2445-gd2c45d97 f68e37b807ef 273b11cef7b6 2024-04-25T00:32:48.859 INFO:teuthology.orchestra.run.smithi032.stdout:mds.i smithi130 running (2m) 60s ago 2m 19.2M - 19.0.0-2445-gd2c45d97 f68e37b807ef 687d1407a90a 2024-04-25T00:32:48.859 INFO:teuthology.orchestra.run.smithi032.stdout:mds.j smithi032 running (2m) 21s ago 2m 16.5M - 19.0.0-2445-gd2c45d97 f68e37b807ef 96ce71e67c04 2024-04-25T00:32:48.859 INFO:teuthology.orchestra.run.smithi032.stdout:mds.k smithi121 running (2m) 60s ago 2m 16.5M - 19.0.0-2445-gd2c45d97 f68e37b807ef ced9fa557c2d 2024-04-25T00:32:48.859 INFO:teuthology.orchestra.run.smithi032.stdout:mds.l smithi130 running (2m) 60s ago 2m 15.3M - 19.0.0-2445-gd2c45d97 f68e37b807ef 62b6c422c947 2024-04-25T00:32:48.859 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.x smithi032 *:9283,8765 running (9m) 21s ago 9m 515M - 19.0.0-2445-gd2c45d97 f68e37b807ef a5f04bc0c425 2024-04-25T00:32:48.860 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.y smithi121 *:8443,8765 running (7m) 60s ago 7m 454M - 19.0.0-2445-gd2c45d97 f68e37b807ef 4f08dc59dd34 2024-04-25T00:32:48.860 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.z smithi130 *:8443,8765 running (7m) 60s ago 7m 451M - 19.0.0-2445-gd2c45d97 f68e37b807ef 681d0365cccf 2024-04-25T00:32:48.860 INFO:teuthology.orchestra.run.smithi032.stdout:mon.a smithi032 running (9m) 21s ago 10m 49.6M 2048M 19.0.0-2445-gd2c45d97 f68e37b807ef 6ab91bb240b1 2024-04-25T00:32:48.860 INFO:teuthology.orchestra.run.smithi032.stdout:mon.b smithi121 running (8m) 60s ago 8m 40.1M 2048M 19.0.0-2445-gd2c45d97 f68e37b807ef 61e6e5458627 2024-04-25T00:32:48.860 INFO:teuthology.orchestra.run.smithi032.stdout:mon.c smithi130 running (8m) 60s ago 8m 42.5M 2048M 19.0.0-2445-gd2c45d97 f68e37b807ef ea82c64cab54 2024-04-25T00:32:48.860 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0 smithi032 running (7m) 21s ago 7m 46.0M 4273M 19.0.0-2445-gd2c45d97 f68e37b807ef 00e3d91f366a 2024-04-25T00:32:48.860 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1 smithi121 running (7m) 60s ago 7m 45.9M 4272M 19.0.0-2445-gd2c45d97 f68e37b807ef 4f10008a4b42 2024-04-25T00:32:48.860 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2 smithi130 running (6m) 60s ago 6m 50.0M 4272M 19.0.0-2445-gd2c45d97 f68e37b807ef 6306faf4c415 2024-04-25T00:32:48.860 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3 smithi032 running (6m) 21s ago 6m 54.6M 4273M 19.0.0-2445-gd2c45d97 f68e37b807ef 909f4433856e 2024-04-25T00:32:48.860 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 smithi121 running (5m) 60s ago 5m 43.2M 4272M 19.0.0-2445-gd2c45d97 f68e37b807ef 5cace27307ef 2024-04-25T00:32:48.861 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5 smithi130 running (5m) 60s ago 5m 45.1M 4272M 19.0.0-2445-gd2c45d97 f68e37b807ef 034d093c0953 2024-04-25T00:32:48.861 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6 smithi032 running (5m) 21s ago 5m 49.0M 4273M 19.0.0-2445-gd2c45d97 f68e37b807ef 80bb66923df2 2024-04-25T00:32:48.861 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7 smithi121 running (4m) 60s ago 4m 55.2M 4272M 19.0.0-2445-gd2c45d97 f68e37b807ef 1f5a12fc1a3c 2024-04-25T00:32:48.861 INFO:teuthology.orchestra.run.smithi032.stdout:osd.8 smithi130 running (4m) 60s ago 4m 51.7M 4272M 19.0.0-2445-gd2c45d97 f68e37b807ef a4bbcd14f447 2024-04-25T00:32:48.861 INFO:teuthology.orchestra.run.smithi032.stdout:osd.9 smithi032 running (3m) 21s ago 3m 52.9M 4273M 19.0.0-2445-gd2c45d97 f68e37b807ef b43d43ef360b 2024-04-25T00:32:48.861 INFO:teuthology.orchestra.run.smithi032.stdout:osd.10 smithi121 running (3m) 60s ago 3m 46.3M 4272M 19.0.0-2445-gd2c45d97 f68e37b807ef 0aa4097e757d 2024-04-25T00:32:48.861 INFO:teuthology.orchestra.run.smithi032.stdout:osd.11 smithi130 running (2m) 60s ago 2m 48.8M 4272M 19.0.0-2445-gd2c45d97 f68e37b807ef 1ba7200dfdb2 2024-04-25T00:32:48.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.860+0000 7fe98652f700 1 -- 172.21.15.32:0/3552612624 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe9680a2c20 msgr2=0x7fe9680a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:48.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.860+0000 7fe98652f700 1 --2- 172.21.15.32:0/3552612624 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe9680a2c20 0x7fe9680a50e0 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7fe9801021f0 tx=0x7fe970008040 comp rx=0 tx=0).stop 2024-04-25T00:32:48.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.860+0000 7fe98652f700 1 -- 172.21.15.32:0/3552612624 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe980102390 msgr2=0x7fe98019abd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:48.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.860+0000 7fe98652f700 1 --2- 172.21.15.32:0/3552612624 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe980102390 0x7fe98019abd0 secure :-1 s=READY pgs=391 cs=0 l=1 rev1=1 crypto rx=0x7fe97400c9e0 tx=0x7fe97400ccf0 comp rx=0 tx=0).stop 2024-04-25T00:32:48.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.860+0000 7fe98652f700 1 -- 172.21.15.32:0/3552612624 shutdown_connections 2024-04-25T00:32:48.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.860+0000 7fe98652f700 1 --2- 172.21.15.32:0/3552612624 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe9680a2c20 0x7fe9680a50e0 unknown :-1 s=CLOSED pgs=240 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:48.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.860+0000 7fe98652f700 1 --2- 172.21.15.32:0/3552612624 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe980102d30 0x7fe9801a1c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:48.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.860+0000 7fe98652f700 1 --2- 172.21.15.32:0/3552612624 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe980101190 0x7fe98019a690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:48.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.860+0000 7fe98652f700 1 --2- 172.21.15.32:0/3552612624 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe980102390 0x7fe98019abd0 unknown :-1 s=CLOSED pgs=391 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:48.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.860+0000 7fe98652f700 1 -- 172.21.15.32:0/3552612624 >> 172.21.15.32:0/3552612624 conn(0x7fe9800fc730 msgr2=0x7fe980069f10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:48.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.860+0000 7fe98652f700 1 -- 172.21.15.32:0/3552612624 shutdown_connections 2024-04-25T00:32:48.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:48.860+0000 7fe98652f700 1 -- 172.21.15.32:0/3552612624 wait complete. 2024-04-25T00:32:48.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:48 smithi130 ceph-mon[32525]: pgmap v355: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 1.9 KiB/s rd, 127 B/s wr, 4 op/s 2024-04-25T00:32:49.675 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- bash -c 'ceph orch ls' 2024-04-25T00:32:49.931 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:32:50.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:50 smithi032 ceph-mon[29357]: from='client.15297 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:32:50.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:50 smithi032 ceph-mon[29357]: pgmap v356: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 3.6 KiB/s rd, 639 B/s wr, 8 op/s; 2 B/s, 0 objects/s recovering 2024-04-25T00:32:50.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:50 smithi121 ceph-mon[32231]: from='client.15297 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:32:50.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:50 smithi121 ceph-mon[32231]: pgmap v356: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 3.6 KiB/s rd, 639 B/s wr, 8 op/s; 2 B/s, 0 objects/s recovering 2024-04-25T00:32:50.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:50 smithi130 ceph-mon[32525]: from='client.15297 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:32:50.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:50 smithi130 ceph-mon[32525]: pgmap v356: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 3.6 KiB/s rd, 639 B/s wr, 8 op/s; 2 B/s, 0 objects/s recovering 2024-04-25T00:32:51.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.786+0000 7f8913473700 1 -- 172.21.15.32:0/2543542846 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f890c102390 msgr2=0x7f890c1027f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:51.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.786+0000 7f8913473700 1 --2- 172.21.15.32:0/2543542846 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f890c102390 0x7f890c1027f0 secure :-1 s=READY pgs=392 cs=0 l=1 rev1=1 crypto rx=0x7f89000099c0 tx=0x7f8900009cd0 comp rx=0 tx=0).stop 2024-04-25T00:32:51.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.786+0000 7f8913473700 1 -- 172.21.15.32:0/2543542846 shutdown_connections 2024-04-25T00:32:51.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.786+0000 7f8913473700 1 --2- 172.21.15.32:0/2543542846 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f890c102d30 0x7f890c1095e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:51.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.786+0000 7f8913473700 1 --2- 172.21.15.32:0/2543542846 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f890c101190 0x7f890c1015b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:51.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.786+0000 7f8913473700 1 --2- 172.21.15.32:0/2543542846 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f890c102390 0x7f890c1027f0 unknown :-1 s=CLOSED pgs=392 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:51.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.786+0000 7f8913473700 1 -- 172.21.15.32:0/2543542846 >> 172.21.15.32:0/2543542846 conn(0x7f890c0fc730 msgr2=0x7f890c0feb70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:51.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.786+0000 7f8913473700 1 -- 172.21.15.32:0/2543542846 shutdown_connections 2024-04-25T00:32:51.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.786+0000 7f8913473700 1 -- 172.21.15.32:0/2543542846 wait complete. 2024-04-25T00:32:51.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.787+0000 7f8913473700 1 Processor -- start 2024-04-25T00:32:51.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.787+0000 7f8913473700 1 -- start start 2024-04-25T00:32:51.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.787+0000 7f8913473700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f890c101190 0x7f890c105120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:51.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.787+0000 7f8913473700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f890c102390 0x7f890c103770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:51.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.787+0000 7f8913473700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f890c102d30 0x7f890c103cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:51.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.787+0000 7f8913473700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f890c10cd60 con 0x7f890c101190 2024-04-25T00:32:51.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.787+0000 7f8913473700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f890c10cbe0 con 0x7f890c102390 2024-04-25T00:32:51.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.787+0000 7f8913473700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f890c10cee0 con 0x7f890c102d30 2024-04-25T00:32:51.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.787+0000 7f891120f700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f890c101190 0x7f890c105120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:51.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.787+0000 7f891120f700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f890c101190 0x7f890c105120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:60414/0 (socket says 172.21.15.32:60414) 2024-04-25T00:32:51.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.787+0000 7f891120f700 1 -- 172.21.15.32:0/766744524 learned_addr learned my addr 172.21.15.32:0/766744524 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:51.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.787+0000 7f8911a10700 1 --2- 172.21.15.32:0/766744524 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f890c102d30 0x7f890c103cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:51.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.787+0000 7f8910a0e700 1 --2- 172.21.15.32:0/766744524 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f890c102390 0x7f890c103770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:51.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.787+0000 7f891120f700 1 -- 172.21.15.32:0/766744524 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f890c102d30 msgr2=0x7f890c103cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:51.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.787+0000 7f891120f700 1 --2- 172.21.15.32:0/766744524 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f890c102d30 0x7f890c103cb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:51.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.787+0000 7f891120f700 1 -- 172.21.15.32:0/766744524 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f890c102390 msgr2=0x7f890c103770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:51.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.787+0000 7f891120f700 1 --2- 172.21.15.32:0/766744524 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f890c102390 0x7f890c103770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:51.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.787+0000 7f891120f700 1 -- 172.21.15.32:0/766744524 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f890c104560 con 0x7f890c101190 2024-04-25T00:32:51.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.787+0000 7f8910a0e700 1 --2- 172.21.15.32:0/766744524 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f890c102390 0x7f890c103770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:32:51.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.788+0000 7f891120f700 1 --2- 172.21.15.32:0/766744524 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f890c101190 0x7f890c105120 secure :-1 s=READY pgs=393 cs=0 l=1 rev1=1 crypto rx=0x7f890400d980 tx=0x7f890400dc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:51.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.788+0000 7f890a7fc700 1 -- 172.21.15.32:0/766744524 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8904017030 con 0x7f890c101190 2024-04-25T00:32:51.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.788+0000 7f890a7fc700 1 -- 172.21.15.32:0/766744524 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8904004500 con 0x7f890c101190 2024-04-25T00:32:51.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.788+0000 7f8913473700 1 -- 172.21.15.32:0/766744524 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f890c1a2670 con 0x7f890c101190 2024-04-25T00:32:51.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.788+0000 7f890a7fc700 1 -- 172.21.15.32:0/766744524 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8904003aa0 con 0x7f890c101190 2024-04-25T00:32:51.792 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.788+0000 7f8913473700 1 -- 172.21.15.32:0/766744524 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f890c1a2b80 con 0x7f890c101190 2024-04-25T00:32:51.792 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.789+0000 7f8913473700 1 -- 172.21.15.32:0/766744524 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f890c068f20 con 0x7f890c101190 2024-04-25T00:32:51.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.790+0000 7f890a7fc700 1 -- 172.21.15.32:0/766744524 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 18) ==== 137833+0+0 (secure 0 0 0) 0x7f890401a070 con 0x7f890c101190 2024-04-25T00:32:51.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.790+0000 7f890a7fc700 1 --2- 172.21.15.32:0/766744524 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f88f40a2c20 0x7f88f40a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:51.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.791+0000 7f8910a0e700 1 --2- 172.21.15.32:0/766744524 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f88f40a2c20 0x7f88f40a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:51.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.791+0000 7f890a7fc700 1 -- 172.21.15.32:0/766744524 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7f89040ced60 con 0x7f890c101190 2024-04-25T00:32:51.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.791+0000 7f8910a0e700 1 --2- 172.21.15.32:0/766744524 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f88f40a2c20 0x7f88f40a50e0 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7f89000039c0 tx=0x7f8900005fd0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:51.795 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:51.795+0000 7f890a7fc700 1 -- 172.21.15.32:0/766744524 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f890409b8f0 con 0x7f890c101190 2024-04-25T00:32:52.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:52.012+0000 7f8913473700 1 -- 172.21.15.32:0/766744524 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch ls", "target": ["mon-mgr", ""]}) -- 0x7f890c1a2ea0 con 0x7f88f40a2c20 2024-04-25T00:32:52.018 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:52.018+0000 7f890a7fc700 1 -- 172.21.15.32:0/766744524 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+980 (secure 0 0 0) 0x7f890c1a2ea0 con 0x7f88f40a2c20 2024-04-25T00:32:52.019 INFO:teuthology.orchestra.run.smithi032.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-25T00:32:52.019 INFO:teuthology.orchestra.run.smithi032.stdout:mds.all 12/12 63s ago 2m smithi032=a;smithi032=d;smithi032=g;smithi032=j;smithi121=b;smithi121=e;smithi121=h;smithi121=k;smithi130=c;smithi130=f;smithi130=i;smithi130=l;count:12 2024-04-25T00:32:52.019 INFO:teuthology.orchestra.run.smithi032.stdout:mgr 3/3 63s ago 8m smithi032=x;smithi121=y;smithi130=z;count:3 2024-04-25T00:32:52.019 INFO:teuthology.orchestra.run.smithi032.stdout:mon 3/3 63s ago 8m smithi032:172.21.15.32=a;smithi121:172.21.15.121=b;smithi130:172.21.15.130=c;count:3 2024-04-25T00:32:52.019 INFO:teuthology.orchestra.run.smithi032.stdout:osd 12 63s ago - 2024-04-25T00:32:52.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:52.020+0000 7f8913473700 1 -- 172.21.15.32:0/766744524 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f88f40a2c20 msgr2=0x7f88f40a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:52.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:52.020+0000 7f8913473700 1 --2- 172.21.15.32:0/766744524 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f88f40a2c20 0x7f88f40a50e0 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7f89000039c0 tx=0x7f8900005fd0 comp rx=0 tx=0).stop 2024-04-25T00:32:52.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:52.020+0000 7f8913473700 1 -- 172.21.15.32:0/766744524 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f890c101190 msgr2=0x7f890c105120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:52.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:52.020+0000 7f8913473700 1 --2- 172.21.15.32:0/766744524 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f890c101190 0x7f890c105120 secure :-1 s=READY pgs=393 cs=0 l=1 rev1=1 crypto rx=0x7f890400d980 tx=0x7f890400dc90 comp rx=0 tx=0).stop 2024-04-25T00:32:52.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:52.020+0000 7f8913473700 1 -- 172.21.15.32:0/766744524 shutdown_connections 2024-04-25T00:32:52.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:52.020+0000 7f8913473700 1 --2- 172.21.15.32:0/766744524 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f88f40a2c20 0x7f88f40a50e0 unknown :-1 s=CLOSED pgs=241 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:52.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:52.020+0000 7f8913473700 1 --2- 172.21.15.32:0/766744524 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f890c102d30 0x7f890c103cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:52.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:52.020+0000 7f8913473700 1 --2- 172.21.15.32:0/766744524 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f890c101190 0x7f890c105120 unknown :-1 s=CLOSED pgs=393 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:52.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:52.020+0000 7f8913473700 1 --2- 172.21.15.32:0/766744524 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f890c102390 0x7f890c103770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:52.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:52.020+0000 7f8913473700 1 -- 172.21.15.32:0/766744524 >> 172.21.15.32:0/766744524 conn(0x7f890c0fc730 msgr2=0x7f890c0fe260 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:52.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:52.020+0000 7f8913473700 1 -- 172.21.15.32:0/766744524 shutdown_connections 2024-04-25T00:32:52.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:52.021+0000 7f8913473700 1 -- 172.21.15.32:0/766744524 wait complete. 2024-04-25T00:32:52.668 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:52 smithi032 ceph-mon[29357]: pgmap v357: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 2.4 KiB/s rd, 639 B/s wr, 5 op/s; 16 B/s, 1 objects/s recovering 2024-04-25T00:32:52.697 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- bash -c 'ceph orch host ls' 2024-04-25T00:32:52.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:52 smithi121 ceph-mon[32231]: pgmap v357: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 2.4 KiB/s rd, 639 B/s wr, 5 op/s; 16 B/s, 1 objects/s recovering 2024-04-25T00:32:52.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:52 smithi130 ceph-mon[32525]: pgmap v357: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 2.4 KiB/s rd, 639 B/s wr, 5 op/s; 16 B/s, 1 objects/s recovering 2024-04-25T00:32:52.958 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:32:53.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:53 smithi032 ceph-mon[29357]: from='client.15303 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:32:53.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:53 smithi121 ceph-mon[32231]: from='client.15303 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:32:53.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:53 smithi130 ceph-mon[32525]: from='client.15303 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:32:54.739 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:54 smithi121 ceph-mon[32231]: pgmap v358: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 2.8 KiB/s rd, 485 B/s wr, 7 op/s; 15 B/s, 1 objects/s recovering 2024-04-25T00:32:54.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:54 smithi032 ceph-mon[29357]: pgmap v358: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 2.8 KiB/s rd, 485 B/s wr, 7 op/s; 15 B/s, 1 objects/s recovering 2024-04-25T00:32:54.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:54 smithi130 ceph-mon[32525]: pgmap v358: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 2.8 KiB/s rd, 485 B/s wr, 7 op/s; 15 B/s, 1 objects/s recovering 2024-04-25T00:32:54.815 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.815+0000 7eff982bf700 1 -- 172.21.15.32:0/3684569969 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff90103e30 msgr2=0x7eff9010a6e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:54.816 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.815+0000 7eff982bf700 1 --2- 172.21.15.32:0/3684569969 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff90103e30 0x7eff9010a6e0 secure :-1 s=READY pgs=394 cs=0 l=1 rev1=1 crypto rx=0x7eff800099c0 tx=0x7eff80009cd0 comp rx=0 tx=0).stop 2024-04-25T00:32:54.816 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.815+0000 7eff982bf700 1 -- 172.21.15.32:0/3684569969 shutdown_connections 2024-04-25T00:32:54.816 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.815+0000 7eff982bf700 1 --2- 172.21.15.32:0/3684569969 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff90103e30 0x7eff9010a6e0 unknown :-1 s=CLOSED pgs=394 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:54.816 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.815+0000 7eff982bf700 1 --2- 172.21.15.32:0/3684569969 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7eff90102290 0x7eff901026b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:54.816 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.815+0000 7eff982bf700 1 --2- 172.21.15.32:0/3684569969 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7eff90103490 0x7eff901038f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:54.816 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.815+0000 7eff982bf700 1 -- 172.21.15.32:0/3684569969 >> 172.21.15.32:0/3684569969 conn(0x7eff900fd830 msgr2=0x7eff900ffc70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:54.816 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.815+0000 7eff982bf700 1 -- 172.21.15.32:0/3684569969 shutdown_connections 2024-04-25T00:32:54.816 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.815+0000 7eff982bf700 1 -- 172.21.15.32:0/3684569969 wait complete. 2024-04-25T00:32:54.818 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.816+0000 7eff982bf700 1 Processor -- start 2024-04-25T00:32:54.819 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.816+0000 7eff982bf700 1 -- start start 2024-04-25T00:32:54.819 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.816+0000 7eff982bf700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff90102290 0x7eff901983d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:54.819 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.816+0000 7eff982bf700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7eff90103490 0x7eff90198910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:54.819 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.816+0000 7eff982bf700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7eff90103e30 0x7eff9019f990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:54.819 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.816+0000 7eff982bf700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7eff9010de50 con 0x7eff90102290 2024-04-25T00:32:54.819 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.816+0000 7eff982bf700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7eff9010dcd0 con 0x7eff90103e30 2024-04-25T00:32:54.819 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.816+0000 7eff982bf700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7eff9010dfd0 con 0x7eff90103490 2024-04-25T00:32:54.819 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.816+0000 7eff9605b700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff90102290 0x7eff901983d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:54.819 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.816+0000 7eff9605b700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff90102290 0x7eff901983d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:60440/0 (socket says 172.21.15.32:60440) 2024-04-25T00:32:54.819 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.816+0000 7eff9605b700 1 -- 172.21.15.32:0/43669884 learned_addr learned my addr 172.21.15.32:0/43669884 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:54.820 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.816+0000 7eff9685c700 1 --2- 172.21.15.32:0/43669884 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7eff90103e30 0x7eff9019f990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:54.820 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.816+0000 7eff9585a700 1 --2- 172.21.15.32:0/43669884 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7eff90103490 0x7eff90198910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:54.820 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.816+0000 7eff9605b700 1 -- 172.21.15.32:0/43669884 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7eff90103490 msgr2=0x7eff90198910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:54.820 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.816+0000 7eff9605b700 1 --2- 172.21.15.32:0/43669884 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7eff90103490 0x7eff90198910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:54.820 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.816+0000 7eff9605b700 1 -- 172.21.15.32:0/43669884 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7eff90103e30 msgr2=0x7eff9019f990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:54.820 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.816+0000 7eff9605b700 1 --2- 172.21.15.32:0/43669884 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7eff90103e30 0x7eff9019f990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:54.820 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.816+0000 7eff9605b700 1 -- 172.21.15.32:0/43669884 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7eff901a0030 con 0x7eff90102290 2024-04-25T00:32:54.820 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.816+0000 7eff9585a700 1 --2- 172.21.15.32:0/43669884 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7eff90103490 0x7eff90198910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:32:54.820 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.817+0000 7eff9605b700 1 --2- 172.21.15.32:0/43669884 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff90102290 0x7eff901983d0 secure :-1 s=READY pgs=395 cs=0 l=1 rev1=1 crypto rx=0x7eff88007f40 tx=0x7eff8800d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:54.820 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.817+0000 7eff8f7fe700 1 -- 172.21.15.32:0/43669884 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7eff8800dd50 con 0x7eff90102290 2024-04-25T00:32:54.820 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.817+0000 7eff8f7fe700 1 -- 172.21.15.32:0/43669884 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7eff88004020 con 0x7eff90102290 2024-04-25T00:32:54.820 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.817+0000 7eff982bf700 1 -- 172.21.15.32:0/43669884 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7eff901a0320 con 0x7eff90102290 2024-04-25T00:32:54.820 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.817+0000 7eff8f7fe700 1 -- 172.21.15.32:0/43669884 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7eff88013630 con 0x7eff90102290 2024-04-25T00:32:54.820 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.817+0000 7eff982bf700 1 -- 172.21.15.32:0/43669884 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7eff901a0860 con 0x7eff90102290 2024-04-25T00:32:54.820 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.818+0000 7eff982bf700 1 -- 172.21.15.32:0/43669884 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7eff90068f20 con 0x7eff90102290 2024-04-25T00:32:54.821 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.819+0000 7eff8f7fe700 1 -- 172.21.15.32:0/43669884 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 18) ==== 137833+0+0 (secure 0 0 0) 0x7eff8801a020 con 0x7eff90102290 2024-04-25T00:32:54.821 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.819+0000 7eff8f7fe700 1 --2- 172.21.15.32:0/43669884 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7eff780a2c20 0x7eff780a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:54.821 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.820+0000 7eff9585a700 1 --2- 172.21.15.32:0/43669884 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7eff780a2c20 0x7eff780a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:54.821 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.820+0000 7eff8f7fe700 1 -- 172.21.15.32:0/43669884 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7eff880cec80 con 0x7eff90102290 2024-04-25T00:32:54.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.820+0000 7eff9585a700 1 --2- 172.21.15.32:0/43669884 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7eff780a2c20 0x7eff780a50e0 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7eff84005d90 tx=0x7eff84005d00 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:54.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:54.824+0000 7eff8f7fe700 1 -- 172.21.15.32:0/43669884 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7eff88010070 con 0x7eff90102290 2024-04-25T00:32:55.036 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:55.035+0000 7eff982bf700 1 -- 172.21.15.32:0/43669884 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch host ls", "target": ["mon-mgr", ""]}) -- 0x7eff90062670 con 0x7eff780a2c20 2024-04-25T00:32:55.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:55.037+0000 7eff8f7fe700 1 -- 172.21.15.32:0/43669884 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+191 (secure 0 0 0) 0x7eff90062670 con 0x7eff780a2c20 2024-04-25T00:32:55.037 INFO:teuthology.orchestra.run.smithi032.stdout:HOST ADDR LABELS STATUS 2024-04-25T00:32:55.037 INFO:teuthology.orchestra.run.smithi032.stdout:smithi032 172.21.15.32 2024-04-25T00:32:55.037 INFO:teuthology.orchestra.run.smithi032.stdout:smithi121 172.21.15.121 2024-04-25T00:32:55.037 INFO:teuthology.orchestra.run.smithi032.stdout:smithi130 172.21.15.130 2024-04-25T00:32:55.037 INFO:teuthology.orchestra.run.smithi032.stdout:3 hosts in cluster 2024-04-25T00:32:55.039 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:55.039+0000 7eff982bf700 1 -- 172.21.15.32:0/43669884 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7eff780a2c20 msgr2=0x7eff780a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:55.039 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:55.039+0000 7eff982bf700 1 --2- 172.21.15.32:0/43669884 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7eff780a2c20 0x7eff780a50e0 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7eff84005d90 tx=0x7eff84005d00 comp rx=0 tx=0).stop 2024-04-25T00:32:55.039 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:55.039+0000 7eff982bf700 1 -- 172.21.15.32:0/43669884 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff90102290 msgr2=0x7eff901983d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:55.039 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:55.039+0000 7eff982bf700 1 --2- 172.21.15.32:0/43669884 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff90102290 0x7eff901983d0 secure :-1 s=READY pgs=395 cs=0 l=1 rev1=1 crypto rx=0x7eff88007f40 tx=0x7eff8800d3b0 comp rx=0 tx=0).stop 2024-04-25T00:32:55.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:55.039+0000 7eff982bf700 1 -- 172.21.15.32:0/43669884 shutdown_connections 2024-04-25T00:32:55.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:55.039+0000 7eff982bf700 1 --2- 172.21.15.32:0/43669884 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7eff780a2c20 0x7eff780a50e0 unknown :-1 s=CLOSED pgs=242 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:55.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:55.039+0000 7eff982bf700 1 --2- 172.21.15.32:0/43669884 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7eff90103e30 0x7eff9019f990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:55.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:55.039+0000 7eff982bf700 1 --2- 172.21.15.32:0/43669884 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff90102290 0x7eff901983d0 unknown :-1 s=CLOSED pgs=395 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:55.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:55.039+0000 7eff982bf700 1 --2- 172.21.15.32:0/43669884 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7eff90103490 0x7eff90198910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:55.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:55.039+0000 7eff982bf700 1 -- 172.21.15.32:0/43669884 >> 172.21.15.32:0/43669884 conn(0x7eff900fd830 msgr2=0x7eff90106730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:55.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:55.039+0000 7eff982bf700 1 -- 172.21.15.32:0/43669884 shutdown_connections 2024-04-25T00:32:55.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:55.039+0000 7eff982bf700 1 -- 172.21.15.32:0/43669884 wait complete. 2024-04-25T00:32:55.873 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- bash -c 'ceph orch device ls' 2024-04-25T00:32:56.130 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:32:56.541 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:56 smithi032 ceph-mon[29357]: pgmap v359: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 409 B/s wr, 6 op/s; 13 B/s, 1 objects/s recovering 2024-04-25T00:32:56.541 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:56 smithi032 ceph-mon[29357]: from='client.15309 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:32:56.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:56 smithi121 ceph-mon[32231]: pgmap v359: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 409 B/s wr, 6 op/s; 13 B/s, 1 objects/s recovering 2024-04-25T00:32:56.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:56 smithi121 ceph-mon[32231]: from='client.15309 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:32:56.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:56 smithi130 ceph-mon[32525]: pgmap v359: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 409 B/s wr, 6 op/s; 13 B/s, 1 objects/s recovering 2024-04-25T00:32:56.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:56 smithi130 ceph-mon[32525]: from='client.15309 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:32:57.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.908+0000 7f86d7ca5700 1 -- 172.21.15.32:0/3080874169 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f86d01011b0 msgr2=0x7f86d01015d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:57.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.908+0000 7f86d7ca5700 1 --2- 172.21.15.32:0/3080874169 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f86d01011b0 0x7f86d01015d0 secure :-1 s=READY pgs=396 cs=0 l=1 rev1=1 crypto rx=0x7f86c00099c0 tx=0x7f86c0009cd0 comp rx=0 tx=0).stop 2024-04-25T00:32:57.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.909+0000 7f86d7ca5700 1 -- 172.21.15.32:0/3080874169 shutdown_connections 2024-04-25T00:32:57.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.909+0000 7f86d7ca5700 1 --2- 172.21.15.32:0/3080874169 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f86d0102d50 0x7f86d0109600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:57.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.909+0000 7f86d7ca5700 1 --2- 172.21.15.32:0/3080874169 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f86d01011b0 0x7f86d01015d0 unknown :-1 s=CLOSED pgs=396 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:57.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.909+0000 7f86d7ca5700 1 --2- 172.21.15.32:0/3080874169 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f86d01023b0 0x7f86d0102810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:57.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.909+0000 7f86d7ca5700 1 -- 172.21.15.32:0/3080874169 >> 172.21.15.32:0/3080874169 conn(0x7f86d00fc750 msgr2=0x7f86d00feb90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:57.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.909+0000 7f86d7ca5700 1 -- 172.21.15.32:0/3080874169 shutdown_connections 2024-04-25T00:32:57.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.909+0000 7f86d7ca5700 1 -- 172.21.15.32:0/3080874169 wait complete. 2024-04-25T00:32:57.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.909+0000 7f86d7ca5700 1 Processor -- start 2024-04-25T00:32:57.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.909+0000 7f86d7ca5700 1 -- start start 2024-04-25T00:32:57.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.910+0000 7f86d7ca5700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f86d01011b0 0x7f86d0105210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:57.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.910+0000 7f86d7ca5700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f86d01023b0 0x7f86d0103860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:57.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.910+0000 7f86d7ca5700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f86d0102d50 0x7f86d0103da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:57.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.910+0000 7f86d7ca5700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f86d010cd90 con 0x7f86d01023b0 2024-04-25T00:32:57.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.910+0000 7f86d7ca5700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f86d010cc10 con 0x7f86d0102d50 2024-04-25T00:32:57.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.910+0000 7f86d7ca5700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f86d010cf10 con 0x7f86d01011b0 2024-04-25T00:32:57.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.910+0000 7f86d5240700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f86d01023b0 0x7f86d0103860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:57.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.910+0000 7f86d5240700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f86d01023b0 0x7f86d0103860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:60458/0 (socket says 172.21.15.32:60458) 2024-04-25T00:32:57.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.910+0000 7f86d5240700 1 -- 172.21.15.32:0/4235273579 learned_addr learned my addr 172.21.15.32:0/4235273579 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:32:57.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.910+0000 7f86d5a41700 1 --2- 172.21.15.32:0/4235273579 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f86d01011b0 0x7f86d0105210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:57.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.910+0000 7f86d6242700 1 --2- 172.21.15.32:0/4235273579 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f86d0102d50 0x7f86d0103da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:57.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.910+0000 7f86d5240700 1 -- 172.21.15.32:0/4235273579 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f86d01011b0 msgr2=0x7f86d0105210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:57.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.910+0000 7f86d5240700 1 --2- 172.21.15.32:0/4235273579 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f86d01011b0 0x7f86d0105210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:57.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.910+0000 7f86d5240700 1 -- 172.21.15.32:0/4235273579 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f86d0102d50 msgr2=0x7f86d0103da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:57.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.910+0000 7f86d5240700 1 --2- 172.21.15.32:0/4235273579 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f86d0102d50 0x7f86d0103da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:57.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.910+0000 7f86d5240700 1 -- 172.21.15.32:0/4235273579 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f86d0104650 con 0x7f86d01023b0 2024-04-25T00:32:57.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.910+0000 7f86d5240700 1 --2- 172.21.15.32:0/4235273579 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f86d01023b0 0x7f86d0103860 secure :-1 s=READY pgs=397 cs=0 l=1 rev1=1 crypto rx=0x7f86cc00d980 tx=0x7f86cc00dc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:57.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.910+0000 7f86c6ffd700 1 -- 172.21.15.32:0/4235273579 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f86cc016070 con 0x7f86d01023b0 2024-04-25T00:32:57.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.910+0000 7f86d7ca5700 1 -- 172.21.15.32:0/4235273579 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f86d01a2650 con 0x7f86d01023b0 2024-04-25T00:32:57.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.910+0000 7f86c6ffd700 1 -- 172.21.15.32:0/4235273579 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f86cc004d10 con 0x7f86d01023b0 2024-04-25T00:32:57.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.911+0000 7f86d7ca5700 1 -- 172.21.15.32:0/4235273579 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f86d01a2b90 con 0x7f86d01023b0 2024-04-25T00:32:57.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.911+0000 7f86c6ffd700 1 -- 172.21.15.32:0/4235273579 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f86cc005610 con 0x7f86d01023b0 2024-04-25T00:32:57.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.912+0000 7f86d7ca5700 1 -- 172.21.15.32:0/4235273579 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f86d0068f20 con 0x7f86d01023b0 2024-04-25T00:32:57.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.913+0000 7f86c6ffd700 1 -- 172.21.15.32:0/4235273579 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 18) ==== 137833+0+0 (secure 0 0 0) 0x7f86cc004750 con 0x7f86d01023b0 2024-04-25T00:32:57.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.913+0000 7f86c6ffd700 1 --2- 172.21.15.32:0/4235273579 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f86bc0a2c20 0x7f86bc0a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:32:57.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.913+0000 7f86c6ffd700 1 -- 172.21.15.32:0/4235273579 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7f86cc0cf730 con 0x7f86d01023b0 2024-04-25T00:32:57.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.913+0000 7f86d5a41700 1 --2- 172.21.15.32:0/4235273579 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f86bc0a2c20 0x7f86bc0a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:32:57.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.914+0000 7f86d5a41700 1 --2- 172.21.15.32:0/4235273579 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f86bc0a2c20 0x7f86bc0a50e0 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7f86c000b580 tx=0x7f86c0005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:32:57.917 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:57.917+0000 7f86c6ffd700 1 -- 172.21.15.32:0/4235273579 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f86cc01f020 con 0x7f86d01023b0 2024-04-25T00:32:58.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:58.132+0000 7f86d7ca5700 1 -- 172.21.15.32:0/4235273579 --> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] -- mgr_command(tid 0: {"prefix": "orch device ls", "target": ["mon-mgr", ""]}) -- 0x7f86d006b070 con 0x7f86bc0a2c20 2024-04-25T00:32:58.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:58.134+0000 7f86c6ffd700 1 -- 172.21.15.32:0/4235273579 <== mgr.14148 v2:172.21.15.32:6800/585810931 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+532 (secure 0 0 0) 0x7f86d006b070 con 0x7f86bc0a2c20 2024-04-25T00:32:58.134 INFO:teuthology.orchestra.run.smithi032.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REFRESHED REJECT REASONS 2024-04-25T00:32:58.134 INFO:teuthology.orchestra.run.smithi032.stdout:smithi032 /dev/nvme0n1 ssd INTEL_SSDPED1D480GA_PHMB747402DN480DGN 447G No 3m ago Has a FileSystem, LVM detected 2024-04-25T00:32:58.134 INFO:teuthology.orchestra.run.smithi032.stdout:smithi121 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400ZF400BGN 372G No 3m ago Has a FileSystem, LVM detected 2024-04-25T00:32:58.134 INFO:teuthology.orchestra.run.smithi032.stdout:smithi130 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620500Q1400BGN 372G No 2m ago Has a FileSystem, LVM detected 2024-04-25T00:32:58.136 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:58.136+0000 7f86d7ca5700 1 -- 172.21.15.32:0/4235273579 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f86bc0a2c20 msgr2=0x7f86bc0a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:58.136 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:58.136+0000 7f86d7ca5700 1 --2- 172.21.15.32:0/4235273579 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f86bc0a2c20 0x7f86bc0a50e0 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7f86c000b580 tx=0x7f86c0005fb0 comp rx=0 tx=0).stop 2024-04-25T00:32:58.136 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:58.136+0000 7f86d7ca5700 1 -- 172.21.15.32:0/4235273579 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f86d01023b0 msgr2=0x7f86d0103860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:32:58.136 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:58.136+0000 7f86d7ca5700 1 --2- 172.21.15.32:0/4235273579 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f86d01023b0 0x7f86d0103860 secure :-1 s=READY pgs=397 cs=0 l=1 rev1=1 crypto rx=0x7f86cc00d980 tx=0x7f86cc00dc90 comp rx=0 tx=0).stop 2024-04-25T00:32:58.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:58.136+0000 7f86d7ca5700 1 -- 172.21.15.32:0/4235273579 shutdown_connections 2024-04-25T00:32:58.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:58.136+0000 7f86d7ca5700 1 --2- 172.21.15.32:0/4235273579 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f86bc0a2c20 0x7f86bc0a50e0 unknown :-1 s=CLOSED pgs=243 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:58.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:58.136+0000 7f86d7ca5700 1 --2- 172.21.15.32:0/4235273579 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f86d0102d50 0x7f86d0103da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:58.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:58.136+0000 7f86d7ca5700 1 --2- 172.21.15.32:0/4235273579 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f86d01011b0 0x7f86d0105210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:58.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:58.136+0000 7f86d7ca5700 1 --2- 172.21.15.32:0/4235273579 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f86d01023b0 0x7f86d0103860 unknown :-1 s=CLOSED pgs=397 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:32:58.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:58.136+0000 7f86d7ca5700 1 -- 172.21.15.32:0/4235273579 >> 172.21.15.32:0/4235273579 conn(0x7f86d00fc750 msgr2=0x7f86d006a960 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:32:58.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:58.136+0000 7f86d7ca5700 1 -- 172.21.15.32:0/4235273579 shutdown_connections 2024-04-25T00:32:58.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:32:58.136+0000 7f86d7ca5700 1 -- 172.21.15.32:0/4235273579 wait complete. 2024-04-25T00:32:58.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:58 smithi032 ceph-mon[29357]: pgmap v360: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 358 B/s wr, 5 op/s; 11 B/s, 1 objects/s recovering 2024-04-25T00:32:58.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:58 smithi121 ceph-mon[32231]: pgmap v360: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 358 B/s wr, 5 op/s; 11 B/s, 1 objects/s recovering 2024-04-25T00:32:58.841 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-25T00:32:58.856 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi032.front.sepia.ceph.com 2024-04-25T00:32:58.856 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- bash -c 'ceph fs dump' 2024-04-25T00:32:58.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:58 smithi130 ceph-mon[32525]: pgmap v360: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 358 B/s wr, 5 op/s; 11 B/s, 1 objects/s recovering 2024-04-25T00:32:59.114 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:32:59.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:32:59 smithi032 ceph-mon[29357]: from='client.15315 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:32:59.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:32:59 smithi121 ceph-mon[32231]: from='client.15315 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:32:59.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:32:59 smithi130 ceph-mon[32525]: from='client.15315 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:33:00.654 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:00 smithi032 ceph-mon[29357]: pgmap v361: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 341 B/s wr, 7 op/s; 11 B/s, 1 objects/s recovering 2024-04-25T00:33:00.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:00 smithi121 ceph-mon[32231]: pgmap v361: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 341 B/s wr, 7 op/s; 11 B/s, 1 objects/s recovering 2024-04-25T00:33:00.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:00 smithi130 ceph-mon[32525]: pgmap v361: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 341 B/s wr, 7 op/s; 11 B/s, 1 objects/s recovering 2024-04-25T00:33:00.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.910+0000 7f22ff5ec700 1 -- 172.21.15.32:0/2976511001 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22f81034b0 msgr2=0x7f22f8103910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:00.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.910+0000 7f22ff5ec700 1 --2- 172.21.15.32:0/2976511001 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22f81034b0 0x7f22f8103910 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 crypto rx=0x7f22f40099c0 tx=0x7f22f4009cd0 comp rx=0 tx=0).stop 2024-04-25T00:33:00.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.910+0000 7f22ff5ec700 1 -- 172.21.15.32:0/2976511001 shutdown_connections 2024-04-25T00:33:00.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.910+0000 7f22ff5ec700 1 --2- 172.21.15.32:0/2976511001 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f22f8103e50 0x7f22f810a700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:00.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.910+0000 7f22ff5ec700 1 --2- 172.21.15.32:0/2976511001 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22f81022b0 0x7f22f81026d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:00.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.910+0000 7f22ff5ec700 1 --2- 172.21.15.32:0/2976511001 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22f81034b0 0x7f22f8103910 unknown :-1 s=CLOSED pgs=398 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:00.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.910+0000 7f22ff5ec700 1 -- 172.21.15.32:0/2976511001 >> 172.21.15.32:0/2976511001 conn(0x7f22f80fd850 msgr2=0x7f22f80ffc90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:00.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.910+0000 7f22ff5ec700 1 -- 172.21.15.32:0/2976511001 shutdown_connections 2024-04-25T00:33:00.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.910+0000 7f22ff5ec700 1 -- 172.21.15.32:0/2976511001 wait complete. 2024-04-25T00:33:00.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.911+0000 7f22ff5ec700 1 Processor -- start 2024-04-25T00:33:00.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.911+0000 7f22ff5ec700 1 -- start start 2024-04-25T00:33:00.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.911+0000 7f22ff5ec700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f22f81022b0 0x7f22f8198450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:00.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.911+0000 7f22ff5ec700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22f81034b0 0x7f22f8198990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:00.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.911+0000 7f22ff5ec700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22f8103e50 0x7f22f819fa10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:00.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.912+0000 7f22ff5ec700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f22f810dea0 con 0x7f22f8103e50 2024-04-25T00:33:00.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.912+0000 7f22ff5ec700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f22f810dd20 con 0x7f22f81022b0 2024-04-25T00:33:00.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.912+0000 7f22ff5ec700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f22f810e020 con 0x7f22f81034b0 2024-04-25T00:33:00.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.912+0000 7f22fdb89700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22f8103e50 0x7f22f819fa10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:00.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.912+0000 7f22fdb89700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22f8103e50 0x7f22f819fa10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:60486/0 (socket says 172.21.15.32:60486) 2024-04-25T00:33:00.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.912+0000 7f22fdb89700 1 -- 172.21.15.32:0/2725716910 learned_addr learned my addr 172.21.15.32:0/2725716910 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:00.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.912+0000 7f22fcb87700 1 --2- 172.21.15.32:0/2725716910 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22f81034b0 0x7f22f8198990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:00.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.912+0000 7f22fd388700 1 --2- 172.21.15.32:0/2725716910 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f22f81022b0 0x7f22f8198450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:00.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.912+0000 7f22fdb89700 1 -- 172.21.15.32:0/2725716910 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22f81034b0 msgr2=0x7f22f8198990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:00.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.912+0000 7f22fdb89700 1 --2- 172.21.15.32:0/2725716910 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22f81034b0 0x7f22f8198990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:00.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.912+0000 7f22fdb89700 1 -- 172.21.15.32:0/2725716910 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f22f81022b0 msgr2=0x7f22f8198450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:00.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.912+0000 7f22fdb89700 1 --2- 172.21.15.32:0/2725716910 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f22f81022b0 0x7f22f8198450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:00.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.912+0000 7f22fdb89700 1 -- 172.21.15.32:0/2725716910 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f22f81a00b0 con 0x7f22f8103e50 2024-04-25T00:33:00.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.912+0000 7f22fdb89700 1 --2- 172.21.15.32:0/2725716910 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22f8103e50 0x7f22f819fa10 secure :-1 s=READY pgs=399 cs=0 l=1 rev1=1 crypto rx=0x7f22f0007da0 tx=0x7f22f000a340 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:00.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.912+0000 7f22ee7fc700 1 -- 172.21.15.32:0/2725716910 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f22f0016020 con 0x7f22f8103e50 2024-04-25T00:33:00.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.913+0000 7f22ff5ec700 1 -- 172.21.15.32:0/2725716910 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f22f81a03a0 con 0x7f22f8103e50 2024-04-25T00:33:00.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.913+0000 7f22ee7fc700 1 -- 172.21.15.32:0/2725716910 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f22f000ac50 con 0x7f22f8103e50 2024-04-25T00:33:00.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.913+0000 7f22ee7fc700 1 -- 172.21.15.32:0/2725716910 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f22f00115e0 con 0x7f22f8103e50 2024-04-25T00:33:00.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.913+0000 7f22ff5ec700 1 -- 172.21.15.32:0/2725716910 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f22f81a0890 con 0x7f22f8103e50 2024-04-25T00:33:00.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.914+0000 7f22ff5ec700 1 -- 172.21.15.32:0/2725716910 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f22f8068f20 con 0x7f22f8103e50 2024-04-25T00:33:00.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.915+0000 7f22ee7fc700 1 -- 172.21.15.32:0/2725716910 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 18) ==== 137833+0+0 (secure 0 0 0) 0x7f22f0019020 con 0x7f22f8103e50 2024-04-25T00:33:00.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.916+0000 7f22ee7fc700 1 --2- 172.21.15.32:0/2725716910 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f22e40a2c20 0x7f22e40a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:00.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.916+0000 7f22ee7fc700 1 -- 172.21.15.32:0/2725716910 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7f22f00cf6d0 con 0x7f22f8103e50 2024-04-25T00:33:00.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.916+0000 7f22fd388700 1 --2- 172.21.15.32:0/2725716910 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f22e40a2c20 0x7f22e40a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:00.917 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.916+0000 7f22fd388700 1 --2- 172.21.15.32:0/2725716910 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f22e40a2c20 0x7f22e40a50e0 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7f22f8103310 tx=0x7f22e800b410 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:00.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:00.919+0000 7f22ee7fc700 1 -- 172.21.15.32:0/2725716910 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f22f0020020 con 0x7f22f8103e50 2024-04-25T00:33:01.172 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:01.172+0000 7f22ff5ec700 1 -- 172.21.15.32:0/2725716910 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump"} v 0) -- 0x7f22f802db60 con 0x7f22f8103e50 2024-04-25T00:33:01.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:01.172+0000 7f22ee7fc700 1 -- 172.21.15.32:0/2725716910 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump"}]=0 dumped fsmap epoch 24 v24) ==== 76+0+3054 (secure 0 0 0) 0x7f22f000ae00 con 0x7f22f8103e50 2024-04-25T00:33:01.173 INFO:teuthology.orchestra.run.smithi032.stdout:e24 2024-04-25T00:33:01.173 INFO:teuthology.orchestra.run.smithi032.stdout:enable_multiple, ever_enabled_multiple: 1,1 2024-04-25T00:33:01.173 INFO:teuthology.orchestra.run.smithi032.stdout:default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2,11=minor log segments,12=quiesce subvolumes} 2024-04-25T00:33:01.174 INFO:teuthology.orchestra.run.smithi032.stdout:legacy client fscid: 1 2024-04-25T00:33:01.174 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:33:01.174 INFO:teuthology.orchestra.run.smithi032.stdout:Filesystem 'cephfs' (1) 2024-04-25T00:33:01.174 INFO:teuthology.orchestra.run.smithi032.stdout:fs_name cephfs 2024-04-25T00:33:01.174 INFO:teuthology.orchestra.run.smithi032.stdout:epoch 24 2024-04-25T00:33:01.174 INFO:teuthology.orchestra.run.smithi032.stdout:flags 32 joinable allow_snaps allow_multimds_snaps allow_standby_replay 2024-04-25T00:33:01.174 INFO:teuthology.orchestra.run.smithi032.stdout:created 2024-04-25T00:30:47.866875+0000 2024-04-25T00:33:01.174 INFO:teuthology.orchestra.run.smithi032.stdout:modified 2024-04-25T00:31:14.972514+0000 2024-04-25T00:33:01.174 INFO:teuthology.orchestra.run.smithi032.stdout:tableserver 0 2024-04-25T00:33:01.174 INFO:teuthology.orchestra.run.smithi032.stdout:root 0 2024-04-25T00:33:01.174 INFO:teuthology.orchestra.run.smithi032.stdout:session_timeout 300 2024-04-25T00:33:01.174 INFO:teuthology.orchestra.run.smithi032.stdout:session_autoclose 300 2024-04-25T00:33:01.174 INFO:teuthology.orchestra.run.smithi032.stdout:max_file_size 1099511627776 2024-04-25T00:33:01.174 INFO:teuthology.orchestra.run.smithi032.stdout:max_xattr_size 65536 2024-04-25T00:33:01.175 INFO:teuthology.orchestra.run.smithi032.stdout:required_client_features {} 2024-04-25T00:33:01.175 INFO:teuthology.orchestra.run.smithi032.stdout:last_failure 0 2024-04-25T00:33:01.175 INFO:teuthology.orchestra.run.smithi032.stdout:last_failure_osd_epoch 0 2024-04-25T00:33:01.175 INFO:teuthology.orchestra.run.smithi032.stdout:compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2,11=minor log segments,12=quiesce subvolumes} 2024-04-25T00:33:01.175 INFO:teuthology.orchestra.run.smithi032.stdout:max_mds 3 2024-04-25T00:33:01.175 INFO:teuthology.orchestra.run.smithi032.stdout:in 0,1,2 2024-04-25T00:33:01.175 INFO:teuthology.orchestra.run.smithi032.stdout:up {0=24506,1=24481,2=24476} 2024-04-25T00:33:01.175 INFO:teuthology.orchestra.run.smithi032.stdout:failed 2024-04-25T00:33:01.175 INFO:teuthology.orchestra.run.smithi032.stdout:damaged 2024-04-25T00:33:01.175 INFO:teuthology.orchestra.run.smithi032.stdout:stopped 2024-04-25T00:33:01.175 INFO:teuthology.orchestra.run.smithi032.stdout:data_pools [3,4] 2024-04-25T00:33:01.175 INFO:teuthology.orchestra.run.smithi032.stdout:metadata_pool 2 2024-04-25T00:33:01.175 INFO:teuthology.orchestra.run.smithi032.stdout:inline_data disabled 2024-04-25T00:33:01.175 INFO:teuthology.orchestra.run.smithi032.stdout:balancer 2024-04-25T00:33:01.175 INFO:teuthology.orchestra.run.smithi032.stdout:bal_rank_mask -1 2024-04-25T00:33:01.176 INFO:teuthology.orchestra.run.smithi032.stdout:standby_count_wanted 0 2024-04-25T00:33:01.176 INFO:teuthology.orchestra.run.smithi032.stdout:qdb_cluster leader: 24506 members: 24476,24481,24506 2024-04-25T00:33:01.176 INFO:teuthology.orchestra.run.smithi032.stdout:[mds.b{0:24506} state up:active seq 3 addr [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] compat {c=[1],r=[1],i=[1fff]}] 2024-04-25T00:33:01.176 INFO:teuthology.orchestra.run.smithi032.stdout:[mds.c{0:24457} state up:standby-replay seq 1 addr [v2:172.21.15.130:6836/1920876316,v1:172.21.15.130:6837/1920876316] compat {c=[1],r=[1],i=[1fff]}] 2024-04-25T00:33:01.176 INFO:teuthology.orchestra.run.smithi032.stdout:[mds.i{1:24481} state up:active seq 11 addr [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] compat {c=[1],r=[1],i=[1fff]}] 2024-04-25T00:33:01.176 INFO:teuthology.orchestra.run.smithi032.stdout:[mds.f{1:24445} state up:standby-replay seq 1 addr [v2:172.21.15.130:6834/2892666533,v1:172.21.15.130:6835/2892666533] compat {c=[1],r=[1],i=[1fff]}] 2024-04-25T00:33:01.176 INFO:teuthology.orchestra.run.smithi032.stdout:[mds.h{2:24476} state up:active seq 12 addr [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] compat {c=[1],r=[1],i=[1fff]}] 2024-04-25T00:33:01.176 INFO:teuthology.orchestra.run.smithi032.stdout:[mds.e{2:24443} state up:standby-replay seq 1 addr [v2:172.21.15.121:6834/271690252,v1:172.21.15.121:6835/271690252] compat {c=[1],r=[1],i=[1fff]}] 2024-04-25T00:33:01.176 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:33:01.176 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:33:01.176 INFO:teuthology.orchestra.run.smithi032.stdout:Standby daemons: 2024-04-25T00:33:01.176 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:33:01.176 INFO:teuthology.orchestra.run.smithi032.stdout:[mds.j{-1:14559} state up:standby seq 1 addr [v2:172.21.15.32:6834/3511697670,v1:172.21.15.32:6835/3511697670] compat {c=[1],r=[1],i=[1fff]}] 2024-04-25T00:33:01.176 INFO:teuthology.orchestra.run.smithi032.stdout:[mds.d{-1:14571} state up:standby seq 1 addr [v2:172.21.15.32:6836/2212842316,v1:172.21.15.32:6837/2212842316] compat {c=[1],r=[1],i=[1fff]}] 2024-04-25T00:33:01.177 INFO:teuthology.orchestra.run.smithi032.stdout:[mds.g{-1:14643} state up:standby seq 1 addr [v2:172.21.15.32:6838/187492674,v1:172.21.15.32:6839/187492674] compat {c=[1],r=[1],i=[1fff]}] 2024-04-25T00:33:01.177 INFO:teuthology.orchestra.run.smithi032.stdout:[mds.a{-1:14655} state up:standby seq 1 addr [v2:172.21.15.32:6840/1948785545,v1:172.21.15.32:6841/1948785545] compat {c=[1],r=[1],i=[1fff]}] 2024-04-25T00:33:01.177 INFO:teuthology.orchestra.run.smithi032.stdout:[mds.l{-1:24412} state up:standby seq 1 addr [v2:172.21.15.130:6832/3000660373,v1:172.21.15.130:6833/3000660373] compat {c=[1],r=[1],i=[1fff]}] 2024-04-25T00:33:01.177 INFO:teuthology.orchestra.run.smithi032.stdout:[mds.k{-1:24416} state up:standby seq 1 addr [v2:172.21.15.121:6832/436688740,v1:172.21.15.121:6833/436688740] compat {c=[1],r=[1],i=[1fff]}] 2024-04-25T00:33:01.178 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:01.174+0000 7f22ff5ec700 1 -- 172.21.15.32:0/2725716910 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f22e40a2c20 msgr2=0x7f22e40a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:01.178 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:01.174+0000 7f22ff5ec700 1 --2- 172.21.15.32:0/2725716910 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f22e40a2c20 0x7f22e40a50e0 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7f22f8103310 tx=0x7f22e800b410 comp rx=0 tx=0).stop 2024-04-25T00:33:01.178 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:01.174+0000 7f22ff5ec700 1 -- 172.21.15.32:0/2725716910 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22f8103e50 msgr2=0x7f22f819fa10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:01.178 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:01.174+0000 7f22ff5ec700 1 --2- 172.21.15.32:0/2725716910 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22f8103e50 0x7f22f819fa10 secure :-1 s=READY pgs=399 cs=0 l=1 rev1=1 crypto rx=0x7f22f0007da0 tx=0x7f22f000a340 comp rx=0 tx=0).stop 2024-04-25T00:33:01.178 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:01.175+0000 7f22ff5ec700 1 -- 172.21.15.32:0/2725716910 shutdown_connections 2024-04-25T00:33:01.178 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:01.175+0000 7f22ff5ec700 1 --2- 172.21.15.32:0/2725716910 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f22e40a2c20 0x7f22e40a50e0 unknown :-1 s=CLOSED pgs=244 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:01.178 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:01.175+0000 7f22ff5ec700 1 --2- 172.21.15.32:0/2725716910 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22f8103e50 0x7f22f819fa10 unknown :-1 s=CLOSED pgs=399 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:01.178 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:01.175+0000 7f22ff5ec700 1 --2- 172.21.15.32:0/2725716910 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f22f81022b0 0x7f22f8198450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:01.178 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:01.175+0000 7f22ff5ec700 1 --2- 172.21.15.32:0/2725716910 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22f81034b0 0x7f22f8198990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:01.178 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:01.175+0000 7f22ff5ec700 1 -- 172.21.15.32:0/2725716910 >> 172.21.15.32:0/2725716910 conn(0x7f22f80fd850 msgr2=0x7f22f81067b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:01.178 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:01.175+0000 7f22ff5ec700 1 -- 172.21.15.32:0/2725716910 shutdown_connections 2024-04-25T00:33:01.178 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:01.175+0000 7f22ff5ec700 1 -- 172.21.15.32:0/2725716910 wait complete. 2024-04-25T00:33:01.178 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 24 2024-04-25T00:33:01.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:01 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2725716910' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2024-04-25T00:33:01.838 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cb997156-0299-11ef-bc93-c7b262605968 -- bash -c 'ceph osd dump' 2024-04-25T00:33:01.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:01 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2725716910' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2024-04-25T00:33:01.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:01 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2725716910' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2024-04-25T00:33:02.088 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:33:02.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:02 smithi032 ceph-mon[29357]: pgmap v362: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s; 9 B/s, 0 objects/s recovering 2024-04-25T00:33:02.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:02 smithi121 ceph-mon[32231]: pgmap v362: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s; 9 B/s, 0 objects/s recovering 2024-04-25T00:33:02.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:02 smithi130 ceph-mon[32525]: pgmap v362: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s; 9 B/s, 0 objects/s recovering 2024-04-25T00:33:03.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.826+0000 7f71c693a700 1 -- 172.21.15.32:0/3325220032 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71c01022b0 msgr2=0x7f71c01026d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:03.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.826+0000 7f71c693a700 1 --2- 172.21.15.32:0/3325220032 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71c01022b0 0x7f71c01026d0 secure :-1 s=READY pgs=400 cs=0 l=1 rev1=1 crypto rx=0x7f71b00099c0 tx=0x7f71b0009cd0 comp rx=0 tx=0).stop 2024-04-25T00:33:03.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.826+0000 7f71c693a700 1 -- 172.21.15.32:0/3325220032 shutdown_connections 2024-04-25T00:33:03.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.826+0000 7f71c693a700 1 --2- 172.21.15.32:0/3325220032 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f71c0103e50 0x7f71c010a700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:03.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.826+0000 7f71c693a700 1 --2- 172.21.15.32:0/3325220032 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71c01022b0 0x7f71c01026d0 unknown :-1 s=CLOSED pgs=400 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:03.828 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.826+0000 7f71c693a700 1 --2- 172.21.15.32:0/3325220032 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f71c01034b0 0x7f71c0103910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:03.828 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.826+0000 7f71c693a700 1 -- 172.21.15.32:0/3325220032 >> 172.21.15.32:0/3325220032 conn(0x7f71c00fd850 msgr2=0x7f71c00ffc90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:03.828 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.826+0000 7f71c693a700 1 -- 172.21.15.32:0/3325220032 shutdown_connections 2024-04-25T00:33:03.828 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.826+0000 7f71c693a700 1 -- 172.21.15.32:0/3325220032 wait complete. 2024-04-25T00:33:03.828 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.826+0000 7f71c693a700 1 Processor -- start 2024-04-25T00:33:03.828 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.826+0000 7f71c693a700 1 -- start start 2024-04-25T00:33:03.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.827+0000 7f71c693a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f71c01022b0 0x7f71c01984b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:03.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.827+0000 7f71c693a700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71c01034b0 0x7f71c01989f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:03.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.827+0000 7f71c693a700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f71c0103e50 0x7f71c019fa70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:03.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.827+0000 7f71c693a700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f71c010de70 con 0x7f71c01034b0 2024-04-25T00:33:03.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.827+0000 7f71c693a700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f71c010dcf0 con 0x7f71c0103e50 2024-04-25T00:33:03.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.827+0000 7f71c693a700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f71c010dff0 con 0x7f71c01022b0 2024-04-25T00:33:03.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.827+0000 7f71bf7fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71c01034b0 0x7f71c01989f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:03.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.827+0000 7f71bf7fe700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71c01034b0 0x7f71c01989f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:43926/0 (socket says 172.21.15.32:43926) 2024-04-25T00:33:03.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.827+0000 7f71bf7fe700 1 -- 172.21.15.32:0/2823208417 learned_addr learned my addr 172.21.15.32:0/2823208417 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:03.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.827+0000 7f71c4ed7700 1 --2- 172.21.15.32:0/2823208417 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f71c0103e50 0x7f71c019fa70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:03.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.827+0000 7f71bffff700 1 --2- 172.21.15.32:0/2823208417 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f71c01022b0 0x7f71c01984b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:03.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.827+0000 7f71bf7fe700 1 -- 172.21.15.32:0/2823208417 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f71c01022b0 msgr2=0x7f71c01984b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:03.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.827+0000 7f71bf7fe700 1 --2- 172.21.15.32:0/2823208417 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f71c01022b0 0x7f71c01984b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:03.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.827+0000 7f71bf7fe700 1 -- 172.21.15.32:0/2823208417 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f71c0103e50 msgr2=0x7f71c019fa70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:03.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.827+0000 7f71bf7fe700 1 --2- 172.21.15.32:0/2823208417 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f71c0103e50 0x7f71c019fa70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:03.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.827+0000 7f71bf7fe700 1 -- 172.21.15.32:0/2823208417 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f71c01a0110 con 0x7f71c01034b0 2024-04-25T00:33:03.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.828+0000 7f71bf7fe700 1 --2- 172.21.15.32:0/2823208417 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71c01034b0 0x7f71c01989f0 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 crypto rx=0x7f71b400c950 tx=0x7f71b400cc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:03.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.828+0000 7f71bd7fa700 1 -- 172.21.15.32:0/2823208417 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f71b40078d0 con 0x7f71c01034b0 2024-04-25T00:33:03.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.828+0000 7f71c693a700 1 -- 172.21.15.32:0/2823208417 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f71c01a0400 con 0x7f71c01034b0 2024-04-25T00:33:03.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.828+0000 7f71bd7fa700 1 -- 172.21.15.32:0/2823208417 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f71b400f450 con 0x7f71c01034b0 2024-04-25T00:33:03.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.828+0000 7f71bd7fa700 1 -- 172.21.15.32:0/2823208417 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f71b4018660 con 0x7f71c01034b0 2024-04-25T00:33:03.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.828+0000 7f71c693a700 1 -- 172.21.15.32:0/2823208417 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f71c01a0990 con 0x7f71c01034b0 2024-04-25T00:33:03.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.829+0000 7f71c693a700 1 -- 172.21.15.32:0/2823208417 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f71c0068f20 con 0x7f71c01034b0 2024-04-25T00:33:03.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.830+0000 7f71bd7fa700 1 -- 172.21.15.32:0/2823208417 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 18) ==== 137833+0+0 (secure 0 0 0) 0x7f71b400f600 con 0x7f71c01034b0 2024-04-25T00:33:03.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.831+0000 7f71bd7fa700 1 --2- 172.21.15.32:0/2823208417 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f71a80a2c20 0x7f71a80a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:03.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.831+0000 7f71bffff700 1 --2- 172.21.15.32:0/2823208417 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f71a80a2c20 0x7f71a80a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:03.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.831+0000 7f71bd7fa700 1 -- 172.21.15.32:0/2823208417 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7f71b40cf280 con 0x7f71c01034b0 2024-04-25T00:33:03.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.831+0000 7f71bffff700 1 --2- 172.21.15.32:0/2823208417 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f71a80a2c20 0x7f71a80a50e0 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7f71b000b580 tx=0x7f71b0005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:03.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:03.835+0000 7f71bd7fa700 1 -- 172.21.15.32:0/2823208417 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f71b4014030 con 0x7f71c01034b0 2024-04-25T00:33:04.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.044+0000 7f71c693a700 1 -- 172.21.15.32:0/2823208417 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump"} v 0) -- 0x7f71c002db30 con 0x7f71c01034b0 2024-04-25T00:33:04.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.045+0000 7f71bd7fa700 1 -- 172.21.15.32:0/2823208417 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump"}]=0 v93) ==== 56+0+5706 (secure 0 0 0) 0x7f71b409bdb0 con 0x7f71c01034b0 2024-04-25T00:33:04.046 INFO:teuthology.orchestra.run.smithi032.stdout:epoch 93 2024-04-25T00:33:04.046 INFO:teuthology.orchestra.run.smithi032.stdout:fsid cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:33:04.047 INFO:teuthology.orchestra.run.smithi032.stdout:created 2024-04-25T00:22:46.105398+0000 2024-04-25T00:33:04.047 INFO:teuthology.orchestra.run.smithi032.stdout:modified 2024-04-25T00:32:45.458564+0000 2024-04-25T00:33:04.047 INFO:teuthology.orchestra.run.smithi032.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2024-04-25T00:33:04.047 INFO:teuthology.orchestra.run.smithi032.stdout:crush_version 27 2024-04-25T00:33:04.047 INFO:teuthology.orchestra.run.smithi032.stdout:full_ratio 0.95 2024-04-25T00:33:04.047 INFO:teuthology.orchestra.run.smithi032.stdout:backfillfull_ratio 0.9 2024-04-25T00:33:04.048 INFO:teuthology.orchestra.run.smithi032.stdout:nearfull_ratio 0.85 2024-04-25T00:33:04.048 INFO:teuthology.orchestra.run.smithi032.stdout:require_min_compat_client luminous 2024-04-25T00:33:04.048 INFO:teuthology.orchestra.run.smithi032.stdout:min_compat_client luminous 2024-04-25T00:33:04.048 INFO:teuthology.orchestra.run.smithi032.stdout:require_osd_release squid 2024-04-25T00:33:04.048 INFO:teuthology.orchestra.run.smithi032.stdout:stretch_mode_enabled false 2024-04-25T00:33:04.049 INFO:teuthology.orchestra.run.smithi032.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 21 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 12.00 2024-04-25T00:33:04.049 INFO:teuthology.orchestra.run.smithi032.stdout:pool 2 'cephfs_metadata' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 84 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.88 2024-04-25T00:33:04.049 INFO:teuthology.orchestra.run.smithi032.stdout:pool 3 'cephfs_data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 64 pgp_num 64 autoscale_mode off last_change 85 flags hashpspool stripe_width 0 pg_num_min 64 target_size_ratio 0.05 application cephfs read_balance_score 1.50 2024-04-25T00:33:04.049 INFO:teuthology.orchestra.run.smithi032.stdout:pool 4 'cephfs_data_ec' erasure profile cephfs_data_ec size 4 min_size 3 crush_rule 1 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 89 flags hashpspool,ec_overwrites stripe_width 8192 pg_num_min 64 target_size_ratio 0.9 application cephfs 2024-04-25T00:33:04.049 INFO:teuthology.orchestra.run.smithi032.stdout:max_osd 12 2024-04-25T00:33:04.049 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0 up in weight 1 up_from 8 up_thru 90 down_at 0 last_clean_interval [0,0) [v2:172.21.15.32:6802/3984695193,v1:172.21.15.32:6803/3984695193] [v2:172.21.15.32:6804/3984695193,v1:172.21.15.32:6805/3984695193] exists,up 074681d6-1be7-49a1-9518-9968ad62dc08 2024-04-25T00:33:04.049 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1 up in weight 1 up_from 13 up_thru 87 down_at 0 last_clean_interval [0,0) [v2:172.21.15.121:6800/431442351,v1:172.21.15.121:6801/431442351] [v2:172.21.15.121:6802/431442351,v1:172.21.15.121:6803/431442351] exists,up e7ea9910-062b-4801-b1a5-a90a56d73b66 2024-04-25T00:33:04.050 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2 up in weight 1 up_from 18 up_thru 90 down_at 0 last_clean_interval [0,0) [v2:172.21.15.130:6800/1393844110,v1:172.21.15.130:6801/1393844110] [v2:172.21.15.130:6802/1393844110,v1:172.21.15.130:6803/1393844110] exists,up 8b49a8b4-65a2-4b96-8ab0-350f338cd6f3 2024-04-25T00:33:04.050 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3 up in weight 1 up_from 25 up_thru 87 down_at 0 last_clean_interval [0,0) [v2:172.21.15.32:6810/1510492708,v1:172.21.15.32:6811/1510492708] [v2:172.21.15.32:6812/1510492708,v1:172.21.15.32:6813/1510492708] exists,up 3956963a-c4a2-41ee-a138-3c5acef75dcb 2024-04-25T00:33:04.050 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 up in weight 1 up_from 33 up_thru 90 down_at 0 last_clean_interval [0,0) [v2:172.21.15.121:6808/1286757826,v1:172.21.15.121:6809/1286757826] [v2:172.21.15.121:6810/1286757826,v1:172.21.15.121:6811/1286757826] exists,up 5dede06a-2cc8-494b-a0ec-13551677b878 2024-04-25T00:33:04.050 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5 up in weight 1 up_from 38 up_thru 87 down_at 0 last_clean_interval [0,0) [v2:172.21.15.130:6808/909656116,v1:172.21.15.130:6809/909656116] [v2:172.21.15.130:6810/909656116,v1:172.21.15.130:6811/909656116] exists,up 3360c304-a7ae-4efc-a931-3b6261bbf24e 2024-04-25T00:33:04.050 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6 up in weight 1 up_from 46 up_thru 92 down_at 0 last_clean_interval [0,0) [v2:172.21.15.32:6818/3198957027,v1:172.21.15.32:6819/3198957027] [v2:172.21.15.32:6820/3198957027,v1:172.21.15.32:6821/3198957027] exists,up f60af4f3-4dd1-4021-a778-d0d97e9a674e 2024-04-25T00:33:04.050 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7 up in weight 1 up_from 51 up_thru 87 down_at 0 last_clean_interval [0,0) [v2:172.21.15.121:6816/3677614004,v1:172.21.15.121:6817/3677614004] [v2:172.21.15.121:6818/3677614004,v1:172.21.15.121:6819/3677614004] exists,up 7181e550-d24b-40ee-8d52-9b650a041739 2024-04-25T00:33:04.051 INFO:teuthology.orchestra.run.smithi032.stdout:osd.8 up in weight 1 up_from 59 up_thru 92 down_at 0 last_clean_interval [0,0) [v2:172.21.15.130:6816/285467211,v1:172.21.15.130:6817/285467211] [v2:172.21.15.130:6818/285467211,v1:172.21.15.130:6819/285467211] exists,up 700b2360-e592-406b-a97e-cf9387feae70 2024-04-25T00:33:04.051 INFO:teuthology.orchestra.run.smithi032.stdout:osd.9 up in weight 1 up_from 67 up_thru 90 down_at 0 last_clean_interval [0,0) [v2:172.21.15.32:6826/4290214050,v1:172.21.15.32:6827/4290214050] [v2:172.21.15.32:6828/4290214050,v1:172.21.15.32:6829/4290214050] exists,up 4457f5a8-6991-48c5-a382-dbca97cb206e 2024-04-25T00:33:04.051 INFO:teuthology.orchestra.run.smithi032.stdout:osd.10 up in weight 1 up_from 72 up_thru 90 down_at 0 last_clean_interval [0,0) [v2:172.21.15.121:6824/3217408278,v1:172.21.15.121:6825/3217408278] [v2:172.21.15.121:6826/3217408278,v1:172.21.15.121:6827/3217408278] exists,up 16161ce2-7971-4403-95d4-1c6b2977ce2f 2024-04-25T00:33:04.051 INFO:teuthology.orchestra.run.smithi032.stdout:osd.11 up in weight 1 up_from 80 up_thru 92 down_at 0 last_clean_interval [0,0) [v2:172.21.15.130:6824/2386788822,v1:172.21.15.130:6825/2386788822] [v2:172.21.15.130:6826/2386788822,v1:172.21.15.130:6827/2386788822] exists,up fdf0b14b-2f62-4c0b-aec5-a8304d03042b 2024-04-25T00:33:04.051 INFO:teuthology.orchestra.run.smithi032.stdout:pg_upmap_items 2.b [8,11] 2024-04-25T00:33:04.051 INFO:teuthology.orchestra.run.smithi032.stdout:pg_upmap_items 2.e [2,1] 2024-04-25T00:33:04.052 INFO:teuthology.orchestra.run.smithi032.stdout:pg_upmap_items 2.f [8,11] 2024-04-25T00:33:04.052 INFO:teuthology.orchestra.run.smithi032.stdout:pg_upmap_items 2.10 [2,4] 2024-04-25T00:33:04.052 INFO:teuthology.orchestra.run.smithi032.stdout:pg_upmap_items 3.6 [3,6] 2024-04-25T00:33:04.052 INFO:teuthology.orchestra.run.smithi032.stdout:pg_upmap_items 3.17 [4,7] 2024-04-25T00:33:04.052 INFO:teuthology.orchestra.run.smithi032.stdout:pg_upmap_items 3.1b [5,9] 2024-04-25T00:33:04.052 INFO:teuthology.orchestra.run.smithi032.stdout:pg_upmap_items 3.20 [5,9] 2024-04-25T00:33:04.053 INFO:teuthology.orchestra.run.smithi032.stdout:pg_upmap_items 3.23 [4,8] 2024-04-25T00:33:04.053 INFO:teuthology.orchestra.run.smithi032.stdout:pg_upmap_items 3.2c [4,9] 2024-04-25T00:33:04.053 INFO:teuthology.orchestra.run.smithi032.stdout:pg_upmap_items 4.c [9,10] 2024-04-25T00:33:04.053 INFO:teuthology.orchestra.run.smithi032.stdout:pg_upmap_items 4.12 [9,10] 2024-04-25T00:33:04.053 INFO:teuthology.orchestra.run.smithi032.stdout:pg_upmap_items 4.1c [9,8] 2024-04-25T00:33:04.053 INFO:teuthology.orchestra.run.smithi032.stdout:pg_upmap_items 4.1d [5,8] 2024-04-25T00:33:04.054 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:0/2999636731 expires 2024-04-26T00:23:42.776439+0000 2024-04-25T00:33:04.054 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:6801/4160192081 expires 2024-04-26T00:23:42.776439+0000 2024-04-25T00:33:04.054 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:6800/4160192081 expires 2024-04-26T00:23:42.776439+0000 2024-04-25T00:33:04.054 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:0/53550767 expires 2024-04-26T00:23:42.776439+0000 2024-04-25T00:33:04.054 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:0/1066471448 expires 2024-04-26T00:23:14.613921+0000 2024-04-25T00:33:04.054 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:0/3987019264 expires 2024-04-26T00:23:14.613921+0000 2024-04-25T00:33:04.055 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:0/3041942084 expires 2024-04-26T00:23:14.613921+0000 2024-04-25T00:33:04.055 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:0/3004184616 expires 2024-04-26T00:23:42.776439+0000 2024-04-25T00:33:04.055 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:6800/1637256323 expires 2024-04-26T00:23:14.613921+0000 2024-04-25T00:33:04.055 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:6801/1637256323 expires 2024-04-26T00:23:14.613921+0000 2024-04-25T00:33:04.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.047+0000 7f71c693a700 1 -- 172.21.15.32:0/2823208417 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f71a80a2c20 msgr2=0x7f71a80a50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:04.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.047+0000 7f71c693a700 1 --2- 172.21.15.32:0/2823208417 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f71a80a2c20 0x7f71a80a50e0 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7f71b000b580 tx=0x7f71b0005fb0 comp rx=0 tx=0).stop 2024-04-25T00:33:04.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.047+0000 7f71c693a700 1 -- 172.21.15.32:0/2823208417 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71c01034b0 msgr2=0x7f71c01989f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:04.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.047+0000 7f71c693a700 1 --2- 172.21.15.32:0/2823208417 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71c01034b0 0x7f71c01989f0 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 crypto rx=0x7f71b400c950 tx=0x7f71b400cc60 comp rx=0 tx=0).stop 2024-04-25T00:33:04.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.048+0000 7f71c693a700 1 -- 172.21.15.32:0/2823208417 shutdown_connections 2024-04-25T00:33:04.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.048+0000 7f71c693a700 1 --2- 172.21.15.32:0/2823208417 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f71a80a2c20 0x7f71a80a50e0 unknown :-1 s=CLOSED pgs=245 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:04.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.048+0000 7f71c693a700 1 --2- 172.21.15.32:0/2823208417 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f71c0103e50 0x7f71c019fa70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:04.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.048+0000 7f71c693a700 1 --2- 172.21.15.32:0/2823208417 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f71c01022b0 0x7f71c01984b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:04.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.048+0000 7f71c693a700 1 --2- 172.21.15.32:0/2823208417 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f71c01034b0 0x7f71c01989f0 unknown :-1 s=CLOSED pgs=401 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:04.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.048+0000 7f71c693a700 1 -- 172.21.15.32:0/2823208417 >> 172.21.15.32:0/2823208417 conn(0x7f71c00fd850 msgr2=0x7f71c0106800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:04.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.048+0000 7f71c693a700 1 -- 172.21.15.32:0/2823208417 shutdown_connections 2024-04-25T00:33:04.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.048+0000 7f71c693a700 1 -- 172.21.15.32:0/2823208417 wait complete. 2024-04-25T00:33:04.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:04 smithi032 ceph-mon[29357]: pgmap v363: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 7 op/s 2024-04-25T00:33:04.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:04 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2823208417' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-25T00:33:04.786 INFO:teuthology.run_tasks:Running task fs.ready... 2024-04-25T00:33:04.798 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-25T00:33:04.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:04 smithi121 ceph-mon[32231]: pgmap v363: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 7 op/s 2024-04-25T00:33:04.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:04 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2823208417' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-25T00:33:04.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:04 smithi130 ceph-mon[32525]: pgmap v363: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 7 op/s 2024-04-25T00:33:04.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:04 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2823208417' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-25T00:33:04.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.911+0000 7fddb2d61640 1 -- 172.21.15.32:0/597887028 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fddac101890 msgr2=0x7fddac101d10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:04.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.911+0000 7fddb2d61640 1 --2- 172.21.15.32:0/597887028 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fddac101890 0x7fddac101d10 secure :-1 s=READY pgs=402 cs=0 l=1 rev1=1 crypto rx=0x7fdda0009900 tx=0x7fdda002a410 comp rx=0 tx=0).stop 2024-04-25T00:33:04.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.911+0000 7fddb2d61640 1 -- 172.21.15.32:0/597887028 shutdown_connections 2024-04-25T00:33:04.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.911+0000 7fddb2d61640 1 --2- 172.21.15.32:0/597887028 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fddac102250 0x7fddac108ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:04.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.911+0000 7fddb2d61640 1 --2- 172.21.15.32:0/597887028 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fddac101890 0x7fddac101d10 unknown :-1 s=CLOSED pgs=402 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:04.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.911+0000 7fddb2d61640 1 --2- 172.21.15.32:0/597887028 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fddac1006a0 0x7fddac100ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:04.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.911+0000 7fddb2d61640 1 -- 172.21.15.32:0/597887028 >> 172.21.15.32:0/597887028 conn(0x7fddac0fbc30 msgr2=0x7fddac0fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:04.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.911+0000 7fddb2d61640 1 -- 172.21.15.32:0/597887028 shutdown_connections 2024-04-25T00:33:04.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.911+0000 7fddb2d61640 1 -- 172.21.15.32:0/597887028 wait complete. 2024-04-25T00:33:04.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.911+0000 7fddb2d61640 1 Processor -- start 2024-04-25T00:33:04.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.911+0000 7fddb2d61640 1 -- start start 2024-04-25T00:33:04.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.912+0000 7fddb2d61640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fddac1006a0 0x7fddac197ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:04.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.912+0000 7fddb2d61640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fddac101890 0x7fddac197ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:04.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.912+0000 7fddb2d61640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fddac102250 0x7fddac19f060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:04.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.912+0000 7fddb2d61640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fddac10d4a0 con 0x7fddac101890 2024-04-25T00:33:04.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.912+0000 7fddb2d61640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fddac10d320 con 0x7fddac102250 2024-04-25T00:33:04.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.912+0000 7fddb2d61640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fddac10d620 con 0x7fddac1006a0 2024-04-25T00:33:04.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.912+0000 7fddabfff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fddac101890 0x7fddac197ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:04.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.912+0000 7fddabfff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fddac101890 0x7fddac197ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:43948/0 (socket says 172.21.15.32:43948) 2024-04-25T00:33:04.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.912+0000 7fddabfff640 1 -- 172.21.15.32:0/1134507197 learned_addr learned my addr 172.21.15.32:0/1134507197 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:04.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.912+0000 7fddb0afd640 1 --2- 172.21.15.32:0/1134507197 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fddac1006a0 0x7fddac197ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:04.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.912+0000 7fddb12fe640 1 --2- 172.21.15.32:0/1134507197 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fddac102250 0x7fddac19f060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:04.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.912+0000 7fddabfff640 1 -- 172.21.15.32:0/1134507197 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fddac1006a0 msgr2=0x7fddac197ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:04.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.912+0000 7fddabfff640 1 --2- 172.21.15.32:0/1134507197 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fddac1006a0 0x7fddac197ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:04.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.912+0000 7fddabfff640 1 -- 172.21.15.32:0/1134507197 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fddac102250 msgr2=0x7fddac19f060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:04.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.912+0000 7fddabfff640 1 --2- 172.21.15.32:0/1134507197 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fddac102250 0x7fddac19f060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:04.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.912+0000 7fddabfff640 1 -- 172.21.15.32:0/1134507197 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fddac19f6d0 con 0x7fddac101890 2024-04-25T00:33:04.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.912+0000 7fddb0afd640 1 --2- 172.21.15.32:0/1134507197 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fddac1006a0 0x7fddac197ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:04.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.912+0000 7fddb12fe640 1 --2- 172.21.15.32:0/1134507197 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fddac102250 0x7fddac19f060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:04.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.912+0000 7fddabfff640 1 --2- 172.21.15.32:0/1134507197 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fddac101890 0x7fddac197ff0 secure :-1 s=READY pgs=403 cs=0 l=1 rev1=1 crypto rx=0x7fdda0031440 tx=0x7fdda0032e20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:04.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.912+0000 7fdda9ffb640 1 -- 172.21.15.32:0/1134507197 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdda003f030 con 0x7fddac101890 2024-04-25T00:33:04.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.912+0000 7fdda9ffb640 1 -- 172.21.15.32:0/1134507197 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fdda0003ce0 con 0x7fddac101890 2024-04-25T00:33:04.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.912+0000 7fddb2d61640 1 -- 172.21.15.32:0/1134507197 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fddac19f960 con 0x7fddac101890 2024-04-25T00:33:04.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.912+0000 7fdda9ffb640 1 -- 172.21.15.32:0/1134507197 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdda0035070 con 0x7fddac101890 2024-04-25T00:33:04.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.913+0000 7fddb2d61640 1 -- 172.21.15.32:0/1134507197 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fddac19fe50 con 0x7fddac101890 2024-04-25T00:33:04.917 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.914+0000 7fddb2d61640 1 -- 172.21.15.32:0/1134507197 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fddac107a60 con 0x7fddac101890 2024-04-25T00:33:04.917 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.915+0000 7fdda9ffb640 1 -- 172.21.15.32:0/1134507197 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 18) ==== 137833+0+0 (secure 0 0 0) 0x7fdda0042030 con 0x7fddac101890 2024-04-25T00:33:04.917 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.916+0000 7fdda9ffb640 1 --2- 172.21.15.32:0/1134507197 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdd940a2b90 0x7fdd940a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:04.917 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.916+0000 7fdda9ffb640 1 -- 172.21.15.32:0/1134507197 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7fdda00efad0 con 0x7fddac101890 2024-04-25T00:33:04.917 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.916+0000 7fddb0afd640 1 --2- 172.21.15.32:0/1134507197 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdd940a2b90 0x7fdd940a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:04.917 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.916+0000 7fddb0afd640 1 --2- 172.21.15.32:0/1134507197 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdd940a2b90 0x7fdd940a5050 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7fdd9c0058c0 tx=0x7fdd9c005850 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:04.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:04.920+0000 7fdda9ffb640 1 -- 172.21.15.32:0/1134507197 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fdda00bc660 con 0x7fddac101890 2024-04-25T00:33:05.106 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.106+0000 7fddb2d61640 1 -- 172.21.15.32:0/1134507197 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fddac062340 con 0x7fddac101890 2024-04-25T00:33:05.108 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.107+0000 7fdda9ffb640 1 -- 172.21.15.32:0/1134507197 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v93) ==== 74+0+24843 (secure 0 0 0) 0x7fdda00bbf60 con 0x7fddac101890 2024-04-25T00:33:05.108 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:33:05.108 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":93,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:32:45.458564+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-25T00:30:46.239556+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-25T00:30:47.257182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2024-04-25T00:30:51.193975+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.b","mappings":[{"from":8,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":1}]},{"pgid":"2.f","mappings":[{"from":8,"to":11}]},{"pgid":"2.10","mappings":[{"from":2,"to":4}]},{"pgid":"3.6","mappings":[{"from":3,"to":6}]},{"pgid":"3.17","mappings":[{"from":4,"to":7}]},{"pgid":"3.1b","mappings":[{"from":5,"to":9}]},{"pgid":"3.20","mappings":[{"from":5,"to":9}]},{"pgid":"3.23","mappings":[{"from":4,"to":8}]},{"pgid":"3.2c","mappings":[{"from":4,"to":9}]},{"pgid":"4.c","mappings":[{"from":9,"to":10}]},{"pgid":"4.12","mappings":[{"from":9,"to":10}]},{"pgid":"4.1c","mappings":[{"from":9,"to":8}]},{"pgid":"4.1d","mappings":[{"from":5,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-num-failure-domains":"0","crush-osds-per-failure-domain":"0","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:33:05.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.109+0000 7fddb2d61640 1 -- 172.21.15.32:0/1134507197 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdd940a2b90 msgr2=0x7fdd940a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:05.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.109+0000 7fddb2d61640 1 --2- 172.21.15.32:0/1134507197 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdd940a2b90 0x7fdd940a5050 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7fdd9c0058c0 tx=0x7fdd9c005850 comp rx=0 tx=0).stop 2024-04-25T00:33:05.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.109+0000 7fddb2d61640 1 -- 172.21.15.32:0/1134507197 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fddac101890 msgr2=0x7fddac197ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:05.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.109+0000 7fddb2d61640 1 --2- 172.21.15.32:0/1134507197 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fddac101890 0x7fddac197ff0 secure :-1 s=READY pgs=403 cs=0 l=1 rev1=1 crypto rx=0x7fdda0031440 tx=0x7fdda0032e20 comp rx=0 tx=0).stop 2024-04-25T00:33:05.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.109+0000 7fddb2d61640 1 -- 172.21.15.32:0/1134507197 shutdown_connections 2024-04-25T00:33:05.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.109+0000 7fddb2d61640 1 --2- 172.21.15.32:0/1134507197 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fdd940a2b90 0x7fdd940a5050 unknown :-1 s=CLOSED pgs=246 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.111 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.109+0000 7fddb2d61640 1 --2- 172.21.15.32:0/1134507197 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fddac102250 0x7fddac19f060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.111 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.109+0000 7fddb2d61640 1 --2- 172.21.15.32:0/1134507197 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fddac101890 0x7fddac197ff0 unknown :-1 s=CLOSED pgs=403 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.111 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.109+0000 7fddb2d61640 1 --2- 172.21.15.32:0/1134507197 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fddac1006a0 0x7fddac197ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.111 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.109+0000 7fddb2d61640 1 -- 172.21.15.32:0/1134507197 >> 172.21.15.32:0/1134507197 conn(0x7fddac0fbc30 msgr2=0x7fddac104c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:05.111 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.109+0000 7fddb2d61640 1 -- 172.21.15.32:0/1134507197 shutdown_connections 2024-04-25T00:33:05.111 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.109+0000 7fddb2d61640 1 -- 172.21.15.32:0/1134507197 wait complete. 2024-04-25T00:33:05.121 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-25T00:26:07.306839+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-04-25T00:30:46.239556+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '84', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.8799999952316284, 'score_stable': 1.8799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.8799999952316284, 'raw_score_stable': 1.8799999952316284, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-04-25T00:30:47.257182+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '85', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.05000000074505806}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_data_ec', 'create_time': '2024-04-25T00:30:51.193975+0000', 'flags': 5, 'flags_names': 'hashpspool,ec_overwrites', 'type': 3, 'size': 4, 'min_size': 3, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '89', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'cephfs_data_ec', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 8192, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}] 2024-04-25T00:33:05.122 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-25T00:33:05.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.227+0000 7f6ea5f29640 1 -- 172.21.15.32:0/3190131434 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6ea0103350 msgr2=0x7f6ea0109be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:05.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.227+0000 7f6ea5f29640 1 --2- 172.21.15.32:0/3190131434 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6ea0103350 0x7f6ea0109be0 secure :-1 s=READY pgs=404 cs=0 l=1 rev1=1 crypto rx=0x7f6e98009900 tx=0x7f6e9802a410 comp rx=0 tx=0).stop 2024-04-25T00:33:05.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.228+0000 7f6ea5f29640 1 -- 172.21.15.32:0/3190131434 shutdown_connections 2024-04-25T00:33:05.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.228+0000 7f6ea5f29640 1 --2- 172.21.15.32:0/3190131434 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6ea0103350 0x7f6ea0109be0 unknown :-1 s=CLOSED pgs=404 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.228+0000 7f6ea5f29640 1 --2- 172.21.15.32:0/3190131434 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6ea0102990 0x7f6ea0102e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.228+0000 7f6ea5f29640 1 --2- 172.21.15.32:0/3190131434 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6ea01017a0 0x7f6ea0101bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.228+0000 7f6ea5f29640 1 -- 172.21.15.32:0/3190131434 >> 172.21.15.32:0/3190131434 conn(0x7f6ea00fcd10 msgr2=0x7f6ea00ff150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:05.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.228+0000 7f6ea5f29640 1 -- 172.21.15.32:0/3190131434 shutdown_connections 2024-04-25T00:33:05.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.228+0000 7f6ea5f29640 1 -- 172.21.15.32:0/3190131434 wait complete. 2024-04-25T00:33:05.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.228+0000 7f6ea5f29640 1 Processor -- start 2024-04-25T00:33:05.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.228+0000 7f6ea5f29640 1 -- start start 2024-04-25T00:33:05.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.228+0000 7f6ea5f29640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6ea01017a0 0x7f6ea00754f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:05.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.228+0000 7f6ea5f29640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6ea0102990 0x7f6ea0073b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:05.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.228+0000 7f6ea5f29640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6ea0103350 0x7f6ea0074070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:05.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.228+0000 7f6ea5f29640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f6ea010d490 con 0x7f6ea0103350 2024-04-25T00:33:05.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.228+0000 7f6ea5f29640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f6ea010d310 con 0x7f6ea0102990 2024-04-25T00:33:05.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.228+0000 7f6ea5f29640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f6ea010d610 con 0x7f6ea01017a0 2024-04-25T00:33:05.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.229+0000 7f6e9ffff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6ea0103350 0x7f6ea0074070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:05.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.229+0000 7f6e9ffff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6ea0103350 0x7f6ea0074070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:43960/0 (socket says 172.21.15.32:43960) 2024-04-25T00:33:05.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.229+0000 7f6e9ffff640 1 -- 172.21.15.32:0/974096158 learned_addr learned my addr 172.21.15.32:0/974096158 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:05.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.229+0000 7f6e9f7fe640 1 --2- 172.21.15.32:0/974096158 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6ea01017a0 0x7f6ea00754f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:05.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.229+0000 7f6e9effd640 1 --2- 172.21.15.32:0/974096158 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6ea0102990 0x7f6ea0073b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:05.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.229+0000 7f6e9ffff640 1 -- 172.21.15.32:0/974096158 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6ea01017a0 msgr2=0x7f6ea00754f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:05.233 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.229+0000 7f6e9ffff640 1 --2- 172.21.15.32:0/974096158 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6ea01017a0 0x7f6ea00754f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.233 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.229+0000 7f6e9ffff640 1 -- 172.21.15.32:0/974096158 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6ea0102990 msgr2=0x7f6ea0073b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:05.233 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.229+0000 7f6e9ffff640 1 --2- 172.21.15.32:0/974096158 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6ea0102990 0x7f6ea0073b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.233 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.229+0000 7f6e9ffff640 1 -- 172.21.15.32:0/974096158 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6ea0074900 con 0x7f6ea0103350 2024-04-25T00:33:05.233 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.229+0000 7f6e9ffff640 1 --2- 172.21.15.32:0/974096158 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6ea0103350 0x7f6ea0074070 secure :-1 s=READY pgs=405 cs=0 l=1 rev1=1 crypto rx=0x7f6e980314e0 tx=0x7f6e980043c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:05.233 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.229+0000 7f6e9cff9640 1 -- 172.21.15.32:0/974096158 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6e9803f030 con 0x7f6ea0103350 2024-04-25T00:33:05.233 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.229+0000 7f6e9cff9640 1 -- 172.21.15.32:0/974096158 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6e98003ce0 con 0x7f6ea0103350 2024-04-25T00:33:05.233 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.229+0000 7f6e9cff9640 1 -- 172.21.15.32:0/974096158 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6e98035070 con 0x7f6ea0103350 2024-04-25T00:33:05.233 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.229+0000 7f6ea5f29640 1 -- 172.21.15.32:0/974096158 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6ea01a1b00 con 0x7f6ea0103350 2024-04-25T00:33:05.233 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.229+0000 7f6ea5f29640 1 -- 172.21.15.32:0/974096158 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f6ea01a2050 con 0x7f6ea0103350 2024-04-25T00:33:05.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.230+0000 7f6ea5f29640 1 -- 172.21.15.32:0/974096158 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6ea0108a60 con 0x7f6ea0103350 2024-04-25T00:33:05.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.232+0000 7f6e9cff9640 1 -- 172.21.15.32:0/974096158 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 18) ==== 137833+0+0 (secure 0 0 0) 0x7f6e98042030 con 0x7f6ea0103350 2024-04-25T00:33:05.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.232+0000 7f6e9cff9640 1 --2- 172.21.15.32:0/974096158 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f6e880a2b90 0x7f6e880a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:05.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.233+0000 7f6e9f7fe640 1 --2- 172.21.15.32:0/974096158 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f6e880a2b90 0x7f6e880a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:05.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.233+0000 7f6e9cff9640 1 -- 172.21.15.32:0/974096158 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7f6e980efd40 con 0x7f6ea0103350 2024-04-25T00:33:05.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.233+0000 7f6e9f7fe640 1 --2- 172.21.15.32:0/974096158 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f6e880a2b90 0x7f6e880a5050 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7f6ea01027f0 tx=0x7f6e900094f0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:05.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.237+0000 7f6e9cff9640 1 -- 172.21.15.32:0/974096158 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f6e980bc990 con 0x7f6ea0103350 2024-04-25T00:33:05.424 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.424+0000 7f6ea5f29640 1 -- 172.21.15.32:0/974096158 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7f6ea00623e0 con 0x7f6ea0103350 2024-04-25T00:33:05.425 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.424+0000 7f6e9cff9640 1 -- 172.21.15.32:0/974096158 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v93) ==== 93+0+10 (secure 0 0 0) 0x7f6e980bc290 con 0x7f6ea0103350 2024-04-25T00:33:05.425 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 1 2024-04-25T00:33:05.427 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.426+0000 7f6ea5f29640 1 -- 172.21.15.32:0/974096158 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f6e880a2b90 msgr2=0x7f6e880a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:05.427 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.426+0000 7f6ea5f29640 1 --2- 172.21.15.32:0/974096158 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f6e880a2b90 0x7f6e880a5050 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7f6ea01027f0 tx=0x7f6e900094f0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.427 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.426+0000 7f6ea5f29640 1 -- 172.21.15.32:0/974096158 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6ea0103350 msgr2=0x7f6ea0074070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:05.427 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.426+0000 7f6ea5f29640 1 --2- 172.21.15.32:0/974096158 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6ea0103350 0x7f6ea0074070 secure :-1 s=READY pgs=405 cs=0 l=1 rev1=1 crypto rx=0x7f6e980314e0 tx=0x7f6e980043c0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.427 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.426+0000 7f6ea5f29640 1 -- 172.21.15.32:0/974096158 shutdown_connections 2024-04-25T00:33:05.428 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.426+0000 7f6ea5f29640 1 --2- 172.21.15.32:0/974096158 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f6e880a2b90 0x7f6e880a5050 unknown :-1 s=CLOSED pgs=247 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.428 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.426+0000 7f6ea5f29640 1 --2- 172.21.15.32:0/974096158 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6ea0103350 0x7f6ea0074070 unknown :-1 s=CLOSED pgs=405 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.428 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.426+0000 7f6ea5f29640 1 --2- 172.21.15.32:0/974096158 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6ea0102990 0x7f6ea0073b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.428 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.426+0000 7f6ea5f29640 1 --2- 172.21.15.32:0/974096158 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6ea01017a0 0x7f6ea00754f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.428 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.426+0000 7f6ea5f29640 1 -- 172.21.15.32:0/974096158 >> 172.21.15.32:0/974096158 conn(0x7f6ea00fcd10 msgr2=0x7f6ea0105c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:05.428 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.426+0000 7f6ea5f29640 1 -- 172.21.15.32:0/974096158 shutdown_connections 2024-04-25T00:33:05.428 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.426+0000 7f6ea5f29640 1 -- 172.21.15.32:0/974096158 wait complete. 2024-04-25T00:33:05.438 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-04-25T00:33:05.545 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.543+0000 7f7858163640 1 -- 172.21.15.32:0/1809218328 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7850101890 msgr2=0x7f7850101d10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:05.546 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.543+0000 7f7858163640 1 --2- 172.21.15.32:0/1809218328 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7850101890 0x7f7850101d10 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 crypto rx=0x7f7840009900 tx=0x7f784002a410 comp rx=0 tx=0).stop 2024-04-25T00:33:05.546 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.543+0000 7f7858163640 1 -- 172.21.15.32:0/1809218328 shutdown_connections 2024-04-25T00:33:05.546 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.543+0000 7f7858163640 1 --2- 172.21.15.32:0/1809218328 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7850102250 0x7f7850108ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.546 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.543+0000 7f7858163640 1 --2- 172.21.15.32:0/1809218328 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7850101890 0x7f7850101d10 unknown :-1 s=CLOSED pgs=406 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.546 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.543+0000 7f7858163640 1 --2- 172.21.15.32:0/1809218328 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f78501006a0 0x7f7850100ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.546 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.543+0000 7f7858163640 1 -- 172.21.15.32:0/1809218328 >> 172.21.15.32:0/1809218328 conn(0x7f78500fbc30 msgr2=0x7f78500fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:05.546 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.543+0000 7f7858163640 1 -- 172.21.15.32:0/1809218328 shutdown_connections 2024-04-25T00:33:05.546 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.543+0000 7f7858163640 1 -- 172.21.15.32:0/1809218328 wait complete. 2024-04-25T00:33:05.546 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.543+0000 7f7858163640 1 Processor -- start 2024-04-25T00:33:05.546 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.544+0000 7f7858163640 1 -- start start 2024-04-25T00:33:05.546 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.544+0000 7f7858163640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f78501006a0 0x7f7850104630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:05.546 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.544+0000 7f7858163640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7850101890 0x7f7850102c70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:05.546 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.544+0000 7f7858163640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7850102250 0x7f78501031b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:05.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.544+0000 7f7858163640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f785010c320 con 0x7f7850102250 2024-04-25T00:33:05.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.544+0000 7f7858163640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f785010c1a0 con 0x7f7850101890 2024-04-25T00:33:05.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.544+0000 7f7858163640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f785010c4a0 con 0x7f78501006a0 2024-04-25T00:33:05.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.544+0000 7f7856700640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7850102250 0x7f78501031b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:05.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.544+0000 7f7856700640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7850102250 0x7f78501031b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:43974/0 (socket says 172.21.15.32:43974) 2024-04-25T00:33:05.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.544+0000 7f7856700640 1 -- 172.21.15.32:0/2825056569 learned_addr learned my addr 172.21.15.32:0/2825056569 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:05.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.544+0000 7f78556fe640 1 --2- 172.21.15.32:0/2825056569 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7850101890 0x7f7850102c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:05.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.544+0000 7f7855eff640 1 --2- 172.21.15.32:0/2825056569 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f78501006a0 0x7f7850104630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:05.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.544+0000 7f7856700640 1 -- 172.21.15.32:0/2825056569 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f78501006a0 msgr2=0x7f7850104630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:05.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.544+0000 7f7856700640 1 --2- 172.21.15.32:0/2825056569 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f78501006a0 0x7f7850104630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.544+0000 7f7856700640 1 -- 172.21.15.32:0/2825056569 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7850101890 msgr2=0x7f7850102c70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:05.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.544+0000 7f7856700640 1 --2- 172.21.15.32:0/2825056569 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7850101890 0x7f7850102c70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.544+0000 7f7856700640 1 -- 172.21.15.32:0/2825056569 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7850103a40 con 0x7f7850102250 2024-04-25T00:33:05.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.544+0000 7f7856700640 1 --2- 172.21.15.32:0/2825056569 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7850102250 0x7f78501031b0 secure :-1 s=READY pgs=407 cs=0 l=1 rev1=1 crypto rx=0x7f784800eaa0 tx=0x7f784800ef60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:05.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.545+0000 7f7846ffd640 1 -- 172.21.15.32:0/2825056569 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f784800cdb0 con 0x7f7850102250 2024-04-25T00:33:05.548 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.545+0000 7f7858163640 1 -- 172.21.15.32:0/2825056569 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f78501a1c90 con 0x7f7850102250 2024-04-25T00:33:05.548 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.545+0000 7f7846ffd640 1 -- 172.21.15.32:0/2825056569 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7848005020 con 0x7f7850102250 2024-04-25T00:33:05.548 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.545+0000 7f7846ffd640 1 -- 172.21.15.32:0/2825056569 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f78480116d0 con 0x7f7850102250 2024-04-25T00:33:05.548 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.545+0000 7f7858163640 1 -- 172.21.15.32:0/2825056569 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f78501a21b0 con 0x7f7850102250 2024-04-25T00:33:05.549 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.546+0000 7f7858163640 1 -- 172.21.15.32:0/2825056569 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f78501a23e0 con 0x7f7850102250 2024-04-25T00:33:05.549 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.547+0000 7f7846ffd640 1 -- 172.21.15.32:0/2825056569 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 18) ==== 137833+0+0 (secure 0 0 0) 0x7f78480054d0 con 0x7f7850102250 2024-04-25T00:33:05.549 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.547+0000 7f7846ffd640 1 --2- 172.21.15.32:0/2825056569 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f783c0a2b90 0x7f783c0a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:05.549 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.547+0000 7f7855eff640 1 --2- 172.21.15.32:0/2825056569 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f783c0a2b90 0x7f783c0a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:05.549 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.547+0000 7f7846ffd640 1 -- 172.21.15.32:0/2825056569 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7f78480cf610 con 0x7f7850102250 2024-04-25T00:33:05.549 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.548+0000 7f7855eff640 1 --2- 172.21.15.32:0/2825056569 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f783c0a2b90 0x7f783c0a5050 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7f784c005d00 tx=0x7f784c005c50 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:05.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.552+0000 7f7846ffd640 1 -- 172.21.15.32:0/2825056569 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f7848016030 con 0x7f7850102250 2024-04-25T00:33:05.737 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.737+0000 7f7858163640 1 -- 172.21.15.32:0/2825056569 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) -- 0x7f78501a2770 con 0x7f7850102250 2024-04-25T00:33:05.738 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.737+0000 7f7846ffd640 1 -- 172.21.15.32:0/2825056569 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v93) ==== 104+0+11 (secure 0 0 0) 0x7f784809c1a0 con 0x7f7850102250 2024-04-25T00:33:05.738 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 32 2024-04-25T00:33:05.740 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.739+0000 7f7858163640 1 -- 172.21.15.32:0/2825056569 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f783c0a2b90 msgr2=0x7f783c0a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:05.740 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.739+0000 7f7858163640 1 --2- 172.21.15.32:0/2825056569 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f783c0a2b90 0x7f783c0a5050 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7f784c005d00 tx=0x7f784c005c50 comp rx=0 tx=0).stop 2024-04-25T00:33:05.740 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.739+0000 7f7858163640 1 -- 172.21.15.32:0/2825056569 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7850102250 msgr2=0x7f78501031b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:05.740 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.739+0000 7f7858163640 1 --2- 172.21.15.32:0/2825056569 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7850102250 0x7f78501031b0 secure :-1 s=READY pgs=407 cs=0 l=1 rev1=1 crypto rx=0x7f784800eaa0 tx=0x7f784800ef60 comp rx=0 tx=0).stop 2024-04-25T00:33:05.740 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.739+0000 7f7858163640 1 -- 172.21.15.32:0/2825056569 shutdown_connections 2024-04-25T00:33:05.740 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.739+0000 7f7858163640 1 --2- 172.21.15.32:0/2825056569 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f783c0a2b90 0x7f783c0a5050 unknown :-1 s=CLOSED pgs=248 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.740 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.739+0000 7f7858163640 1 --2- 172.21.15.32:0/2825056569 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7850102250 0x7f78501031b0 unknown :-1 s=CLOSED pgs=407 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.740 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.739+0000 7f7858163640 1 --2- 172.21.15.32:0/2825056569 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7850101890 0x7f7850102c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.741 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.739+0000 7f7858163640 1 --2- 172.21.15.32:0/2825056569 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f78501006a0 0x7f7850104630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.741 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.739+0000 7f7858163640 1 -- 172.21.15.32:0/2825056569 >> 172.21.15.32:0/2825056569 conn(0x7f78500fbc30 msgr2=0x7f78500fd6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:05.741 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.739+0000 7f7858163640 1 -- 172.21.15.32:0/2825056569 shutdown_connections 2024-04-25T00:33:05.741 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.739+0000 7f7858163640 1 -- 172.21.15.32:0/2825056569 wait complete. 2024-04-25T00:33:05.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:05 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1134507197' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:33:05.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:05 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/974096158' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:33:05.751 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-04-25T00:33:05.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:05 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1134507197' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:33:05.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:05 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/974096158' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:33:05.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.856+0000 7fd60feb3640 1 -- 172.21.15.32:0/1625008888 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd6081006a0 msgr2=0x7fd608100ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:05.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.856+0000 7fd60feb3640 1 --2- 172.21.15.32:0/1625008888 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd6081006a0 0x7fd608100ac0 secure :-1 s=READY pgs=408 cs=0 l=1 rev1=1 crypto rx=0x7fd604009900 tx=0x7fd60402a410 comp rx=0 tx=0).stop 2024-04-25T00:33:05.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.856+0000 7fd60feb3640 1 -- 172.21.15.32:0/1625008888 shutdown_connections 2024-04-25T00:33:05.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.856+0000 7fd60feb3640 1 --2- 172.21.15.32:0/1625008888 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd608102250 0x7fd608108ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.856+0000 7fd60feb3640 1 --2- 172.21.15.32:0/1625008888 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd608101890 0x7fd608101d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.856+0000 7fd60feb3640 1 --2- 172.21.15.32:0/1625008888 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd6081006a0 0x7fd608100ac0 unknown :-1 s=CLOSED pgs=408 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.856+0000 7fd60feb3640 1 -- 172.21.15.32:0/1625008888 >> 172.21.15.32:0/1625008888 conn(0x7fd6080fbc30 msgr2=0x7fd6080fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:05.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.856+0000 7fd60feb3640 1 -- 172.21.15.32:0/1625008888 shutdown_connections 2024-04-25T00:33:05.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.856+0000 7fd60feb3640 1 -- 172.21.15.32:0/1625008888 wait complete. 2024-04-25T00:33:05.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.856+0000 7fd60feb3640 1 Processor -- start 2024-04-25T00:33:05.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.856+0000 7fd60feb3640 1 -- start start 2024-04-25T00:33:05.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.856+0000 7fd60feb3640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd6081006a0 0x7fd608197b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:05.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.856+0000 7fd60feb3640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd608101890 0x7fd608198050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:05.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.856+0000 7fd60feb3640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd608102250 0x7fd60819f0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:05.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.856+0000 7fd60feb3640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fd60810c2f0 con 0x7fd608102250 2024-04-25T00:33:05.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.856+0000 7fd60feb3640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fd60810c170 con 0x7fd608101890 2024-04-25T00:33:05.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.856+0000 7fd60feb3640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fd60810c470 con 0x7fd6081006a0 2024-04-25T00:33:05.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.857+0000 7fd60e450640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd608102250 0x7fd60819f0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:05.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.857+0000 7fd60e450640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd608102250 0x7fd60819f0c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:43990/0 (socket says 172.21.15.32:43990) 2024-04-25T00:33:05.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.857+0000 7fd60e450640 1 -- 172.21.15.32:0/3980147775 learned_addr learned my addr 172.21.15.32:0/3980147775 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:05.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.857+0000 7fd60dc4f640 1 --2- 172.21.15.32:0/3980147775 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd6081006a0 0x7fd608197b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:05.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.857+0000 7fd60d44e640 1 --2- 172.21.15.32:0/3980147775 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd608101890 0x7fd608198050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:05.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.857+0000 7fd60e450640 1 -- 172.21.15.32:0/3980147775 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd6081006a0 msgr2=0x7fd608197b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:05.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.857+0000 7fd60e450640 1 --2- 172.21.15.32:0/3980147775 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd6081006a0 0x7fd608197b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.857+0000 7fd60e450640 1 -- 172.21.15.32:0/3980147775 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd608101890 msgr2=0x7fd608198050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:05.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.857+0000 7fd60e450640 1 --2- 172.21.15.32:0/3980147775 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd608101890 0x7fd608198050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:05.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.857+0000 7fd60e450640 1 -- 172.21.15.32:0/3980147775 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd60819f730 con 0x7fd608102250 2024-04-25T00:33:05.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.857+0000 7fd60d44e640 1 --2- 172.21.15.32:0/3980147775 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd608101890 0x7fd608198050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:05.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.857+0000 7fd60e450640 1 --2- 172.21.15.32:0/3980147775 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd608102250 0x7fd60819f0c0 secure :-1 s=READY pgs=409 cs=0 l=1 rev1=1 crypto rx=0x7fd60000eaa0 tx=0x7fd60000ef60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:05.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.857+0000 7fd5feffd640 1 -- 172.21.15.32:0/3980147775 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd60000cdb0 con 0x7fd608102250 2024-04-25T00:33:05.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.857+0000 7fd5feffd640 1 -- 172.21.15.32:0/3980147775 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd600005020 con 0x7fd608102250 2024-04-25T00:33:05.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.857+0000 7fd60feb3640 1 -- 172.21.15.32:0/3980147775 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd60819fa20 con 0x7fd608102250 2024-04-25T00:33:05.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.857+0000 7fd5feffd640 1 -- 172.21.15.32:0/3980147775 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd6000116b0 con 0x7fd608102250 2024-04-25T00:33:05.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.857+0000 7fd60feb3640 1 -- 172.21.15.32:0/3980147775 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd60819ff70 con 0x7fd608102250 2024-04-25T00:33:05.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.858+0000 7fd60feb3640 1 -- 172.21.15.32:0/3980147775 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd6081a0230 con 0x7fd608102250 2024-04-25T00:33:05.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.859+0000 7fd5feffd640 1 -- 172.21.15.32:0/3980147775 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 18) ==== 137833+0+0 (secure 0 0 0) 0x7fd600005560 con 0x7fd608102250 2024-04-25T00:33:05.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.860+0000 7fd5feffd640 1 --2- 172.21.15.32:0/3980147775 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fd5f40a2c10 0x7fd5f40a50d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:05.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.860+0000 7fd5feffd640 1 -- 172.21.15.32:0/3980147775 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7fd6000cf9b0 con 0x7fd608102250 2024-04-25T00:33:05.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.860+0000 7fd60dc4f640 1 --2- 172.21.15.32:0/3980147775 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fd5f40a2c10 0x7fd5f40a50d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:05.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.861+0000 7fd60dc4f640 1 --2- 172.21.15.32:0/3980147775 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fd5f40a2c10 0x7fd5f40a50d0 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7fd604005ab0 tx=0x7fd604000bc0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:05.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:05.864+0000 7fd5feffd640 1 -- 172.21.15.32:0/3980147775 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fd600016030 con 0x7fd608102250 2024-04-25T00:33:05.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:05 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1134507197' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:33:05.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:05 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/974096158' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:33:06.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.048+0000 7fd60feb3640 1 -- 172.21.15.32:0/3980147775 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) -- 0x7fd6081a0400 con 0x7fd608102250 2024-04-25T00:33:06.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.049+0000 7fd5feffd640 1 -- 172.21.15.32:0/3980147775 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v93) ==== 100+0+11 (secure 0 0 0) 0x7fd60009c5c0 con 0x7fd608102250 2024-04-25T00:33:06.049 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 64 2024-04-25T00:33:06.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.050+0000 7fd60feb3640 1 -- 172.21.15.32:0/3980147775 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fd5f40a2c10 msgr2=0x7fd5f40a50d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:06.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.050+0000 7fd60feb3640 1 --2- 172.21.15.32:0/3980147775 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fd5f40a2c10 0x7fd5f40a50d0 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7fd604005ab0 tx=0x7fd604000bc0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.050+0000 7fd60feb3640 1 -- 172.21.15.32:0/3980147775 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd608102250 msgr2=0x7fd60819f0c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:06.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.050+0000 7fd60feb3640 1 --2- 172.21.15.32:0/3980147775 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd608102250 0x7fd60819f0c0 secure :-1 s=READY pgs=409 cs=0 l=1 rev1=1 crypto rx=0x7fd60000eaa0 tx=0x7fd60000ef60 comp rx=0 tx=0).stop 2024-04-25T00:33:06.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.050+0000 7fd60feb3640 1 -- 172.21.15.32:0/3980147775 shutdown_connections 2024-04-25T00:33:06.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.050+0000 7fd60feb3640 1 --2- 172.21.15.32:0/3980147775 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fd5f40a2c10 0x7fd5f40a50d0 unknown :-1 s=CLOSED pgs=249 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.051+0000 7fd60feb3640 1 --2- 172.21.15.32:0/3980147775 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd608102250 0x7fd60819f0c0 unknown :-1 s=CLOSED pgs=409 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.051+0000 7fd60feb3640 1 --2- 172.21.15.32:0/3980147775 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd608101890 0x7fd608198050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.051+0000 7fd60feb3640 1 --2- 172.21.15.32:0/3980147775 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd6081006a0 0x7fd608197b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.051+0000 7fd60feb3640 1 -- 172.21.15.32:0/3980147775 >> 172.21.15.32:0/3980147775 conn(0x7fd6080fbc30 msgr2=0x7fd6080fd700 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:06.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.051+0000 7fd60feb3640 1 -- 172.21.15.32:0/3980147775 shutdown_connections 2024-04-25T00:33:06.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.051+0000 7fd60feb3640 1 -- 172.21.15.32:0/3980147775 wait complete. 2024-04-25T00:33:06.063 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data_ec pg_num 2024-04-25T00:33:06.172 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.169+0000 7f8a75612640 1 -- 172.21.15.32:0/2242466986 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8a70101880 msgr2=0x7f8a70101d00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:06.172 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.169+0000 7f8a75612640 1 --2- 172.21.15.32:0/2242466986 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8a70101880 0x7f8a70101d00 secure :-1 s=READY pgs=410 cs=0 l=1 rev1=1 crypto rx=0x7f8a60009900 tx=0x7f8a6002a410 comp rx=0 tx=0).stop 2024-04-25T00:33:06.172 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.169+0000 7f8a75612640 1 -- 172.21.15.32:0/2242466986 shutdown_connections 2024-04-25T00:33:06.172 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.169+0000 7f8a75612640 1 --2- 172.21.15.32:0/2242466986 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8a70102240 0x7f8a70108ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.169+0000 7f8a75612640 1 --2- 172.21.15.32:0/2242466986 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8a70101880 0x7f8a70101d00 unknown :-1 s=CLOSED pgs=410 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.169+0000 7f8a75612640 1 --2- 172.21.15.32:0/2242466986 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8a70100690 0x7f8a70100ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.169+0000 7f8a75612640 1 -- 172.21.15.32:0/2242466986 >> 172.21.15.32:0/2242466986 conn(0x7f8a700fbc00 msgr2=0x7f8a700fe040 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:06.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.169+0000 7f8a75612640 1 -- 172.21.15.32:0/2242466986 shutdown_connections 2024-04-25T00:33:06.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.169+0000 7f8a75612640 1 -- 172.21.15.32:0/2242466986 wait complete. 2024-04-25T00:33:06.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.170+0000 7f8a75612640 1 Processor -- start 2024-04-25T00:33:06.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.170+0000 7f8a75612640 1 -- start start 2024-04-25T00:33:06.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.170+0000 7f8a75612640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8a70100690 0x7f8a70104620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:06.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.170+0000 7f8a75612640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8a70101880 0x7f8a70102c60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:06.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.170+0000 7f8a75612640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8a70102240 0x7f8a701031a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:06.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.170+0000 7f8a75612640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f8a7010c340 con 0x7f8a70102240 2024-04-25T00:33:06.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.170+0000 7f8a75612640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f8a7010c1c0 con 0x7f8a70101880 2024-04-25T00:33:06.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.170+0000 7f8a75612640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f8a7010c4c0 con 0x7f8a70100690 2024-04-25T00:33:06.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.170+0000 7f8a6f7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8a70102240 0x7f8a701031a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:06.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.170+0000 7f8a6f7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8a70102240 0x7f8a701031a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44000/0 (socket says 172.21.15.32:44000) 2024-04-25T00:33:06.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.170+0000 7f8a6f7fe640 1 -- 172.21.15.32:0/4106422203 learned_addr learned my addr 172.21.15.32:0/4106422203 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:06.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.170+0000 7f8a6e7fc640 1 --2- 172.21.15.32:0/4106422203 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8a70101880 0x7f8a70102c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:06.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.170+0000 7f8a6effd640 1 --2- 172.21.15.32:0/4106422203 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8a70100690 0x7f8a70104620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:06.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.170+0000 7f8a6f7fe640 1 -- 172.21.15.32:0/4106422203 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8a70100690 msgr2=0x7f8a70104620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:06.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.171+0000 7f8a6f7fe640 1 --2- 172.21.15.32:0/4106422203 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8a70100690 0x7f8a70104620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.171+0000 7f8a6f7fe640 1 -- 172.21.15.32:0/4106422203 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8a70101880 msgr2=0x7f8a70102c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:06.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.171+0000 7f8a6f7fe640 1 --2- 172.21.15.32:0/4106422203 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8a70101880 0x7f8a70102c60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.171+0000 7f8a6f7fe640 1 -- 172.21.15.32:0/4106422203 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8a70103a30 con 0x7f8a70102240 2024-04-25T00:33:06.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.171+0000 7f8a6f7fe640 1 --2- 172.21.15.32:0/4106422203 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8a70102240 0x7f8a701031a0 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 crypto rx=0x7f8a6800eaa0 tx=0x7f8a6800ef60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:06.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.171+0000 7f8a57fff640 1 -- 172.21.15.32:0/4106422203 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8a6800cdb0 con 0x7f8a70102240 2024-04-25T00:33:06.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.171+0000 7f8a57fff640 1 -- 172.21.15.32:0/4106422203 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8a68005020 con 0x7f8a70102240 2024-04-25T00:33:06.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.171+0000 7f8a75612640 1 -- 172.21.15.32:0/4106422203 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8a701a1bf0 con 0x7f8a70102240 2024-04-25T00:33:06.175 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.171+0000 7f8a57fff640 1 -- 172.21.15.32:0/4106422203 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8a680116b0 con 0x7f8a70102240 2024-04-25T00:33:06.175 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.171+0000 7f8a75612640 1 -- 172.21.15.32:0/4106422203 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8a701a20c0 con 0x7f8a70102240 2024-04-25T00:33:06.175 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.172+0000 7f8a75612640 1 -- 172.21.15.32:0/4106422203 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8a70067cd0 con 0x7f8a70102240 2024-04-25T00:33:06.175 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.173+0000 7f8a57fff640 1 -- 172.21.15.32:0/4106422203 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 18) ==== 137833+0+0 (secure 0 0 0) 0x7f8a68005560 con 0x7f8a70102240 2024-04-25T00:33:06.176 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.174+0000 7f8a57fff640 1 --2- 172.21.15.32:0/4106422203 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8a580a2b90 0x7f8a580a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:06.176 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.174+0000 7f8a57fff640 1 -- 172.21.15.32:0/4106422203 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7f8a680cf7b0 con 0x7f8a70102240 2024-04-25T00:33:06.176 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.174+0000 7f8a6effd640 1 --2- 172.21.15.32:0/4106422203 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8a580a2b90 0x7f8a580a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:06.176 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.174+0000 7f8a6effd640 1 --2- 172.21.15.32:0/4106422203 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8a580a2b90 0x7f8a580a5050 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7f8a640058c0 tx=0x7f8a64005850 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:06.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.178+0000 7f8a57fff640 1 -- 172.21.15.32:0/4106422203 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f8a68016030 con 0x7f8a70102240 2024-04-25T00:33:06.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.364+0000 7f8a75612640 1 -- 172.21.15.32:0/4106422203 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"} v 0) -- 0x7f8a701a2a80 con 0x7f8a70102240 2024-04-25T00:33:06.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.365+0000 7f8a57fff640 1 -- 172.21.15.32:0/4106422203 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]=0 v93) ==== 103+0+11 (secure 0 0 0) 0x7f8a6809c340 con 0x7f8a70102240 2024-04-25T00:33:06.365 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 32 2024-04-25T00:33:06.367 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.367+0000 7f8a75612640 1 -- 172.21.15.32:0/4106422203 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8a580a2b90 msgr2=0x7f8a580a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:06.367 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.367+0000 7f8a75612640 1 --2- 172.21.15.32:0/4106422203 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8a580a2b90 0x7f8a580a5050 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7f8a640058c0 tx=0x7f8a64005850 comp rx=0 tx=0).stop 2024-04-25T00:33:06.367 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.367+0000 7f8a75612640 1 -- 172.21.15.32:0/4106422203 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8a70102240 msgr2=0x7f8a701031a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:06.367 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.367+0000 7f8a75612640 1 --2- 172.21.15.32:0/4106422203 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8a70102240 0x7f8a701031a0 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 crypto rx=0x7f8a6800eaa0 tx=0x7f8a6800ef60 comp rx=0 tx=0).stop 2024-04-25T00:33:06.368 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.367+0000 7f8a75612640 1 -- 172.21.15.32:0/4106422203 shutdown_connections 2024-04-25T00:33:06.368 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.367+0000 7f8a75612640 1 --2- 172.21.15.32:0/4106422203 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8a580a2b90 0x7f8a580a5050 unknown :-1 s=CLOSED pgs=250 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.368 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.367+0000 7f8a75612640 1 --2- 172.21.15.32:0/4106422203 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8a70102240 0x7f8a701031a0 unknown :-1 s=CLOSED pgs=411 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.368 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.367+0000 7f8a75612640 1 --2- 172.21.15.32:0/4106422203 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8a70101880 0x7f8a70102c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.368 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.367+0000 7f8a75612640 1 --2- 172.21.15.32:0/4106422203 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8a70100690 0x7f8a70104620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.368 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.367+0000 7f8a75612640 1 -- 172.21.15.32:0/4106422203 >> 172.21.15.32:0/4106422203 conn(0x7f8a700fbc00 msgr2=0x7f8a700fd6b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:06.368 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.367+0000 7f8a75612640 1 -- 172.21.15.32:0/4106422203 shutdown_connections 2024-04-25T00:33:06.368 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.367+0000 7f8a75612640 1 -- 172.21.15.32:0/4106422203 wait complete. 2024-04-25T00:33:06.379 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:33:06.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.482+0000 7f0b73da9640 1 -- 172.21.15.32:0/1462044955 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0b6c102220 msgr2=0x7f0b6c104610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:06.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.482+0000 7f0b73da9640 1 --2- 172.21.15.32:0/1462044955 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0b6c102220 0x7f0b6c104610 secure :-1 s=READY pgs=412 cs=0 l=1 rev1=1 crypto rx=0x7f0b68009900 tx=0x7f0b6802a410 comp rx=0 tx=0).stop 2024-04-25T00:33:06.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.482+0000 7f0b73da9640 1 -- 172.21.15.32:0/1462044955 shutdown_connections 2024-04-25T00:33:06.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.482+0000 7f0b73da9640 1 --2- 172.21.15.32:0/1462044955 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0b6c107480 0x7f0b6c1098d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.482+0000 7f0b73da9640 1 --2- 172.21.15.32:0/1462044955 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0b6c104b50 0x7f0b6c106f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.482+0000 7f0b73da9640 1 --2- 172.21.15.32:0/1462044955 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0b6c102220 0x7f0b6c104610 unknown :-1 s=CLOSED pgs=412 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.482+0000 7f0b73da9640 1 -- 172.21.15.32:0/1462044955 >> 172.21.15.32:0/1462044955 conn(0x7f0b6c0fbc30 msgr2=0x7f0b6c0fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:06.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.483+0000 7f0b73da9640 1 -- 172.21.15.32:0/1462044955 shutdown_connections 2024-04-25T00:33:06.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.483+0000 7f0b73da9640 1 -- 172.21.15.32:0/1462044955 wait complete. 2024-04-25T00:33:06.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.483+0000 7f0b73da9640 1 Processor -- start 2024-04-25T00:33:06.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.483+0000 7f0b73da9640 1 -- start start 2024-04-25T00:33:06.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.483+0000 7f0b73da9640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0b6c102220 0x7f0b6c101b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:06.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.483+0000 7f0b73da9640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0b6c104b50 0x7f0b6c1001d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:06.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.483+0000 7f0b73da9640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0b6c107480 0x7f0b6c100710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:06.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.483+0000 7f0b73da9640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f0b6c10d450 con 0x7f0b6c102220 2024-04-25T00:33:06.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.483+0000 7f0b73da9640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f0b6c10d2d0 con 0x7f0b6c104b50 2024-04-25T00:33:06.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.483+0000 7f0b73da9640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f0b6c10d5d0 con 0x7f0b6c107480 2024-04-25T00:33:06.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.483+0000 7f0b71b45640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0b6c102220 0x7f0b6c101b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:06.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.483+0000 7f0b71b45640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0b6c102220 0x7f0b6c101b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44026/0 (socket says 172.21.15.32:44026) 2024-04-25T00:33:06.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.483+0000 7f0b71b45640 1 -- 172.21.15.32:0/4192571043 learned_addr learned my addr 172.21.15.32:0/4192571043 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:06.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.484+0000 7f0b71344640 1 --2- 172.21.15.32:0/4192571043 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0b6c104b50 0x7f0b6c1001d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:06.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.484+0000 7f0b71b45640 1 -- 172.21.15.32:0/4192571043 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0b6c107480 msgr2=0x7f0b6c100710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:06.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.484+0000 7f0b72346640 1 --2- 172.21.15.32:0/4192571043 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0b6c107480 0x7f0b6c100710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:06.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.484+0000 7f0b71b45640 1 --2- 172.21.15.32:0/4192571043 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0b6c107480 0x7f0b6c100710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.484+0000 7f0b71b45640 1 -- 172.21.15.32:0/4192571043 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0b6c104b50 msgr2=0x7f0b6c1001d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:06.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.484+0000 7f0b71b45640 1 --2- 172.21.15.32:0/4192571043 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0b6c104b50 0x7f0b6c1001d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.484+0000 7f0b71b45640 1 -- 172.21.15.32:0/4192571043 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0b6c100fa0 con 0x7f0b6c102220 2024-04-25T00:33:06.489 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.484+0000 7f0b71344640 1 --2- 172.21.15.32:0/4192571043 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0b6c104b50 0x7f0b6c1001d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:33:06.489 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.484+0000 7f0b72346640 1 --2- 172.21.15.32:0/4192571043 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0b6c107480 0x7f0b6c100710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:33:06.489 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.484+0000 7f0b71b45640 1 --2- 172.21.15.32:0/4192571043 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0b6c102220 0x7f0b6c101b90 secure :-1 s=READY pgs=413 cs=0 l=1 rev1=1 crypto rx=0x7f0b68009230 tx=0x7f0b68004360 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:06.489 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.484+0000 7f0b62ffd640 1 -- 172.21.15.32:0/4192571043 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b6803f030 con 0x7f0b6c102220 2024-04-25T00:33:06.489 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.484+0000 7f0b62ffd640 1 -- 172.21.15.32:0/4192571043 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0b68003ce0 con 0x7f0b6c102220 2024-04-25T00:33:06.489 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.484+0000 7f0b73da9640 1 -- 172.21.15.32:0/4192571043 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0b6c1a1be0 con 0x7f0b6c102220 2024-04-25T00:33:06.489 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.484+0000 7f0b62ffd640 1 -- 172.21.15.32:0/4192571043 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b68035070 con 0x7f0b6c102220 2024-04-25T00:33:06.489 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.484+0000 7f0b73da9640 1 -- 172.21.15.32:0/4192571043 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0b6c1a2130 con 0x7f0b6c102220 2024-04-25T00:33:06.490 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.485+0000 7f0b73da9640 1 -- 172.21.15.32:0/4192571043 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0b6c0fd720 con 0x7f0b6c102220 2024-04-25T00:33:06.490 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.487+0000 7f0b62ffd640 1 -- 172.21.15.32:0/4192571043 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 18) ==== 137833+0+0 (secure 0 0 0) 0x7f0b68042030 con 0x7f0b6c102220 2024-04-25T00:33:06.490 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.487+0000 7f0b62ffd640 1 --2- 172.21.15.32:0/4192571043 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f0b580a2b90 0x7f0b580a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:06.490 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.488+0000 7f0b71344640 1 --2- 172.21.15.32:0/4192571043 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f0b580a2b90 0x7f0b580a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:06.490 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.488+0000 7f0b62ffd640 1 -- 172.21.15.32:0/4192571043 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7f0b680efeb0 con 0x7f0b6c102220 2024-04-25T00:33:06.490 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.488+0000 7f0b71344640 1 --2- 172.21.15.32:0/4192571043 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f0b580a2b90 0x7f0b580a5050 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f0b6c101900 tx=0x7f0b5c005e50 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:06.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.492+0000 7f0b62ffd640 1 -- 172.21.15.32:0/4192571043 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f0b680bca40 con 0x7f0b6c102220 2024-04-25T00:33:06.720 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.719+0000 7f0b73da9640 1 -- 172.21.15.32:0/4192571043 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f0b6c062370 con 0x7f0b6c102220 2024-04-25T00:33:06.720 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.720+0000 7f0b62ffd640 1 -- 172.21.15.32:0/4192571043 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) ==== 94+0+11876 (secure 0 0 0) 0x7f0b680bc340 con 0x7f0b6c102220 2024-04-25T00:33:06.721 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:33:06.721 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14643,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6839/187492674","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6838","nonce":187492674},{"type":"v1","addr":"172.21.15.32:6839","nonce":187492674}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14655,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6841/1948785545","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6840","nonce":1948785545},{"type":"v1","addr":"172.21.15.32:6841","nonce":1948785545}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-25T00:30:47.866875+0000","modified":"2024-04-25T00:31:14.972514+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24506,"mds_1":24481,"mds_2":24476},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24443":{"gid":24443,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24445":{"gid":24445,"name":"f","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24457":{"gid":24457,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24476":{"gid":24476,"name":"h","rank":2,"incarnation":22,"state":"up:active","state_seq":12,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24481":{"gid":24481,"name":"i","rank":1,"incarnation":20,"state":"up:active","state_seq":11,"addr":"172.21.15.130:6839/4267125809","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":4267125809},{"type":"v1","addr":"172.21.15.130:6839","nonce":4267125809}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24506":{"gid":24506,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.121:6839/258444853","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6838","nonce":258444853},{"type":"v1","addr":"172.21.15.121:6839","nonce":258444853}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24506,"qdb_cluster":[24476,24481,24506]},"id":1}]} 2024-04-25T00:33:06.721 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 24 2024-04-25T00:33:06.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.722+0000 7f0b73da9640 1 -- 172.21.15.32:0/4192571043 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f0b580a2b90 msgr2=0x7f0b580a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:06.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.722+0000 7f0b73da9640 1 --2- 172.21.15.32:0/4192571043 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f0b580a2b90 0x7f0b580a5050 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f0b6c101900 tx=0x7f0b5c005e50 comp rx=0 tx=0).stop 2024-04-25T00:33:06.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.722+0000 7f0b73da9640 1 -- 172.21.15.32:0/4192571043 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0b6c102220 msgr2=0x7f0b6c101b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:06.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.722+0000 7f0b73da9640 1 --2- 172.21.15.32:0/4192571043 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0b6c102220 0x7f0b6c101b90 secure :-1 s=READY pgs=413 cs=0 l=1 rev1=1 crypto rx=0x7f0b68009230 tx=0x7f0b68004360 comp rx=0 tx=0).stop 2024-04-25T00:33:06.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.723+0000 7f0b73da9640 1 -- 172.21.15.32:0/4192571043 shutdown_connections 2024-04-25T00:33:06.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.723+0000 7f0b73da9640 1 --2- 172.21.15.32:0/4192571043 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f0b580a2b90 0x7f0b580a5050 unknown :-1 s=CLOSED pgs=251 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.723+0000 7f0b73da9640 1 --2- 172.21.15.32:0/4192571043 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0b6c107480 0x7f0b6c100710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.723+0000 7f0b73da9640 1 --2- 172.21.15.32:0/4192571043 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0b6c104b50 0x7f0b6c1001d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.723+0000 7f0b73da9640 1 --2- 172.21.15.32:0/4192571043 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0b6c102220 0x7f0b6c101b90 unknown :-1 s=CLOSED pgs=413 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.723+0000 7f0b73da9640 1 -- 172.21.15.32:0/4192571043 >> 172.21.15.32:0/4192571043 conn(0x7f0b6c0fbc30 msgr2=0x7f0b6c102c40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:06.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.723+0000 7f0b73da9640 1 -- 172.21.15.32:0/4192571043 shutdown_connections 2024-04-25T00:33:06.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.723+0000 7f0b73da9640 1 -- 172.21.15.32:0/4192571043 wait complete. 2024-04-25T00:33:06.733 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:06 smithi032 ceph-mon[29357]: pgmap v364: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-25T00:33:06.733 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:06 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2825056569' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:33:06.733 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:06 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3980147775' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:33:06.733 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:06 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/4106422203' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:33:06.735 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-25T00:33:06.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:06 smithi121 ceph-mon[32231]: pgmap v364: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-25T00:33:06.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:06 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2825056569' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:33:06.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:06 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3980147775' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:33:06.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:06 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/4106422203' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:33:06.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.838+0000 7fc03a512640 1 -- 172.21.15.32:0/344039266 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc034102230 msgr2=0x7fc034108ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:06.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.838+0000 7fc03a512640 1 --2- 172.21.15.32:0/344039266 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc034102230 0x7fc034108ac0 secure :-1 s=READY pgs=414 cs=0 l=1 rev1=1 crypto rx=0x7fc02c009ca0 tx=0x7fc02c02a8c0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.838+0000 7fc03a512640 1 -- 172.21.15.32:0/344039266 shutdown_connections 2024-04-25T00:33:06.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.838+0000 7fc03a512640 1 --2- 172.21.15.32:0/344039266 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc034102230 0x7fc034108ac0 unknown :-1 s=CLOSED pgs=414 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.838+0000 7fc03a512640 1 --2- 172.21.15.32:0/344039266 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc034101870 0x7fc034101cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.838+0000 7fc03a512640 1 --2- 172.21.15.32:0/344039266 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc034100680 0x7fc034100aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.838+0000 7fc03a512640 1 -- 172.21.15.32:0/344039266 >> 172.21.15.32:0/344039266 conn(0x7fc0340fbbf0 msgr2=0x7fc0340fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:06.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.838+0000 7fc03a512640 1 -- 172.21.15.32:0/344039266 shutdown_connections 2024-04-25T00:33:06.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.838+0000 7fc03a512640 1 -- 172.21.15.32:0/344039266 wait complete. 2024-04-25T00:33:06.842 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.839+0000 7fc03a512640 1 Processor -- start 2024-04-25T00:33:06.842 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.839+0000 7fc03a512640 1 -- start start 2024-04-25T00:33:06.842 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.839+0000 7fc03a512640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc034100680 0x7fc034199c90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:06.842 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.839+0000 7fc03a512640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc034101870 0x7fc03419a1d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:06.842 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.839+0000 7fc03a512640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc034102230 0x7fc0341a1230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:06.842 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.839+0000 7fc03a512640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fc03410c300 con 0x7fc034102230 2024-04-25T00:33:06.842 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.839+0000 7fc03a512640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fc03410c180 con 0x7fc034100680 2024-04-25T00:33:06.842 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.839+0000 7fc03a512640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fc03410c480 con 0x7fc034101870 2024-04-25T00:33:06.842 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.840+0000 7fc038aaf640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc034102230 0x7fc0341a1230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:06.842 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.840+0000 7fc038aaf640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc034102230 0x7fc0341a1230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44040/0 (socket says 172.21.15.32:44040) 2024-04-25T00:33:06.842 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.840+0000 7fc038aaf640 1 -- 172.21.15.32:0/679940316 learned_addr learned my addr 172.21.15.32:0/679940316 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:06.842 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.840+0000 7fc0337fe640 1 --2- 172.21.15.32:0/679940316 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc034101870 0x7fc03419a1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:06.842 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.840+0000 7fc033fff640 1 --2- 172.21.15.32:0/679940316 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc034100680 0x7fc034199c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:06.843 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.840+0000 7fc038aaf640 1 -- 172.21.15.32:0/679940316 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc034101870 msgr2=0x7fc03419a1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:06.843 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.840+0000 7fc038aaf640 1 --2- 172.21.15.32:0/679940316 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc034101870 0x7fc03419a1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.843 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.840+0000 7fc038aaf640 1 -- 172.21.15.32:0/679940316 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc034100680 msgr2=0x7fc034199c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:06.843 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.840+0000 7fc038aaf640 1 --2- 172.21.15.32:0/679940316 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc034100680 0x7fc034199c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:06.843 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.840+0000 7fc038aaf640 1 -- 172.21.15.32:0/679940316 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc0341a18d0 con 0x7fc034102230 2024-04-25T00:33:06.843 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.840+0000 7fc038aaf640 1 --2- 172.21.15.32:0/679940316 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc034102230 0x7fc0341a1230 secure :-1 s=READY pgs=415 cs=0 l=1 rev1=1 crypto rx=0x7fc02c031ae0 tx=0x7fc02c031ee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:06.843 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.840+0000 7fc0317fa640 1 -- 172.21.15.32:0/679940316 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc02c003c30 con 0x7fc034102230 2024-04-25T00:33:06.843 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.840+0000 7fc03a512640 1 -- 172.21.15.32:0/679940316 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc0341a1b60 con 0x7fc034102230 2024-04-25T00:33:06.843 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.840+0000 7fc0317fa640 1 -- 172.21.15.32:0/679940316 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc02c003dd0 con 0x7fc034102230 2024-04-25T00:33:06.843 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.840+0000 7fc0317fa640 1 -- 172.21.15.32:0/679940316 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc02c02fcb0 con 0x7fc034102230 2024-04-25T00:33:06.843 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.840+0000 7fc03a512640 1 -- 172.21.15.32:0/679940316 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc0341a2100 con 0x7fc034102230 2024-04-25T00:33:06.844 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.841+0000 7fc03a512640 1 -- 172.21.15.32:0/679940316 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc0341a2390 con 0x7fc034102230 2024-04-25T00:33:06.844 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.843+0000 7fc0317fa640 1 -- 172.21.15.32:0/679940316 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 18) ==== 137833+0+0 (secure 0 0 0) 0x7fc02c004050 con 0x7fc034102230 2024-04-25T00:33:06.844 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.843+0000 7fc0317fa640 1 --2- 172.21.15.32:0/679940316 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc01c0a2b90 0x7fc01c0a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:06.844 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.843+0000 7fc033fff640 1 --2- 172.21.15.32:0/679940316 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc01c0a2b90 0x7fc01c0a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:06.844 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.843+0000 7fc0317fa640 1 -- 172.21.15.32:0/679940316 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7fc02c0f0860 con 0x7fc034102230 2024-04-25T00:33:06.845 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.844+0000 7fc033fff640 1 --2- 172.21.15.32:0/679940316 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc01c0a2b90 0x7fc01c0a5050 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7fc03419ab60 tx=0x7fc024005f90 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:06.847 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:06.847+0000 7fc0317fa640 1 -- 172.21.15.32:0/679940316 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fc02c0bd3f0 con 0x7fc034102230 2024-04-25T00:33:06.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:06 smithi130 ceph-mon[32525]: pgmap v364: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-25T00:33:06.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:06 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2825056569' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:33:06.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:06 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3980147775' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:33:06.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:06 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/4106422203' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:33:07.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.034+0000 7fc03a512640 1 -- 172.21.15.32:0/679940316 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fc0341a2560 con 0x7fc034102230 2024-04-25T00:33:07.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.036+0000 7fc0317fa640 1 -- 172.21.15.32:0/679940316 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v93) ==== 74+0+24843 (secure 0 0 0) 0x7fc02c0bccf0 con 0x7fc034102230 2024-04-25T00:33:07.037 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:33:07.037 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":93,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:32:45.458564+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-25T00:30:46.239556+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-25T00:30:47.257182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2024-04-25T00:30:51.193975+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.b","mappings":[{"from":8,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":1}]},{"pgid":"2.f","mappings":[{"from":8,"to":11}]},{"pgid":"2.10","mappings":[{"from":2,"to":4}]},{"pgid":"3.6","mappings":[{"from":3,"to":6}]},{"pgid":"3.17","mappings":[{"from":4,"to":7}]},{"pgid":"3.1b","mappings":[{"from":5,"to":9}]},{"pgid":"3.20","mappings":[{"from":5,"to":9}]},{"pgid":"3.23","mappings":[{"from":4,"to":8}]},{"pgid":"3.2c","mappings":[{"from":4,"to":9}]},{"pgid":"4.c","mappings":[{"from":9,"to":10}]},{"pgid":"4.12","mappings":[{"from":9,"to":10}]},{"pgid":"4.1c","mappings":[{"from":9,"to":8}]},{"pgid":"4.1d","mappings":[{"from":5,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-num-failure-domains":"0","crush-osds-per-failure-domain":"0","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:33:07.039 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.038+0000 7fc03a512640 1 -- 172.21.15.32:0/679940316 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc01c0a2b90 msgr2=0x7fc01c0a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:07.039 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.038+0000 7fc03a512640 1 --2- 172.21.15.32:0/679940316 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc01c0a2b90 0x7fc01c0a5050 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7fc03419ab60 tx=0x7fc024005f90 comp rx=0 tx=0).stop 2024-04-25T00:33:07.039 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.038+0000 7fc03a512640 1 -- 172.21.15.32:0/679940316 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc034102230 msgr2=0x7fc0341a1230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:07.039 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.038+0000 7fc03a512640 1 --2- 172.21.15.32:0/679940316 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc034102230 0x7fc0341a1230 secure :-1 s=READY pgs=415 cs=0 l=1 rev1=1 crypto rx=0x7fc02c031ae0 tx=0x7fc02c031ee0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.039+0000 7fc03a512640 1 -- 172.21.15.32:0/679940316 shutdown_connections 2024-04-25T00:33:07.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.039+0000 7fc03a512640 1 --2- 172.21.15.32:0/679940316 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fc01c0a2b90 0x7fc01c0a5050 unknown :-1 s=CLOSED pgs=252 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.039+0000 7fc03a512640 1 --2- 172.21.15.32:0/679940316 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc034102230 0x7fc0341a1230 unknown :-1 s=CLOSED pgs=415 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.039+0000 7fc03a512640 1 --2- 172.21.15.32:0/679940316 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc034101870 0x7fc03419a1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.039+0000 7fc03a512640 1 --2- 172.21.15.32:0/679940316 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc034100680 0x7fc034199c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.039+0000 7fc03a512640 1 -- 172.21.15.32:0/679940316 >> 172.21.15.32:0/679940316 conn(0x7fc0340fbbf0 msgr2=0x7fc0340fd660 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:07.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.039+0000 7fc03a512640 1 -- 172.21.15.32:0/679940316 shutdown_connections 2024-04-25T00:33:07.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.039+0000 7fc03a512640 1 -- 172.21.15.32:0/679940316 wait complete. 2024-04-25T00:33:07.050 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-25T00:26:07.306839+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-04-25T00:30:46.239556+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '84', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.8799999952316284, 'score_stable': 1.8799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.8799999952316284, 'raw_score_stable': 1.8799999952316284, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-04-25T00:30:47.257182+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '85', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.05000000074505806}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_data_ec', 'create_time': '2024-04-25T00:30:51.193975+0000', 'flags': 5, 'flags_names': 'hashpspool,ec_overwrites', 'type': 3, 'size': 4, 'min_size': 3, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '89', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'cephfs_data_ec', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 8192, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}] 2024-04-25T00:33:07.051 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-25T00:33:07.159 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.156+0000 7f741fa2a640 1 -- 172.21.15.32:0/2343560355 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7418100680 msgr2=0x7f7418100aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:07.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.156+0000 7f741fa2a640 1 --2- 172.21.15.32:0/2343560355 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7418100680 0x7f7418100aa0 secure :-1 s=READY pgs=416 cs=0 l=1 rev1=1 crypto rx=0x7f7408009900 tx=0x7f740802a410 comp rx=0 tx=0).stop 2024-04-25T00:33:07.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.157+0000 7f741fa2a640 1 -- 172.21.15.32:0/2343560355 shutdown_connections 2024-04-25T00:33:07.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.157+0000 7f741fa2a640 1 --2- 172.21.15.32:0/2343560355 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7418102230 0x7f7418108ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.157+0000 7f741fa2a640 1 --2- 172.21.15.32:0/2343560355 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7418101870 0x7f7418101cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.157+0000 7f741fa2a640 1 --2- 172.21.15.32:0/2343560355 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7418100680 0x7f7418100aa0 unknown :-1 s=CLOSED pgs=416 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.157+0000 7f741fa2a640 1 -- 172.21.15.32:0/2343560355 >> 172.21.15.32:0/2343560355 conn(0x7f74180fbbf0 msgr2=0x7f74180fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:07.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.157+0000 7f741fa2a640 1 -- 172.21.15.32:0/2343560355 shutdown_connections 2024-04-25T00:33:07.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.157+0000 7f741fa2a640 1 -- 172.21.15.32:0/2343560355 wait complete. 2024-04-25T00:33:07.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.157+0000 7f741fa2a640 1 Processor -- start 2024-04-25T00:33:07.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.157+0000 7f741fa2a640 1 -- start start 2024-04-25T00:33:07.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.158+0000 7f741fa2a640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7418100680 0x7f7418197b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:07.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.158+0000 7f741fa2a640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7418101870 0x7f7418198060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:07.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.158+0000 7f741fa2a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7418102230 0x7f741819f0d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:07.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.158+0000 7f741fa2a640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f741810c300 con 0x7f7418102230 2024-04-25T00:33:07.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.158+0000 7f741fa2a640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f741810c180 con 0x7f7418101870 2024-04-25T00:33:07.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.158+0000 7f741fa2a640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f741810c480 con 0x7f7418100680 2024-04-25T00:33:07.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.158+0000 7f741dfc7640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7418102230 0x7f741819f0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:07.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.158+0000 7f741dfc7640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7418102230 0x7f741819f0d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44058/0 (socket says 172.21.15.32:44058) 2024-04-25T00:33:07.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.158+0000 7f741dfc7640 1 -- 172.21.15.32:0/2255983158 learned_addr learned my addr 172.21.15.32:0/2255983158 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:07.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.158+0000 7f741cfc5640 1 --2- 172.21.15.32:0/2255983158 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7418101870 0x7f7418198060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:07.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.158+0000 7f741d7c6640 1 --2- 172.21.15.32:0/2255983158 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7418100680 0x7f7418197b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:07.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.158+0000 7f741dfc7640 1 -- 172.21.15.32:0/2255983158 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7418100680 msgr2=0x7f7418197b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:07.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.158+0000 7f741dfc7640 1 --2- 172.21.15.32:0/2255983158 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7418100680 0x7f7418197b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.158+0000 7f741dfc7640 1 -- 172.21.15.32:0/2255983158 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7418101870 msgr2=0x7f7418198060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:07.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.158+0000 7f741dfc7640 1 --2- 172.21.15.32:0/2255983158 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7418101870 0x7f7418198060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.158+0000 7f741dfc7640 1 -- 172.21.15.32:0/2255983158 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f741819f740 con 0x7f7418102230 2024-04-25T00:33:07.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.158+0000 7f741dfc7640 1 --2- 172.21.15.32:0/2255983158 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7418102230 0x7f741819f0d0 secure :-1 s=READY pgs=417 cs=0 l=1 rev1=1 crypto rx=0x7f7410009fd0 tx=0x7f741000b260 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:07.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.159+0000 7f740e7fc640 1 -- 172.21.15.32:0/2255983158 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7410017070 con 0x7f7418102230 2024-04-25T00:33:07.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.159+0000 7f741fa2a640 1 -- 172.21.15.32:0/2255983158 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f741819fa30 con 0x7f7418102230 2024-04-25T00:33:07.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.159+0000 7f740e7fc640 1 -- 172.21.15.32:0/2255983158 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7410003e50 con 0x7f7418102230 2024-04-25T00:33:07.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.159+0000 7f740e7fc640 1 -- 172.21.15.32:0/2255983158 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f74100106c0 con 0x7f7418102230 2024-04-25T00:33:07.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.159+0000 7f741fa2a640 1 -- 172.21.15.32:0/2255983158 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f741819ff80 con 0x7f7418102230 2024-04-25T00:33:07.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.160+0000 7f741fa2a640 1 -- 172.21.15.32:0/2255983158 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f74181a01b0 con 0x7f7418102230 2024-04-25T00:33:07.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.161+0000 7f740e7fc640 1 -- 172.21.15.32:0/2255983158 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 18) ==== 137833+0+0 (secure 0 0 0) 0x7f7410003970 con 0x7f7418102230 2024-04-25T00:33:07.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.161+0000 7f740e7fc640 1 --2- 172.21.15.32:0/2255983158 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f74040a2b90 0x7f74040a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:07.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.162+0000 7f740e7fc640 1 -- 172.21.15.32:0/2255983158 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7f74100cf160 con 0x7f7418102230 2024-04-25T00:33:07.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.162+0000 7f741d7c6640 1 --2- 172.21.15.32:0/2255983158 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f74040a2b90 0x7f74040a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:07.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.162+0000 7f741d7c6640 1 --2- 172.21.15.32:0/2255983158 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f74040a2b90 0x7f74040a5050 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f7408005ab0 tx=0x7f7408000bc0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:07.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.165+0000 7f740e7fc640 1 -- 172.21.15.32:0/2255983158 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f741009b640 con 0x7f7418102230 2024-04-25T00:33:07.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.363+0000 7f741fa2a640 1 -- 172.21.15.32:0/2255983158 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7f74181a04a0 con 0x7f7418102230 2024-04-25T00:33:07.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.363+0000 7f740e7fc640 1 -- 172.21.15.32:0/2255983158 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v93) ==== 93+0+10 (secure 0 0 0) 0x7f7410097030 con 0x7f7418102230 2024-04-25T00:33:07.364 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 1 2024-04-25T00:33:07.366 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.365+0000 7f741fa2a640 1 -- 172.21.15.32:0/2255983158 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f74040a2b90 msgr2=0x7f74040a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:07.366 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.365+0000 7f741fa2a640 1 --2- 172.21.15.32:0/2255983158 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f74040a2b90 0x7f74040a5050 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f7408005ab0 tx=0x7f7408000bc0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.366 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.365+0000 7f741fa2a640 1 -- 172.21.15.32:0/2255983158 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7418102230 msgr2=0x7f741819f0d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:07.366 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.365+0000 7f741fa2a640 1 --2- 172.21.15.32:0/2255983158 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7418102230 0x7f741819f0d0 secure :-1 s=READY pgs=417 cs=0 l=1 rev1=1 crypto rx=0x7f7410009fd0 tx=0x7f741000b260 comp rx=0 tx=0).stop 2024-04-25T00:33:07.366 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.365+0000 7f741fa2a640 1 -- 172.21.15.32:0/2255983158 shutdown_connections 2024-04-25T00:33:07.366 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.365+0000 7f741fa2a640 1 --2- 172.21.15.32:0/2255983158 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f74040a2b90 0x7f74040a5050 unknown :-1 s=CLOSED pgs=253 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.366 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.365+0000 7f741fa2a640 1 --2- 172.21.15.32:0/2255983158 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7418102230 0x7f741819f0d0 unknown :-1 s=CLOSED pgs=417 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.366 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.365+0000 7f741fa2a640 1 --2- 172.21.15.32:0/2255983158 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7418101870 0x7f7418198060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.367 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.365+0000 7f741fa2a640 1 --2- 172.21.15.32:0/2255983158 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7418100680 0x7f7418197b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.367 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.365+0000 7f741fa2a640 1 -- 172.21.15.32:0/2255983158 >> 172.21.15.32:0/2255983158 conn(0x7f74180fbbf0 msgr2=0x7f74180688d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:07.367 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.365+0000 7f741fa2a640 1 -- 172.21.15.32:0/2255983158 shutdown_connections 2024-04-25T00:33:07.367 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.365+0000 7f741fa2a640 1 -- 172.21.15.32:0/2255983158 wait complete. 2024-04-25T00:33:07.377 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-04-25T00:33:07.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.482+0000 7f2e0f2da640 1 -- 172.21.15.32:0/3043703242 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2e08102250 msgr2=0x7f2e08108ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:07.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.482+0000 7f2e0f2da640 1 --2- 172.21.15.32:0/3043703242 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2e08102250 0x7f2e08108ae0 secure :-1 s=READY pgs=418 cs=0 l=1 rev1=1 crypto rx=0x7f2df8009ca0 tx=0x7f2df802a8c0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.482+0000 7f2e0f2da640 1 -- 172.21.15.32:0/3043703242 shutdown_connections 2024-04-25T00:33:07.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.482+0000 7f2e0f2da640 1 --2- 172.21.15.32:0/3043703242 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2e08102250 0x7f2e08108ae0 unknown :-1 s=CLOSED pgs=418 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.482+0000 7f2e0f2da640 1 --2- 172.21.15.32:0/3043703242 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f2e08101890 0x7f2e08101d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.482+0000 7f2e0f2da640 1 --2- 172.21.15.32:0/3043703242 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2e081006a0 0x7f2e08100ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.482+0000 7f2e0f2da640 1 -- 172.21.15.32:0/3043703242 >> 172.21.15.32:0/3043703242 conn(0x7f2e080fbc30 msgr2=0x7f2e080fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:07.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.482+0000 7f2e0f2da640 1 -- 172.21.15.32:0/3043703242 shutdown_connections 2024-04-25T00:33:07.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.482+0000 7f2e0f2da640 1 -- 172.21.15.32:0/3043703242 wait complete. 2024-04-25T00:33:07.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.482+0000 7f2e0f2da640 1 Processor -- start 2024-04-25T00:33:07.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.482+0000 7f2e0f2da640 1 -- start start 2024-04-25T00:33:07.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.483+0000 7f2e0f2da640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f2e081006a0 0x7f2e08197ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:07.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.483+0000 7f2e0f2da640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2e08101890 0x7f2e08197ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:07.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.483+0000 7f2e0f2da640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2e08102250 0x7f2e0819f060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:07.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.483+0000 7f2e0f2da640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f2e0810c320 con 0x7f2e08101890 2024-04-25T00:33:07.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.483+0000 7f2e0f2da640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f2e0810c1a0 con 0x7f2e081006a0 2024-04-25T00:33:07.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.483+0000 7f2e0f2da640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f2e0810c4a0 con 0x7f2e08102250 2024-04-25T00:33:07.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.483+0000 7f2e0c875640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2e08101890 0x7f2e08197ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:07.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.483+0000 7f2e0c875640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2e08101890 0x7f2e08197ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44072/0 (socket says 172.21.15.32:44072) 2024-04-25T00:33:07.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.483+0000 7f2e0c875640 1 -- 172.21.15.32:0/3038970204 learned_addr learned my addr 172.21.15.32:0/3038970204 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:07.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.483+0000 7f2e0d877640 1 --2- 172.21.15.32:0/3038970204 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2e08102250 0x7f2e0819f060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:07.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.483+0000 7f2e0d076640 1 --2- 172.21.15.32:0/3038970204 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f2e081006a0 0x7f2e08197ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:07.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.483+0000 7f2e0c875640 1 -- 172.21.15.32:0/3038970204 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2e08102250 msgr2=0x7f2e0819f060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:07.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.483+0000 7f2e0c875640 1 --2- 172.21.15.32:0/3038970204 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2e08102250 0x7f2e0819f060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.483+0000 7f2e0c875640 1 -- 172.21.15.32:0/3038970204 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f2e081006a0 msgr2=0x7f2e08197ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:07.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.483+0000 7f2e0c875640 1 --2- 172.21.15.32:0/3038970204 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f2e081006a0 0x7f2e08197ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.489 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.483+0000 7f2e0c875640 1 -- 172.21.15.32:0/3038970204 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2e0819f6d0 con 0x7f2e08101890 2024-04-25T00:33:07.489 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.483+0000 7f2e0d877640 1 --2- 172.21.15.32:0/3038970204 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2e08102250 0x7f2e0819f060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:07.489 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.483+0000 7f2e0d076640 1 --2- 172.21.15.32:0/3038970204 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f2e081006a0 0x7f2e08197ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:07.489 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.483+0000 7f2e0c875640 1 --2- 172.21.15.32:0/3038970204 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2e08101890 0x7f2e08197ff0 secure :-1 s=READY pgs=419 cs=0 l=1 rev1=1 crypto rx=0x7f2dfc00d950 tx=0x7f2dfc00de10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:07.489 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.483+0000 7f2e067fc640 1 -- 172.21.15.32:0/3038970204 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2dfc017070 con 0x7f2e08101890 2024-04-25T00:33:07.489 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.483+0000 7f2e067fc640 1 -- 172.21.15.32:0/3038970204 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f2dfc009c30 con 0x7f2e08101890 2024-04-25T00:33:07.489 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.483+0000 7f2e0f2da640 1 -- 172.21.15.32:0/3038970204 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2e0819f9c0 con 0x7f2e08101890 2024-04-25T00:33:07.489 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.483+0000 7f2e067fc640 1 -- 172.21.15.32:0/3038970204 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2dfc00f6b0 con 0x7f2e08101890 2024-04-25T00:33:07.490 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.483+0000 7f2e0f2da640 1 -- 172.21.15.32:0/3038970204 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2e0819ff60 con 0x7f2e08101890 2024-04-25T00:33:07.490 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.484+0000 7f2e0f2da640 1 -- 172.21.15.32:0/3038970204 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2e081a0190 con 0x7f2e08101890 2024-04-25T00:33:07.491 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.486+0000 7f2e067fc640 1 -- 172.21.15.32:0/3038970204 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 18) ==== 137833+0+0 (secure 0 0 0) 0x7f2dfc0109d0 con 0x7f2e08101890 2024-04-25T00:33:07.491 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.487+0000 7f2e067fc640 1 --2- 172.21.15.32:0/3038970204 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f2df00a2c10 0x7f2df00a50d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:07.491 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.487+0000 7f2e067fc640 1 -- 172.21.15.32:0/3038970204 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7f2dfc0cf0e0 con 0x7f2e08101890 2024-04-25T00:33:07.491 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.487+0000 7f2e0d076640 1 --2- 172.21.15.32:0/3038970204 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f2df00a2c10 0x7f2df00a50d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:07.491 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.487+0000 7f2e0d076640 1 --2- 172.21.15.32:0/3038970204 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f2df00a2c10 0x7f2df00a50d0 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f2e000058c0 tx=0x7f2e00005850 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:07.491 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.490+0000 7f2e067fc640 1 -- 172.21.15.32:0/3038970204 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f2dfc009de0 con 0x7f2e08101890 2024-04-25T00:33:07.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.674+0000 7f2e0f2da640 1 -- 172.21.15.32:0/3038970204 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) -- 0x7f2e081a0360 con 0x7f2e08101890 2024-04-25T00:33:07.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.674+0000 7f2e067fc640 1 -- 172.21.15.32:0/3038970204 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v93) ==== 104+0+11 (secure 0 0 0) 0x7f2dfc09bc90 con 0x7f2e08101890 2024-04-25T00:33:07.675 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 32 2024-04-25T00:33:07.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.676+0000 7f2e0f2da640 1 -- 172.21.15.32:0/3038970204 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f2df00a2c10 msgr2=0x7f2df00a50d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:07.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.676+0000 7f2e0f2da640 1 --2- 172.21.15.32:0/3038970204 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f2df00a2c10 0x7f2df00a50d0 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f2e000058c0 tx=0x7f2e00005850 comp rx=0 tx=0).stop 2024-04-25T00:33:07.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.676+0000 7f2e0f2da640 1 -- 172.21.15.32:0/3038970204 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2e08101890 msgr2=0x7f2e08197ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:07.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.676+0000 7f2e0f2da640 1 --2- 172.21.15.32:0/3038970204 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2e08101890 0x7f2e08197ff0 secure :-1 s=READY pgs=419 cs=0 l=1 rev1=1 crypto rx=0x7f2dfc00d950 tx=0x7f2dfc00de10 comp rx=0 tx=0).stop 2024-04-25T00:33:07.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.676+0000 7f2e0f2da640 1 -- 172.21.15.32:0/3038970204 shutdown_connections 2024-04-25T00:33:07.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.676+0000 7f2e0f2da640 1 --2- 172.21.15.32:0/3038970204 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f2df00a2c10 0x7f2df00a50d0 unknown :-1 s=CLOSED pgs=254 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.676+0000 7f2e0f2da640 1 --2- 172.21.15.32:0/3038970204 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2e08102250 0x7f2e0819f060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.676+0000 7f2e0f2da640 1 --2- 172.21.15.32:0/3038970204 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2e08101890 0x7f2e08197ff0 unknown :-1 s=CLOSED pgs=419 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.676+0000 7f2e0f2da640 1 --2- 172.21.15.32:0/3038970204 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f2e081006a0 0x7f2e08197ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.676+0000 7f2e0f2da640 1 -- 172.21.15.32:0/3038970204 >> 172.21.15.32:0/3038970204 conn(0x7f2e080fbc30 msgr2=0x7f2e08069d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:07.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.676+0000 7f2e0f2da640 1 -- 172.21.15.32:0/3038970204 shutdown_connections 2024-04-25T00:33:07.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.676+0000 7f2e0f2da640 1 -- 172.21.15.32:0/3038970204 wait complete. 2024-04-25T00:33:07.687 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:07 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/4192571043' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:33:07.687 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:07 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/679940316' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:33:07.687 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:07 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2255983158' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:33:07.688 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-04-25T00:33:07.795 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.793+0000 7fae586ee640 1 -- 172.21.15.32:0/1992341990 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fae501022e0 msgr2=0x7fae501046d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:07.795 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.793+0000 7fae586ee640 1 --2- 172.21.15.32:0/1992341990 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fae501022e0 0x7fae501046d0 secure :-1 s=READY pgs=420 cs=0 l=1 rev1=1 crypto rx=0x7fae4c009900 tx=0x7fae4c02a410 comp rx=0 tx=0).stop 2024-04-25T00:33:07.796 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.793+0000 7fae586ee640 1 -- 172.21.15.32:0/1992341990 shutdown_connections 2024-04-25T00:33:07.796 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.793+0000 7fae586ee640 1 --2- 172.21.15.32:0/1992341990 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fae50107540 0x7fae50109930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.796 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.793+0000 7fae586ee640 1 --2- 172.21.15.32:0/1992341990 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fae50104c10 0x7fae50107000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.796 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.793+0000 7fae586ee640 1 --2- 172.21.15.32:0/1992341990 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fae501022e0 0x7fae501046d0 unknown :-1 s=CLOSED pgs=420 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.796 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.793+0000 7fae586ee640 1 -- 172.21.15.32:0/1992341990 >> 172.21.15.32:0/1992341990 conn(0x7fae500fbc30 msgr2=0x7fae500fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:07.796 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.793+0000 7fae586ee640 1 -- 172.21.15.32:0/1992341990 shutdown_connections 2024-04-25T00:33:07.796 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.793+0000 7fae586ee640 1 -- 172.21.15.32:0/1992341990 wait complete. 2024-04-25T00:33:07.796 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.793+0000 7fae586ee640 1 Processor -- start 2024-04-25T00:33:07.796 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.793+0000 7fae586ee640 1 -- start start 2024-04-25T00:33:07.796 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.793+0000 7fae586ee640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fae501022e0 0x7fae50197a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:07.796 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.793+0000 7fae586ee640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fae50104c10 0x7fae50197f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:07.796 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.794+0000 7fae586ee640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fae50107540 0x7fae5019efd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:07.796 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.794+0000 7fae586ee640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fae5010d470 con 0x7fae50104c10 2024-04-25T00:33:07.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.794+0000 7fae586ee640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fae5010d2f0 con 0x7fae50107540 2024-04-25T00:33:07.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.794+0000 7fae586ee640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fae5010d5f0 con 0x7fae501022e0 2024-04-25T00:33:07.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.794+0000 7fae55c89640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fae50104c10 0x7fae50197f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:07.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.794+0000 7fae55c89640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fae50104c10 0x7fae50197f60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44090/0 (socket says 172.21.15.32:44090) 2024-04-25T00:33:07.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.794+0000 7fae55c89640 1 -- 172.21.15.32:0/283734276 learned_addr learned my addr 172.21.15.32:0/283734276 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:07.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.794+0000 7fae5648a640 1 --2- 172.21.15.32:0/283734276 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fae501022e0 0x7fae50197a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:07.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.794+0000 7fae56c8b640 1 --2- 172.21.15.32:0/283734276 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fae50107540 0x7fae5019efd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:07.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.794+0000 7fae55c89640 1 -- 172.21.15.32:0/283734276 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fae501022e0 msgr2=0x7fae50197a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:07.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.794+0000 7fae55c89640 1 --2- 172.21.15.32:0/283734276 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fae501022e0 0x7fae50197a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.794+0000 7fae55c89640 1 -- 172.21.15.32:0/283734276 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fae50107540 msgr2=0x7fae5019efd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:07.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.794+0000 7fae55c89640 1 --2- 172.21.15.32:0/283734276 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fae50107540 0x7fae5019efd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.794+0000 7fae55c89640 1 -- 172.21.15.32:0/283734276 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fae5019f640 con 0x7fae50104c10 2024-04-25T00:33:07.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.794+0000 7fae55c89640 1 --2- 172.21.15.32:0/283734276 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fae50104c10 0x7fae50197f60 secure :-1 s=READY pgs=421 cs=0 l=1 rev1=1 crypto rx=0x7fae4000c970 tx=0x7fae4000ce30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:07.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.794+0000 7fae477fe640 1 -- 172.21.15.32:0/283734276 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fae40014070 con 0x7fae50104c10 2024-04-25T00:33:07.798 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.794+0000 7fae477fe640 1 -- 172.21.15.32:0/283734276 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fae40004440 con 0x7fae50104c10 2024-04-25T00:33:07.798 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.794+0000 7fae586ee640 1 -- 172.21.15.32:0/283734276 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fae5019f930 con 0x7fae50104c10 2024-04-25T00:33:07.798 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.794+0000 7fae477fe640 1 -- 172.21.15.32:0/283734276 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fae4000f450 con 0x7fae50104c10 2024-04-25T00:33:07.798 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.794+0000 7fae586ee640 1 -- 172.21.15.32:0/283734276 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fae5019fe80 con 0x7fae50104c10 2024-04-25T00:33:07.799 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.795+0000 7fae586ee640 1 -- 172.21.15.32:0/283734276 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fae500fd720 con 0x7fae50104c10 2024-04-25T00:33:07.799 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.796+0000 7fae477fe640 1 -- 172.21.15.32:0/283734276 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 18) ==== 137833+0+0 (secure 0 0 0) 0x7fae40017070 con 0x7fae50104c10 2024-04-25T00:33:07.799 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.797+0000 7fae477fe640 1 --2- 172.21.15.32:0/283734276 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fae3c0a2b90 0x7fae3c0a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:07.799 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.797+0000 7fae477fe640 1 -- 172.21.15.32:0/283734276 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7fae400cf320 con 0x7fae50104c10 2024-04-25T00:33:07.799 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.797+0000 7fae5648a640 1 --2- 172.21.15.32:0/283734276 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fae3c0a2b90 0x7fae3c0a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:07.799 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.798+0000 7fae5648a640 1 --2- 172.21.15.32:0/283734276 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fae3c0a2b90 0x7fae3c0a5050 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7fae4c006010 tx=0x7fae4c031400 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:07.802 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.801+0000 7fae477fe640 1 -- 172.21.15.32:0/283734276 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fae4001e020 con 0x7fae50104c10 2024-04-25T00:33:07.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:07 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/4192571043' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:33:07.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:07 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/679940316' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:33:07.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:07 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2255983158' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:33:07.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:07 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/4192571043' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:33:07.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:07 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/679940316' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:33:07.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:07 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2255983158' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:33:07.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.988+0000 7fae586ee640 1 -- 172.21.15.32:0/283734276 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) -- 0x7fae50062340 con 0x7fae50104c10 2024-04-25T00:33:07.989 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.988+0000 7fae477fe640 1 -- 172.21.15.32:0/283734276 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v93) ==== 100+0+11 (secure 0 0 0) 0x7fae4009beb0 con 0x7fae50104c10 2024-04-25T00:33:07.989 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 64 2024-04-25T00:33:07.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.990+0000 7fae586ee640 1 -- 172.21.15.32:0/283734276 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fae3c0a2b90 msgr2=0x7fae3c0a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:07.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.990+0000 7fae586ee640 1 --2- 172.21.15.32:0/283734276 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fae3c0a2b90 0x7fae3c0a5050 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7fae4c006010 tx=0x7fae4c031400 comp rx=0 tx=0).stop 2024-04-25T00:33:07.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.990+0000 7fae586ee640 1 -- 172.21.15.32:0/283734276 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fae50104c10 msgr2=0x7fae50197f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:07.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.990+0000 7fae586ee640 1 --2- 172.21.15.32:0/283734276 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fae50104c10 0x7fae50197f60 secure :-1 s=READY pgs=421 cs=0 l=1 rev1=1 crypto rx=0x7fae4000c970 tx=0x7fae4000ce30 comp rx=0 tx=0).stop 2024-04-25T00:33:07.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.990+0000 7fae586ee640 1 -- 172.21.15.32:0/283734276 shutdown_connections 2024-04-25T00:33:07.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.990+0000 7fae586ee640 1 --2- 172.21.15.32:0/283734276 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fae3c0a2b90 0x7fae3c0a5050 unknown :-1 s=CLOSED pgs=255 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.990+0000 7fae586ee640 1 --2- 172.21.15.32:0/283734276 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fae50107540 0x7fae5019efd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.990+0000 7fae586ee640 1 --2- 172.21.15.32:0/283734276 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fae50104c10 0x7fae50197f60 unknown :-1 s=CLOSED pgs=421 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.990+0000 7fae586ee640 1 --2- 172.21.15.32:0/283734276 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fae501022e0 0x7fae50197a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:07.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.990+0000 7fae586ee640 1 -- 172.21.15.32:0/283734276 >> 172.21.15.32:0/283734276 conn(0x7fae500fbc30 msgr2=0x7fae50102c00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:07.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.990+0000 7fae586ee640 1 -- 172.21.15.32:0/283734276 shutdown_connections 2024-04-25T00:33:07.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:07.990+0000 7fae586ee640 1 -- 172.21.15.32:0/283734276 wait complete. 2024-04-25T00:33:08.002 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data_ec pg_num 2024-04-25T00:33:08.109 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.106+0000 7fb12abc4640 1 -- 172.21.15.32:0/2190688989 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb124101790 msgr2=0x7fb124101bb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:08.109 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.106+0000 7fb12abc4640 1 --2- 172.21.15.32:0/2190688989 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb124101790 0x7fb124101bb0 secure :-1 s=READY pgs=422 cs=0 l=1 rev1=1 crypto rx=0x7fb11c009ca0 tx=0x7fb11c02a8c0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.109 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.106+0000 7fb12abc4640 1 -- 172.21.15.32:0/2190688989 shutdown_connections 2024-04-25T00:33:08.109 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.106+0000 7fb12abc4640 1 --2- 172.21.15.32:0/2190688989 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb124103340 0x7fb124109bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.109 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.106+0000 7fb12abc4640 1 --2- 172.21.15.32:0/2190688989 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb124102980 0x7fb124102e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.109 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.106+0000 7fb12abc4640 1 --2- 172.21.15.32:0/2190688989 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb124101790 0x7fb124101bb0 unknown :-1 s=CLOSED pgs=422 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.109 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.106+0000 7fb12abc4640 1 -- 172.21.15.32:0/2190688989 >> 172.21.15.32:0/2190688989 conn(0x7fb1240fcd20 msgr2=0x7fb1240ff140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:08.109 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.106+0000 7fb12abc4640 1 -- 172.21.15.32:0/2190688989 shutdown_connections 2024-04-25T00:33:08.109 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.106+0000 7fb12abc4640 1 -- 172.21.15.32:0/2190688989 wait complete. 2024-04-25T00:33:08.109 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.106+0000 7fb12abc4640 1 Processor -- start 2024-04-25T00:33:08.109 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.106+0000 7fb12abc4640 1 -- start start 2024-04-25T00:33:08.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.107+0000 7fb12abc4640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb124101790 0x7fb124197a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:08.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.107+0000 7fb12abc4640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb124102980 0x7fb124197f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:08.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.107+0000 7fb12abc4640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb124103340 0x7fb12419f000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:08.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.107+0000 7fb12abc4640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fb12410d3e0 con 0x7fb124103340 2024-04-25T00:33:08.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.107+0000 7fb12abc4640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fb12410d260 con 0x7fb124101790 2024-04-25T00:33:08.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.107+0000 7fb12abc4640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fb12410d560 con 0x7fb124102980 2024-04-25T00:33:08.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.107+0000 7fb129161640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb124103340 0x7fb12419f000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:08.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.107+0000 7fb129161640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb124103340 0x7fb12419f000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44118/0 (socket says 172.21.15.32:44118) 2024-04-25T00:33:08.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.107+0000 7fb129161640 1 -- 172.21.15.32:0/1828036779 learned_addr learned my addr 172.21.15.32:0/1828036779 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:08.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.107+0000 7fb123fff640 1 --2- 172.21.15.32:0/1828036779 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb124102980 0x7fb124197f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:08.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.107+0000 7fb128960640 1 --2- 172.21.15.32:0/1828036779 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb124101790 0x7fb124197a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:08.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.107+0000 7fb129161640 1 -- 172.21.15.32:0/1828036779 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb124102980 msgr2=0x7fb124197f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:08.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.107+0000 7fb129161640 1 --2- 172.21.15.32:0/1828036779 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb124102980 0x7fb124197f90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.110 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.107+0000 7fb129161640 1 -- 172.21.15.32:0/1828036779 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb124101790 msgr2=0x7fb124197a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:08.111 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.107+0000 7fb129161640 1 --2- 172.21.15.32:0/1828036779 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb124101790 0x7fb124197a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.111 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.107+0000 7fb129161640 1 -- 172.21.15.32:0/1828036779 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb12419f670 con 0x7fb124103340 2024-04-25T00:33:08.111 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.107+0000 7fb129161640 1 --2- 172.21.15.32:0/1828036779 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb124103340 0x7fb12419f000 secure :-1 s=READY pgs=423 cs=0 l=1 rev1=1 crypto rx=0x7fb11400d750 tx=0x7fb11400db50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:08.111 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.107+0000 7fb121ffb640 1 -- 172.21.15.32:0/1828036779 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb114017070 con 0x7fb124103340 2024-04-25T00:33:08.111 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.107+0000 7fb12abc4640 1 -- 172.21.15.32:0/1828036779 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb12419f960 con 0x7fb124103340 2024-04-25T00:33:08.111 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.107+0000 7fb121ffb640 1 -- 172.21.15.32:0/1828036779 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb1140097c0 con 0x7fb124103340 2024-04-25T00:33:08.111 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.107+0000 7fb121ffb640 1 -- 172.21.15.32:0/1828036779 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb114004d00 con 0x7fb124103340 2024-04-25T00:33:08.111 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.108+0000 7fb12abc4640 1 -- 172.21.15.32:0/1828036779 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb12419feb0 con 0x7fb124103340 2024-04-25T00:33:08.112 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.109+0000 7fb12abc4640 1 -- 172.21.15.32:0/1828036779 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb124108ab0 con 0x7fb124103340 2024-04-25T00:33:08.112 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.110+0000 7fb121ffb640 1 -- 172.21.15.32:0/1828036779 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 18) ==== 137833+0+0 (secure 0 0 0) 0x7fb114009970 con 0x7fb124103340 2024-04-25T00:33:08.112 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.110+0000 7fb121ffb640 1 --2- 172.21.15.32:0/1828036779 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb10c0a2b90 0x7fb10c0a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:08.112 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.110+0000 7fb128960640 1 --2- 172.21.15.32:0/1828036779 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb10c0a2b90 0x7fb10c0a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:08.112 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.110+0000 7fb121ffb640 1 -- 172.21.15.32:0/1828036779 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7fb1140ced80 con 0x7fb124103340 2024-04-25T00:33:08.112 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.111+0000 7fb128960640 1 --2- 172.21.15.32:0/1828036779 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb10c0a2b90 0x7fb10c0a5050 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7fb11c006010 tx=0x7fb11c000f40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:08.115 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.115+0000 7fb121ffb640 1 -- 172.21.15.32:0/1828036779 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fb11409b910 con 0x7fb124103340 2024-04-25T00:33:08.299 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.299+0000 7fb12abc4640 1 -- 172.21.15.32:0/1828036779 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"} v 0) -- 0x7fb124062340 con 0x7fb124103340 2024-04-25T00:33:08.300 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.299+0000 7fb121ffb640 1 -- 172.21.15.32:0/1828036779 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]=0 v93) ==== 103+0+11 (secure 0 0 0) 0x7fb11409b210 con 0x7fb124103340 2024-04-25T00:33:08.300 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 32 2024-04-25T00:33:08.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.301+0000 7fb12abc4640 1 -- 172.21.15.32:0/1828036779 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb10c0a2b90 msgr2=0x7fb10c0a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:08.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.301+0000 7fb12abc4640 1 --2- 172.21.15.32:0/1828036779 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb10c0a2b90 0x7fb10c0a5050 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7fb11c006010 tx=0x7fb11c000f40 comp rx=0 tx=0).stop 2024-04-25T00:33:08.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.301+0000 7fb12abc4640 1 -- 172.21.15.32:0/1828036779 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb124103340 msgr2=0x7fb12419f000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:08.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.301+0000 7fb12abc4640 1 --2- 172.21.15.32:0/1828036779 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb124103340 0x7fb12419f000 secure :-1 s=READY pgs=423 cs=0 l=1 rev1=1 crypto rx=0x7fb11400d750 tx=0x7fb11400db50 comp rx=0 tx=0).stop 2024-04-25T00:33:08.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.301+0000 7fb12abc4640 1 -- 172.21.15.32:0/1828036779 shutdown_connections 2024-04-25T00:33:08.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.301+0000 7fb12abc4640 1 --2- 172.21.15.32:0/1828036779 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb10c0a2b90 0x7fb10c0a5050 unknown :-1 s=CLOSED pgs=256 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.301+0000 7fb12abc4640 1 --2- 172.21.15.32:0/1828036779 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb124103340 0x7fb12419f000 unknown :-1 s=CLOSED pgs=423 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.301+0000 7fb12abc4640 1 --2- 172.21.15.32:0/1828036779 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb124102980 0x7fb124197f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.301+0000 7fb12abc4640 1 --2- 172.21.15.32:0/1828036779 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb124101790 0x7fb124197a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.301+0000 7fb12abc4640 1 -- 172.21.15.32:0/1828036779 >> 172.21.15.32:0/1828036779 conn(0x7fb1240fcd20 msgr2=0x7fb124105c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:08.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.301+0000 7fb12abc4640 1 -- 172.21.15.32:0/1828036779 shutdown_connections 2024-04-25T00:33:08.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.301+0000 7fb12abc4640 1 -- 172.21.15.32:0/1828036779 wait complete. 2024-04-25T00:33:08.313 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:33:08.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.414+0000 7f620d163640 1 -- 172.21.15.32:0/3194900716 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6208100680 msgr2=0x7f6208100aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:08.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.414+0000 7f620d163640 1 --2- 172.21.15.32:0/3194900716 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6208100680 0x7f6208100aa0 secure :-1 s=READY pgs=424 cs=0 l=1 rev1=1 crypto rx=0x7f6200009900 tx=0x7f620002a3f0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.414+0000 7f620d163640 1 -- 172.21.15.32:0/3194900716 shutdown_connections 2024-04-25T00:33:08.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.414+0000 7f620d163640 1 --2- 172.21.15.32:0/3194900716 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6208102230 0x7f6208108ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.414+0000 7f620d163640 1 --2- 172.21.15.32:0/3194900716 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6208101870 0x7f6208101cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.414+0000 7f620d163640 1 --2- 172.21.15.32:0/3194900716 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6208100680 0x7f6208100aa0 unknown :-1 s=CLOSED pgs=424 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.414+0000 7f620d163640 1 -- 172.21.15.32:0/3194900716 >> 172.21.15.32:0/3194900716 conn(0x7f62080fbbf0 msgr2=0x7f62080fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:08.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.414+0000 7f620d163640 1 -- 172.21.15.32:0/3194900716 shutdown_connections 2024-04-25T00:33:08.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.414+0000 7f620d163640 1 -- 172.21.15.32:0/3194900716 wait complete. 2024-04-25T00:33:08.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.415+0000 7f620d163640 1 Processor -- start 2024-04-25T00:33:08.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.415+0000 7f620d163640 1 -- start start 2024-04-25T00:33:08.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.415+0000 7f620d163640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6208100680 0x7f6208197ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:08.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.415+0000 7f620d163640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6208101870 0x7f6208198020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:08.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.415+0000 7f620d163640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6208102230 0x7f620819f090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:08.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.415+0000 7f620d163640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f620810d3e0 con 0x7f6208102230 2024-04-25T00:33:08.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.416+0000 7f620d163640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f620810d260 con 0x7f6208100680 2024-04-25T00:33:08.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.416+0000 7f620d163640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f620810d560 con 0x7f6208101870 2024-04-25T00:33:08.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.416+0000 7f620759e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6208102230 0x7f620819f090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:08.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.416+0000 7f620759e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6208102230 0x7f620819f090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44146/0 (socket says 172.21.15.32:44146) 2024-04-25T00:33:08.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.416+0000 7f620759e640 1 -- 172.21.15.32:0/3398744650 learned_addr learned my addr 172.21.15.32:0/3398744650 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:08.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.416+0000 7f620659c640 1 --2- 172.21.15.32:0/3398744650 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6208101870 0x7f6208198020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:08.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.416+0000 7f6206d9d640 1 --2- 172.21.15.32:0/3398744650 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6208100680 0x7f6208197ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:08.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.416+0000 7f620759e640 1 -- 172.21.15.32:0/3398744650 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6208101870 msgr2=0x7f6208198020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:08.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.416+0000 7f620759e640 1 --2- 172.21.15.32:0/3398744650 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6208101870 0x7f6208198020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.420 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.416+0000 7f620759e640 1 -- 172.21.15.32:0/3398744650 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6208100680 msgr2=0x7f6208197ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:08.420 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.416+0000 7f620759e640 1 --2- 172.21.15.32:0/3398744650 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6208100680 0x7f6208197ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.420 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.416+0000 7f620759e640 1 -- 172.21.15.32:0/3398744650 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f620819f700 con 0x7f6208102230 2024-04-25T00:33:08.420 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.416+0000 7f6206d9d640 1 --2- 172.21.15.32:0/3398744650 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6208100680 0x7f6208197ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:08.420 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.416+0000 7f620759e640 1 --2- 172.21.15.32:0/3398744650 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6208102230 0x7f620819f090 secure :-1 s=READY pgs=425 cs=0 l=1 rev1=1 crypto rx=0x7f61f800cad0 tx=0x7f61f800cf90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:08.420 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.416+0000 7f61effff640 1 -- 172.21.15.32:0/3398744650 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f61f800ae20 con 0x7f6208102230 2024-04-25T00:33:08.420 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.416+0000 7f620d163640 1 -- 172.21.15.32:0/3398744650 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f620819f9f0 con 0x7f6208102230 2024-04-25T00:33:08.420 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.416+0000 7f61effff640 1 -- 172.21.15.32:0/3398744650 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f61f8005020 con 0x7f6208102230 2024-04-25T00:33:08.420 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.416+0000 7f61effff640 1 -- 172.21.15.32:0/3398744650 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f61f800f6f0 con 0x7f6208102230 2024-04-25T00:33:08.420 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.416+0000 7f620d163640 1 -- 172.21.15.32:0/3398744650 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f620819fef0 con 0x7f6208102230 2024-04-25T00:33:08.420 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.417+0000 7f620d163640 1 -- 172.21.15.32:0/3398744650 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6208107a20 con 0x7f6208102230 2024-04-25T00:33:08.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.419+0000 7f61effff640 1 -- 172.21.15.32:0/3398744650 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 18) ==== 137833+0+0 (secure 0 0 0) 0x7f61f80051d0 con 0x7f6208102230 2024-04-25T00:33:08.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.419+0000 7f61effff640 1 --2- 172.21.15.32:0/3398744650 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f61f00a2b90 0x7f61f00a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:08.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.419+0000 7f61effff640 1 -- 172.21.15.32:0/3398744650 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7f61f80cf820 con 0x7f6208102230 2024-04-25T00:33:08.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.419+0000 7f6206d9d640 1 --2- 172.21.15.32:0/3398744650 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f61f00a2b90 0x7f61f00a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:08.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.420+0000 7f6206d9d640 1 --2- 172.21.15.32:0/3398744650 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f61f00a2b90 0x7f61f00a5050 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7f6200006010 tx=0x7f6200032dd0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:08.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.423+0000 7f61effff640 1 -- 172.21.15.32:0/3398744650 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f61f8014030 con 0x7f6208102230 2024-04-25T00:33:08.648 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.648+0000 7f620d163640 1 -- 172.21.15.32:0/3398744650 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f620802d170 con 0x7f6208102230 2024-04-25T00:33:08.649 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.649+0000 7f61effff640 1 -- 172.21.15.32:0/3398744650 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) ==== 94+0+11876 (secure 0 0 0) 0x7f61f809c3b0 con 0x7f6208102230 2024-04-25T00:33:08.649 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:33:08.649 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14643,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6839/187492674","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6838","nonce":187492674},{"type":"v1","addr":"172.21.15.32:6839","nonce":187492674}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14655,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6841/1948785545","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6840","nonce":1948785545},{"type":"v1","addr":"172.21.15.32:6841","nonce":1948785545}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-25T00:30:47.866875+0000","modified":"2024-04-25T00:31:14.972514+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24506,"mds_1":24481,"mds_2":24476},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24443":{"gid":24443,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24445":{"gid":24445,"name":"f","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24457":{"gid":24457,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24476":{"gid":24476,"name":"h","rank":2,"incarnation":22,"state":"up:active","state_seq":12,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24481":{"gid":24481,"name":"i","rank":1,"incarnation":20,"state":"up:active","state_seq":11,"addr":"172.21.15.130:6839/4267125809","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":4267125809},{"type":"v1","addr":"172.21.15.130:6839","nonce":4267125809}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24506":{"gid":24506,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.121:6839/258444853","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6838","nonce":258444853},{"type":"v1","addr":"172.21.15.121:6839","nonce":258444853}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24506,"qdb_cluster":[24476,24481,24506]},"id":1}]} 2024-04-25T00:33:08.650 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 24 2024-04-25T00:33:08.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.651+0000 7f620d163640 1 -- 172.21.15.32:0/3398744650 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f61f00a2b90 msgr2=0x7f61f00a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:08.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.651+0000 7f620d163640 1 --2- 172.21.15.32:0/3398744650 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f61f00a2b90 0x7f61f00a5050 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7f6200006010 tx=0x7f6200032dd0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.651+0000 7f620d163640 1 -- 172.21.15.32:0/3398744650 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6208102230 msgr2=0x7f620819f090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:08.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.651+0000 7f620d163640 1 --2- 172.21.15.32:0/3398744650 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6208102230 0x7f620819f090 secure :-1 s=READY pgs=425 cs=0 l=1 rev1=1 crypto rx=0x7f61f800cad0 tx=0x7f61f800cf90 comp rx=0 tx=0).stop 2024-04-25T00:33:08.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.651+0000 7f620d163640 1 -- 172.21.15.32:0/3398744650 shutdown_connections 2024-04-25T00:33:08.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.651+0000 7f620d163640 1 --2- 172.21.15.32:0/3398744650 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f61f00a2b90 0x7f61f00a5050 unknown :-1 s=CLOSED pgs=257 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.651+0000 7f620d163640 1 --2- 172.21.15.32:0/3398744650 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6208102230 0x7f620819f090 unknown :-1 s=CLOSED pgs=425 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.651+0000 7f620d163640 1 --2- 172.21.15.32:0/3398744650 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6208101870 0x7f6208198020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.651+0000 7f620d163640 1 --2- 172.21.15.32:0/3398744650 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6208100680 0x7f6208197ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.651+0000 7f620d163640 1 -- 172.21.15.32:0/3398744650 >> 172.21.15.32:0/3398744650 conn(0x7f62080fbbf0 msgr2=0x7f6208104bf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:08.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.651+0000 7f620d163640 1 -- 172.21.15.32:0/3398744650 shutdown_connections 2024-04-25T00:33:08.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.651+0000 7f620d163640 1 -- 172.21.15.32:0/3398744650 wait complete. 2024-04-25T00:33:08.661 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:08 smithi032 ceph-mon[29357]: pgmap v365: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:33:08.662 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:08 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3038970204' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:33:08.662 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:08 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/283734276' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:33:08.662 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:08 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1828036779' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:33:08.663 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-25T00:33:08.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.764+0000 7f58961e7640 1 -- 172.21.15.32:0/183346240 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5890104b90 msgr2=0x7f5890106f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:08.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.764+0000 7f58961e7640 1 --2- 172.21.15.32:0/183346240 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5890104b90 0x7f5890106f80 secure :-1 s=READY pgs=426 cs=0 l=1 rev1=1 crypto rx=0x7f5884009900 tx=0x7f588402a410 comp rx=0 tx=0).stop 2024-04-25T00:33:08.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.764+0000 7f58961e7640 1 -- 172.21.15.32:0/183346240 shutdown_connections 2024-04-25T00:33:08.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.764+0000 7f58961e7640 1 --2- 172.21.15.32:0/183346240 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f58901074c0 0x7f58901098d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.764+0000 7f58961e7640 1 --2- 172.21.15.32:0/183346240 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5890104b90 0x7f5890106f80 unknown :-1 s=CLOSED pgs=426 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.764+0000 7f58961e7640 1 --2- 172.21.15.32:0/183346240 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5890102260 0x7f5890104650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.764+0000 7f58961e7640 1 -- 172.21.15.32:0/183346240 >> 172.21.15.32:0/183346240 conn(0x7f58900fbc30 msgr2=0x7f58900fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:08.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.764+0000 7f58961e7640 1 -- 172.21.15.32:0/183346240 shutdown_connections 2024-04-25T00:33:08.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.764+0000 7f58961e7640 1 -- 172.21.15.32:0/183346240 wait complete. 2024-04-25T00:33:08.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.764+0000 7f58961e7640 1 Processor -- start 2024-04-25T00:33:08.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.765+0000 7f58961e7640 1 -- start start 2024-04-25T00:33:08.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.765+0000 7f58961e7640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5890102260 0x7f5890197b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:08.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.765+0000 7f58961e7640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5890104b90 0x7f5890198040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:08.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.765+0000 7f58961e7640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f58901074c0 0x7f589019f0b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:08.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.765+0000 7f58961e7640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f589010d420 con 0x7f5890102260 2024-04-25T00:33:08.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.765+0000 7f58961e7640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f589010d2a0 con 0x7f58901074c0 2024-04-25T00:33:08.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.765+0000 7f58961e7640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f589010d5a0 con 0x7f5890104b90 2024-04-25T00:33:08.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.765+0000 7f588f7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5890102260 0x7f5890197b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:08.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.765+0000 7f588f7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5890102260 0x7f5890197b00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44168/0 (socket says 172.21.15.32:44168) 2024-04-25T00:33:08.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.765+0000 7f588f7fe640 1 -- 172.21.15.32:0/4231447488 learned_addr learned my addr 172.21.15.32:0/4231447488 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:08.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.765+0000 7f588ffff640 1 --2- 172.21.15.32:0/4231447488 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f58901074c0 0x7f589019f0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:08.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.765+0000 7f588effd640 1 --2- 172.21.15.32:0/4231447488 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5890104b90 0x7f5890198040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:08.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.765+0000 7f588f7fe640 1 -- 172.21.15.32:0/4231447488 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5890104b90 msgr2=0x7f5890198040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:08.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.765+0000 7f588f7fe640 1 --2- 172.21.15.32:0/4231447488 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5890104b90 0x7f5890198040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.765+0000 7f588f7fe640 1 -- 172.21.15.32:0/4231447488 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f58901074c0 msgr2=0x7f589019f0b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:08.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.765+0000 7f588f7fe640 1 --2- 172.21.15.32:0/4231447488 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f58901074c0 0x7f589019f0b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.765+0000 7f588f7fe640 1 -- 172.21.15.32:0/4231447488 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f589019f720 con 0x7f5890102260 2024-04-25T00:33:08.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.766+0000 7f588f7fe640 1 --2- 172.21.15.32:0/4231447488 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5890102260 0x7f5890197b00 secure :-1 s=READY pgs=427 cs=0 l=1 rev1=1 crypto rx=0x7f588000d770 tx=0x7f588000dc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:08.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.766+0000 7f588cff9640 1 -- 172.21.15.32:0/4231447488 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5880017070 con 0x7f5890102260 2024-04-25T00:33:08.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.766+0000 7f58961e7640 1 -- 172.21.15.32:0/4231447488 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f589019fa10 con 0x7f5890102260 2024-04-25T00:33:08.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.766+0000 7f588cff9640 1 -- 172.21.15.32:0/4231447488 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5880005500 con 0x7f5890102260 2024-04-25T00:33:08.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.766+0000 7f588cff9640 1 -- 172.21.15.32:0/4231447488 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5880004e00 con 0x7f5890102260 2024-04-25T00:33:08.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.766+0000 7f58961e7640 1 -- 172.21.15.32:0/4231447488 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f589019ff30 con 0x7f5890102260 2024-04-25T00:33:08.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.767+0000 7f58961e7640 1 -- 172.21.15.32:0/4231447488 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f58900689b0 con 0x7f5890102260 2024-04-25T00:33:08.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.768+0000 7f588cff9640 1 -- 172.21.15.32:0/4231447488 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 18) ==== 137833+0+0 (secure 0 0 0) 0x7f5880004fa0 con 0x7f5890102260 2024-04-25T00:33:08.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.769+0000 7f588cff9640 1 --2- 172.21.15.32:0/4231447488 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f58780a2b90 0x7f58780a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:08.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.769+0000 7f588effd640 1 --2- 172.21.15.32:0/4231447488 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f58780a2b90 0x7f58780a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:08.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.769+0000 7f588cff9640 1 -- 172.21.15.32:0/4231447488 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7f58800cefa0 con 0x7f5890102260 2024-04-25T00:33:08.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.769+0000 7f588effd640 1 --2- 172.21.15.32:0/4231447488 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f58780a2b90 0x7f58780a5050 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f5884006010 tx=0x7f5884000bc0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:08.773 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.773+0000 7f588cff9640 1 -- 172.21.15.32:0/4231447488 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f588009bad0 con 0x7f5890102260 2024-04-25T00:33:08.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:08 smithi121 ceph-mon[32231]: pgmap v365: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:33:08.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:08 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3038970204' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:33:08.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:08 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/283734276' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:33:08.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:08 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1828036779' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:33:08.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:08 smithi130 ceph-mon[32525]: pgmap v365: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:33:08.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:08 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3038970204' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:33:08.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:08 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/283734276' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:33:08.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:08 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1828036779' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:33:08.963 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.962+0000 7f58961e7640 1 -- 172.21.15.32:0/4231447488 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f5890062340 con 0x7f5890102260 2024-04-25T00:33:08.964 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.964+0000 7f588cff9640 1 -- 172.21.15.32:0/4231447488 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v93) ==== 74+0+24843 (secure 0 0 0) 0x7f588009b3d0 con 0x7f5890102260 2024-04-25T00:33:08.965 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:33:08.965 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":93,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:32:45.458564+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-25T00:30:46.239556+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-25T00:30:47.257182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2024-04-25T00:30:51.193975+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.b","mappings":[{"from":8,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":1}]},{"pgid":"2.f","mappings":[{"from":8,"to":11}]},{"pgid":"2.10","mappings":[{"from":2,"to":4}]},{"pgid":"3.6","mappings":[{"from":3,"to":6}]},{"pgid":"3.17","mappings":[{"from":4,"to":7}]},{"pgid":"3.1b","mappings":[{"from":5,"to":9}]},{"pgid":"3.20","mappings":[{"from":5,"to":9}]},{"pgid":"3.23","mappings":[{"from":4,"to":8}]},{"pgid":"3.2c","mappings":[{"from":4,"to":9}]},{"pgid":"4.c","mappings":[{"from":9,"to":10}]},{"pgid":"4.12","mappings":[{"from":9,"to":10}]},{"pgid":"4.1c","mappings":[{"from":9,"to":8}]},{"pgid":"4.1d","mappings":[{"from":5,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-num-failure-domains":"0","crush-osds-per-failure-domain":"0","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:33:08.967 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.966+0000 7f58961e7640 1 -- 172.21.15.32:0/4231447488 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f58780a2b90 msgr2=0x7f58780a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:08.967 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.966+0000 7f58961e7640 1 --2- 172.21.15.32:0/4231447488 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f58780a2b90 0x7f58780a5050 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f5884006010 tx=0x7f5884000bc0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.967 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.966+0000 7f58961e7640 1 -- 172.21.15.32:0/4231447488 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5890102260 msgr2=0x7f5890197b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:08.967 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.966+0000 7f58961e7640 1 --2- 172.21.15.32:0/4231447488 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5890102260 0x7f5890197b00 secure :-1 s=READY pgs=427 cs=0 l=1 rev1=1 crypto rx=0x7f588000d770 tx=0x7f588000dc30 comp rx=0 tx=0).stop 2024-04-25T00:33:08.967 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.966+0000 7f58961e7640 1 -- 172.21.15.32:0/4231447488 shutdown_connections 2024-04-25T00:33:08.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.966+0000 7f58961e7640 1 --2- 172.21.15.32:0/4231447488 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f58780a2b90 0x7f58780a5050 unknown :-1 s=CLOSED pgs=258 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.966+0000 7f58961e7640 1 --2- 172.21.15.32:0/4231447488 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f58901074c0 0x7f589019f0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.966+0000 7f58961e7640 1 --2- 172.21.15.32:0/4231447488 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5890104b90 0x7f5890198040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.966+0000 7f58961e7640 1 --2- 172.21.15.32:0/4231447488 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5890102260 0x7f5890197b00 unknown :-1 s=CLOSED pgs=427 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:08.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.966+0000 7f58961e7640 1 -- 172.21.15.32:0/4231447488 >> 172.21.15.32:0/4231447488 conn(0x7f58900fbc30 msgr2=0x7f5890102b60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:08.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.966+0000 7f58961e7640 1 -- 172.21.15.32:0/4231447488 shutdown_connections 2024-04-25T00:33:08.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:08.966+0000 7f58961e7640 1 -- 172.21.15.32:0/4231447488 wait complete. 2024-04-25T00:33:08.979 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 24, 'flags': 50, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': True, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 32, 'explicitly_allowed_features': 32, 'created': '2024-04-25T00:30:47.866875+0000', 'modified': '2024-04-25T00:31:14.972514+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 300, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}, 'max_mds': 3, 'in': [0, 1, 2], 'up': {'mds_0': 24506, 'mds_1': 24481, 'mds_2': 24476}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_24443': {'gid': 24443, 'name': 'e', 'rank': 2, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.121:6835/271690252', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.121:6834', 'nonce': 271690252}, {'type': 'v1', 'addr': '172.21.15.121:6835', 'nonce': 271690252}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24445': {'gid': 24445, 'name': 'f', 'rank': 1, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.130:6835/2892666533', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.130:6834', 'nonce': 2892666533}, {'type': 'v1', 'addr': '172.21.15.130:6835', 'nonce': 2892666533}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24457': {'gid': 24457, 'name': 'c', 'rank': 0, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.130:6837/1920876316', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.130:6836', 'nonce': 1920876316}, {'type': 'v1', 'addr': '172.21.15.130:6837', 'nonce': 1920876316}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24476': {'gid': 24476, 'name': 'h', 'rank': 2, 'incarnation': 22, 'state': 'up:active', 'state_seq': 12, 'addr': '172.21.15.121:6837/3456022860', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.121:6836', 'nonce': 3456022860}, {'type': 'v1', 'addr': '172.21.15.121:6837', 'nonce': 3456022860}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24481': {'gid': 24481, 'name': 'i', 'rank': 1, 'incarnation': 20, 'state': 'up:active', 'state_seq': 11, 'addr': '172.21.15.130:6839/4267125809', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.130:6838', 'nonce': 4267125809}, {'type': 'v1', 'addr': '172.21.15.130:6839', 'nonce': 4267125809}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24506': {'gid': 24506, 'name': 'b', 'rank': 0, 'incarnation': 14, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.121:6839/258444853', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.121:6838', 'nonce': 258444853}, {'type': 'v1', 'addr': '172.21.15.121:6839', 'nonce': 258444853}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}}, 'data_pools': [3, 4], 'metadata_pool': 2, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 24506, 'qdb_cluster': [24476, 24481, 24506]} 2024-04-25T00:33:08.979 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 3/3 2024-04-25T00:33:08.979 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.h status 2024-04-25T00:33:09.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.081+0000 7fe0ee590640 1 -- 172.21.15.32:0/2448936019 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe0e8102240 msgr2=0x7fe0e8104630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.084 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.081+0000 7fe0ee590640 1 --2- 172.21.15.32:0/2448936019 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe0e8102240 0x7fe0e8104630 secure :-1 s=READY pgs=428 cs=0 l=1 rev1=1 crypto rx=0x7fe0d8009900 tx=0x7fe0d802a410 comp rx=0 tx=0).stop 2024-04-25T00:33:09.084 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.081+0000 7fe0ee590640 1 -- 172.21.15.32:0/2448936019 shutdown_connections 2024-04-25T00:33:09.084 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.081+0000 7fe0ee590640 1 --2- 172.21.15.32:0/2448936019 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe0e81074a0 0x7fe0e81098b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.084 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.081+0000 7fe0ee590640 1 --2- 172.21.15.32:0/2448936019 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe0e8104b70 0x7fe0e8106f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.084 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.081+0000 7fe0ee590640 1 --2- 172.21.15.32:0/2448936019 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe0e8102240 0x7fe0e8104630 unknown :-1 s=CLOSED pgs=428 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.084 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.081+0000 7fe0ee590640 1 -- 172.21.15.32:0/2448936019 >> 172.21.15.32:0/2448936019 conn(0x7fe0e80fbbf0 msgr2=0x7fe0e80fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:09.084 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.081+0000 7fe0ee590640 1 -- 172.21.15.32:0/2448936019 shutdown_connections 2024-04-25T00:33:09.084 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.081+0000 7fe0ee590640 1 -- 172.21.15.32:0/2448936019 wait complete. 2024-04-25T00:33:09.084 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.082+0000 7fe0ee590640 1 Processor -- start 2024-04-25T00:33:09.084 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.082+0000 7fe0ee590640 1 -- start start 2024-04-25T00:33:09.084 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.082+0000 7fe0ee590640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe0e8102240 0x7fe0e8197b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.082+0000 7fe0ee590640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe0e8104b70 0x7fe0e8198060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.082+0000 7fe0ee590640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe0e81074a0 0x7fe0e819f0d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.082+0000 7fe0ee590640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fe0e810d420 con 0x7fe0e81074a0 2024-04-25T00:33:09.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.082+0000 7fe0ee590640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fe0e810d2a0 con 0x7fe0e8104b70 2024-04-25T00:33:09.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.082+0000 7fe0ee590640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fe0e810d5a0 con 0x7fe0e8102240 2024-04-25T00:33:09.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.082+0000 7fe0ecb2d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe0e81074a0 0x7fe0e819f0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.082+0000 7fe0ecb2d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe0e81074a0 0x7fe0e819f0d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44190/0 (socket says 172.21.15.32:44190) 2024-04-25T00:33:09.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.082+0000 7fe0ecb2d640 1 -- 172.21.15.32:0/2872124476 learned_addr learned my addr 172.21.15.32:0/2872124476 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:09.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.082+0000 7fe0e7fff640 1 --2- 172.21.15.32:0/2872124476 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe0e8102240 0x7fe0e8197b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.082+0000 7fe0e77fe640 1 --2- 172.21.15.32:0/2872124476 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe0e8104b70 0x7fe0e8198060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.082+0000 7fe0ecb2d640 1 -- 172.21.15.32:0/2872124476 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe0e8102240 msgr2=0x7fe0e8197b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.082+0000 7fe0ecb2d640 1 --2- 172.21.15.32:0/2872124476 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe0e8102240 0x7fe0e8197b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.082+0000 7fe0ecb2d640 1 -- 172.21.15.32:0/2872124476 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe0e8104b70 msgr2=0x7fe0e8198060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.082+0000 7fe0ecb2d640 1 --2- 172.21.15.32:0/2872124476 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe0e8104b70 0x7fe0e8198060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.086 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.082+0000 7fe0ecb2d640 1 -- 172.21.15.32:0/2872124476 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe0e819f740 con 0x7fe0e81074a0 2024-04-25T00:33:09.086 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.082+0000 7fe0e7fff640 1 --2- 172.21.15.32:0/2872124476 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe0e8102240 0x7fe0e8197b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:09.086 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.082+0000 7fe0e77fe640 1 --2- 172.21.15.32:0/2872124476 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe0e8104b70 0x7fe0e8198060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:09.086 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.083+0000 7fe0ecb2d640 1 --2- 172.21.15.32:0/2872124476 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe0e81074a0 0x7fe0e819f0d0 secure :-1 s=READY pgs=429 cs=0 l=1 rev1=1 crypto rx=0x7fe0e000eaa0 tx=0x7fe0e000ef60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:09.086 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.083+0000 7fe0e57fa640 1 -- 172.21.15.32:0/2872124476 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe0e000cdb0 con 0x7fe0e81074a0 2024-04-25T00:33:09.086 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.083+0000 7fe0e57fa640 1 -- 172.21.15.32:0/2872124476 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe0e0005020 con 0x7fe0e81074a0 2024-04-25T00:33:09.086 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.083+0000 7fe0ee590640 1 -- 172.21.15.32:0/2872124476 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe0e819fa30 con 0x7fe0e81074a0 2024-04-25T00:33:09.086 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.083+0000 7fe0e57fa640 1 -- 172.21.15.32:0/2872124476 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe0e00116b0 con 0x7fe0e81074a0 2024-04-25T00:33:09.086 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.083+0000 7fe0ee590640 1 -- 172.21.15.32:0/2872124476 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe0e819ff30 con 0x7fe0e81074a0 2024-04-25T00:33:09.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.086+0000 7fe0e57fa640 1 -- 172.21.15.32:0/2872124476 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 18) ==== 137833+0+0 (secure 0 0 0) 0x7fe0e0005560 con 0x7fe0e81074a0 2024-04-25T00:33:09.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.086+0000 7fe0f1378740 1 Processor -- start 2024-04-25T00:33:09.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.086+0000 7fe0f1378740 1 -- start start 2024-04-25T00:33:09.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.086+0000 7fe0f1378740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56533675b440 0x56533675b860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.086+0000 7fe0f1378740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56533675be30 0x565336760930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.086+0000 7fe0f1378740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x565336760fa0 0x565336763430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.086+0000 7fe0f1378740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5653366bef00 con 0x56533675b440 2024-04-25T00:33:09.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.086+0000 7fe0f1378740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x565336667750 con 0x565336760fa0 2024-04-25T00:33:09.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.086+0000 7fe0f1378740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5653366baec0 con 0x56533675be30 2024-04-25T00:33:09.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.086+0000 7fe0e77fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56533675b440 0x56533675b860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.086+0000 7fe0e77fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56533675b440 0x56533675b860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44196/0 (socket says 172.21.15.32:44196) 2024-04-25T00:33:09.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.086+0000 7fe0e57fa640 1 --2- 172.21.15.32:0/2872124476 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe0d40a2b90 0x7fe0d40a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.086+0000 7fe0e77fe640 1 -- 172.21.15.32:0/3089105260 learned_addr learned my addr 172.21.15.32:0/3089105260 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:09.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.086+0000 7fe0e7fff640 1 --2- 172.21.15.32:0/3089105260 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56533675be30 0x565336760930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.086+0000 7fe0ecb2d640 1 --2- 172.21.15.32:0/2872124476 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe0d40a2b90 0x7fe0d40a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.086+0000 7fe0e77fe640 1 --2- 172.21.15.32:0/3089105260 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x565336760fa0 0x565336763430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.086+0000 7fe0e57fa640 1 -- 172.21.15.32:0/2872124476 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7fe0e00cf710 con 0x7fe0e81074a0 2024-04-25T00:33:09.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.086+0000 7fe0e77fe640 1 -- 172.21.15.32:0/3089105260 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56533675be30 msgr2=0x565336760930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:09.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.086+0000 7fe0e77fe640 1 --2- 172.21.15.32:0/3089105260 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56533675be30 0x565336760930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.086+0000 7fe0e77fe640 1 -- 172.21.15.32:0/3089105260 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x565336760fa0 msgr2=0x565336763430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:09.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.086+0000 7fe0e77fe640 1 --2- 172.21.15.32:0/3089105260 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x565336760fa0 0x565336763430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.087+0000 7fe0e77fe640 1 -- 172.21.15.32:0/3089105260 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5653363d32d0 con 0x56533675b440 2024-04-25T00:33:09.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.087+0000 7fe0e77fe640 1 --2- 172.21.15.32:0/3089105260 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56533675b440 0x56533675b860 secure :-1 s=READY pgs=430 cs=0 l=1 rev1=1 crypto rx=0x7fe0d0013610 tx=0x7fe0d000b6f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a2a83f609d13676e server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:09.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.087+0000 7fe0ecb2d640 1 --2- 172.21.15.32:0/2872124476 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe0d40a2b90 0x7fe0d40a5050 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7fe0e00cd1b0 tx=0x7fe0e006ec60 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:09.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.087+0000 7fe0de7fc640 1 -- 172.21.15.32:0/3089105260 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe0d0019030 con 0x56533675b440 2024-04-25T00:33:09.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.087+0000 7fe0de7fc640 1 -- 172.21.15.32:0/3089105260 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe0d0005500 con 0x56533675b440 2024-04-25T00:33:09.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.087+0000 7fe0de7fc640 1 -- 172.21.15.32:0/3089105260 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe0d00117a0 con 0x56533675b440 2024-04-25T00:33:09.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.088+0000 7fe0f1378740 1 -- 172.21.15.32:0/3089105260 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56533675b440 msgr2=0x56533675b860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.088+0000 7fe0f1378740 1 --2- 172.21.15.32:0/3089105260 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56533675b440 0x56533675b860 secure :-1 s=READY pgs=430 cs=0 l=1 rev1=1 crypto rx=0x7fe0d0013610 tx=0x7fe0d000b6f0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.088+0000 7fe0f1378740 1 -- 172.21.15.32:0/3089105260 shutdown_connections 2024-04-25T00:33:09.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.088+0000 7fe0f1378740 1 --2- 172.21.15.32:0/3089105260 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x565336760fa0 0x565336763430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.088+0000 7fe0f1378740 1 --2- 172.21.15.32:0/3089105260 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56533675be30 0x565336760930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.088+0000 7fe0f1378740 1 --2- 172.21.15.32:0/3089105260 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56533675b440 0x56533675b860 unknown :-1 s=CLOSED pgs=430 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.088+0000 7fe0f1378740 1 -- 172.21.15.32:0/3089105260 >> 172.21.15.32:0/3089105260 conn(0x565336756ab0 msgr2=0x565336756eb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:09.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.088+0000 7fe0f1378740 1 -- 172.21.15.32:0/3089105260 shutdown_connections 2024-04-25T00:33:09.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.088+0000 7fe0f1378740 1 -- 172.21.15.32:0/3089105260 wait complete. 2024-04-25T00:33:09.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.088+0000 7fe0f1378740 1 -- start start 2024-04-25T00:33:09.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.088+0000 7fe0f1378740 1 Processor -- start 2024-04-25T00:33:09.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.088+0000 7fe0f1378740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56533675b440 0x565336765810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.088+0000 7fe0f1378740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56533675be30 0x565336765d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.088+0000 7fe0f1378740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x565336760fa0 0x56533676cd90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.088+0000 7fe0f1378740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x565336763f50 con 0x56533675be30 2024-04-25T00:33:09.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.088+0000 7fe0f1378740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x56533667acf0 con 0x565336760fa0 2024-04-25T00:33:09.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.088+0000 7fe0f1378740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5653367640d0 con 0x56533675b440 2024-04-25T00:33:09.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.089+0000 7fe0f1378740 10 client.0 _ll_register_callbacks cb 0x56533667a6e0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:33:09.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.089+0000 7fe0e7fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56533675be30 0x565336765d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.089+0000 7fe0e7fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56533675be30 0x565336765d50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44212/0 (socket says 172.21.15.32:44212) 2024-04-25T00:33:09.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.089+0000 7fe0e7fff640 1 -- 172.21.15.32:0/2120499733 learned_addr learned my addr 172.21.15.32:0/2120499733 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:09.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.089+0000 7fe0e7fff640 1 -- 172.21.15.32:0/2120499733 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56533675b440 msgr2=0x565336765810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:09.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.089+0000 7fe0e7fff640 1 --2- 172.21.15.32:0/2120499733 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56533675b440 0x565336765810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.089+0000 7fe0e7fff640 1 -- 172.21.15.32:0/2120499733 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x565336760fa0 msgr2=0x56533676cd90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:09.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.089+0000 7fe0e7fff640 1 --2- 172.21.15.32:0/2120499733 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x565336760fa0 0x56533676cd90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.089+0000 7fe0e7fff640 1 -- 172.21.15.32:0/2120499733 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56533649d6b0 con 0x56533675be30 2024-04-25T00:33:09.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.089+0000 7fe0e7fff640 1 --2- 172.21.15.32:0/2120499733 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56533675be30 0x565336765d50 secure :-1 s=READY pgs=431 cs=0 l=1 rev1=1 crypto rx=0x7fe0d802a930 tx=0x7fe0d8031a20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7845d28afeedc0cb server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:09.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.089+0000 7fe0c77fe640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:33:09.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.089+0000 7fe0c77fe640 1 -- 172.21.15.32:0/2120499733 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe0d8035040 con 0x56533675be30 2024-04-25T00:33:09.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.089+0000 7fe0c77fe640 1 -- 172.21.15.32:0/2120499733 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7fe0d8004380 con 0x56533675be30 2024-04-25T00:33:09.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.089+0000 7fe0f1378740 1 -- 172.21.15.32:0/2120499733 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe0d0005500 con 0x56533675be30 2024-04-25T00:33:09.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.089+0000 7fe0c77fe640 1 -- 172.21.15.32:0/2120499733 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) ==== 24+0+0 (secure 0 0 0) 0x7fe0d8005030 con 0x56533675be30 2024-04-25T00:33:09.097 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.089+0000 7fe0f1378740 10 client.15417 fetch_fsmap learned FSMap version 24 2024-04-25T00:33:09.097 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.089+0000 7fe0f1378740 1 -- 172.21.15.32:0/2120499733 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) -- 0x56533676fde0 con 0x56533675be30 2024-04-25T00:33:09.097 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.089+0000 7fe0c77fe640 1 -- 172.21.15.32:0/2120499733 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe0d0005500 con 0x56533675be30 2024-04-25T00:33:09.097 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.090+0000 7fe0c77fe640 1 -- 172.21.15.32:0/2120499733 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 24) ==== 8346+0+0 (secure 0 0 0) 0x7fe0d80035f0 con 0x56533675be30 2024-04-25T00:33:09.097 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.090+0000 7fe0f1378740 10 client.15417 fetch_fsmap finished waiting for FSMap version 24 2024-04-25T00:33:09.097 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.090+0000 7fe0f1378740 10 client.15417 resolve_mds: resolved name 'h' to daemon mds.h 2024-04-25T00:33:09.097 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.090+0000 7fe0f1378740 20 client.15417 populate_metadata read hostname 'smithi032' 2024-04-25T00:33:09.097 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.090+0000 7fe0f1378740 1 --2- 172.21.15.32:0/2120499733 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x565336772450 0x5653367738c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.097 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.090+0000 7fe0f1378740 4 client.15417 mds_command: new command op to 24476 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:33:09.097 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.090+0000 7fe0f1378740 1 -- 172.21.15.32:0/2120499733 --> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5653366cc440 con 0x565336772450 2024-04-25T00:33:09.097 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.090+0000 7fe0e77fe640 1 --2- 172.21.15.32:0/2120499733 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x565336772450 0x5653367738c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.097 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.090+0000 7fe0e77fe640 1 --2- 172.21.15.32:0/2120499733 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x565336772450 0x5653367738c0 crc :-1 s=READY pgs=18 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=a6f459e84ca66d81 server_cookie=3e163fa1d4ff0dd0 in_seq=0 out_seq=0 2024-04-25T00:33:09.097 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.090+0000 7fe0c77fe640 10 client.15417 ms_handle_connect on v2:172.21.15.121:6836/3456022860 2024-04-25T00:33:09.097 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.091+0000 7fe0c77fe640 1 -- 172.21.15.32:0/2120499733 <== mds.2 v2:172.21.15.121:6836/3456022860 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x5653366cc440 con 0x565336772450 2024-04-25T00:33:09.098 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.091+0000 7fe0c77fe640 10 client.15417 handle_command_reply: tid=0 2024-04-25T00:33:09.098 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.091+0000 7fe0f1378740 1 client.15417 shutdown 2024-04-25T00:33:09.098 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.091+0000 7fe0f1378740 1 -- 172.21.15.32:0/2120499733 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56533675be30 msgr2=0x565336765d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.098 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.091+0000 7fe0f1378740 1 --2- 172.21.15.32:0/2120499733 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56533675be30 0x565336765d50 secure :-1 s=READY pgs=431 cs=0 l=1 rev1=1 crypto rx=0x7fe0d802a930 tx=0x7fe0d8031a20 comp rx=0 tx=0).stop 2024-04-25T00:33:09.098 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.091+0000 7fe0f1378740 1 -- 172.21.15.32:0/2120499733 shutdown_connections 2024-04-25T00:33:09.098 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.091+0000 7fe0f1378740 1 --2- 172.21.15.32:0/2120499733 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x565336772450 0x5653367738c0 crc :-1 s=READY pgs=18 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.098 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.091+0000 7fe0f1378740 1 --2- 172.21.15.32:0/2120499733 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x565336760fa0 0x56533676cd90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.098 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.091+0000 7fe0f1378740 1 --2- 172.21.15.32:0/2120499733 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56533675be30 0x565336765d50 unknown :-1 s=CLOSED pgs=431 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.098 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.091+0000 7fe0c77fe640 0 client.15417 ms_handle_reset on v2:172.21.15.121:6836/3456022860 2024-04-25T00:33:09.098 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.091+0000 7fe0f1378740 1 --2- 172.21.15.32:0/2120499733 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56533675b440 0x565336765810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.098 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.091+0000 7fe0f1378740 1 -- 172.21.15.32:0/2120499733 >> 172.21.15.32:0/2120499733 conn(0x565336756ab0 msgr2=0x565336755ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:09.098 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.092+0000 7fe0f1378740 1 -- 172.21.15.32:0/2120499733 shutdown_connections 2024-04-25T00:33:09.098 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.092+0000 7fe0f1378740 1 -- 172.21.15.32:0/2120499733 wait complete. 2024-04-25T00:33:09.099 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.092+0000 7fe0f1378740 20 client.15417 trim_cache size 0 max 16384 2024-04-25T00:33:09.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.110+0000 7fe0f1378740 1 Processor -- start 2024-04-25T00:33:09.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.110+0000 7fe0f1378740 1 -- start start 2024-04-25T00:33:09.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.110+0000 7fe0f1378740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56533675b440 0x565336760860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.110+0000 7fe0f1378740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56533675be30 0x56533676cd50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.110+0000 7fe0f1378740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x565336760fa0 0x56533675c6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.110+0000 7fe0f1378740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x565336771c00 con 0x565336760fa0 2024-04-25T00:33:09.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.110+0000 7fe0f1378740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5653367722d0 con 0x56533675be30 2024-04-25T00:33:09.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.110+0000 7fe0f1378740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5653367729a0 con 0x56533675b440 2024-04-25T00:33:09.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.110+0000 7fe0e77fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x565336760fa0 0x56533675c6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.110+0000 7fe0e77fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x565336760fa0 0x56533675c6c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44228/0 (socket says 172.21.15.32:44228) 2024-04-25T00:33:09.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.110+0000 7fe0e77fe640 1 -- 172.21.15.32:0/2049304534 learned_addr learned my addr 172.21.15.32:0/2049304534 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:09.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.110+0000 7fe0e7fff640 1 --2- 172.21.15.32:0/2049304534 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56533675be30 0x56533676cd50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.110+0000 7fe0e77fe640 1 -- 172.21.15.32:0/2049304534 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56533675b440 msgr2=0x565336760860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:09.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.110+0000 7fe0e77fe640 1 --2- 172.21.15.32:0/2049304534 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56533675b440 0x565336760860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.110+0000 7fe0e77fe640 1 -- 172.21.15.32:0/2049304534 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56533675be30 msgr2=0x56533676cd50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:09.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.110+0000 7fe0e77fe640 1 --2- 172.21.15.32:0/2049304534 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56533675be30 0x56533676cd50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.110+0000 7fe0e77fe640 1 -- 172.21.15.32:0/2049304534 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5653367568e0 con 0x565336760fa0 2024-04-25T00:33:09.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.110+0000 7fe0e7fff640 1 --2- 172.21.15.32:0/2049304534 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56533675be30 0x56533676cd50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:33:09.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.111+0000 7fe0e77fe640 1 --2- 172.21.15.32:0/2049304534 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x565336760fa0 0x56533675c6c0 secure :-1 s=READY pgs=432 cs=0 l=1 rev1=1 crypto rx=0x56533675c690 tx=0x7fe0d000bf40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b423d6406901b61d server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:09.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.111+0000 7fe0ddffb640 1 -- 172.21.15.32:0/2049304534 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe0d0019040 con 0x565336760fa0 2024-04-25T00:33:09.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.111+0000 7fe0ddffb640 1 -- 172.21.15.32:0/2049304534 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe0d001a040 con 0x565336760fa0 2024-04-25T00:33:09.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.111+0000 7fe0ddffb640 1 -- 172.21.15.32:0/2049304534 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe0d00119c0 con 0x565336760fa0 2024-04-25T00:33:09.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.111+0000 7fe0f1378740 1 -- 172.21.15.32:0/2049304534 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x565336760fa0 msgr2=0x56533675c6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.111+0000 7fe0f1378740 1 --2- 172.21.15.32:0/2049304534 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x565336760fa0 0x56533675c6c0 secure :-1 s=READY pgs=432 cs=0 l=1 rev1=1 crypto rx=0x56533675c690 tx=0x7fe0d000bf40 comp rx=0 tx=0).stop 2024-04-25T00:33:09.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.111+0000 7fe0f1378740 1 -- 172.21.15.32:0/2049304534 shutdown_connections 2024-04-25T00:33:09.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.111+0000 7fe0f1378740 1 --2- 172.21.15.32:0/2049304534 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x565336760fa0 0x56533675c6c0 unknown :-1 s=CLOSED pgs=432 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.111+0000 7fe0f1378740 1 --2- 172.21.15.32:0/2049304534 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56533675be30 0x56533676cd50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.111+0000 7fe0f1378740 1 --2- 172.21.15.32:0/2049304534 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56533675b440 0x565336760860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.111+0000 7fe0f1378740 1 -- 172.21.15.32:0/2049304534 >> 172.21.15.32:0/2049304534 conn(0x565336756ab0 msgr2=0x565336751690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:09.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.111+0000 7fe0f1378740 1 -- 172.21.15.32:0/2049304534 shutdown_connections 2024-04-25T00:33:09.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.111+0000 7fe0f1378740 1 -- 172.21.15.32:0/2049304534 wait complete. 2024-04-25T00:33:09.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.112+0000 7fe0f1378740 1 -- start start 2024-04-25T00:33:09.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.112+0000 7fe0f1378740 1 Processor -- start 2024-04-25T00:33:09.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.112+0000 7fe0f1378740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56533675b440 0x565336768f80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.112+0000 7fe0f1378740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56533675be30 0x5653367694c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.112+0000 7fe0f1378740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x565336760fa0 0x56533676bd90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.112+0000 7fe0f1378740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x565336773740 con 0x565336760fa0 2024-04-25T00:33:09.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.112+0000 7fe0f1378740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x565336773070 con 0x56533675be30 2024-04-25T00:33:09.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.112+0000 7fe0f1378740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x565336773e10 con 0x56533675b440 2024-04-25T00:33:09.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.112+0000 7fe0f1378740 10 client.0 _ll_register_callbacks cb 0x7fe0d8005d50 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:33:09.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.112+0000 7fe0e77fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x565336760fa0 0x56533676bd90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.112+0000 7fe0e77fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x565336760fa0 0x56533676bd90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44232/0 (socket says 172.21.15.32:44232) 2024-04-25T00:33:09.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.112+0000 7fe0e77fe640 1 -- 172.21.15.32:0/236948879 learned_addr learned my addr 172.21.15.32:0/236948879 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:09.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.112+0000 7fe0e7fff640 1 --2- 172.21.15.32:0/236948879 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56533675be30 0x5653367694c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.112+0000 7fe0e77fe640 1 --2- 172.21.15.32:0/236948879 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56533675b440 0x565336768f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.112+0000 7fe0e77fe640 1 -- 172.21.15.32:0/236948879 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56533675b440 msgr2=0x565336768f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:09.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.113+0000 7fe0e77fe640 1 --2- 172.21.15.32:0/236948879 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56533675b440 0x565336768f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.113+0000 7fe0e77fe640 1 -- 172.21.15.32:0/236948879 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56533675be30 msgr2=0x5653367694c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:09.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.113+0000 7fe0e77fe640 1 --2- 172.21.15.32:0/236948879 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56533675be30 0x5653367694c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.113+0000 7fe0e77fe640 1 -- 172.21.15.32:0/236948879 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56533676c3d0 con 0x565336760fa0 2024-04-25T00:33:09.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.113+0000 7fe0e77fe640 1 --2- 172.21.15.32:0/236948879 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x565336760fa0 0x56533676bd90 secure :-1 s=READY pgs=433 cs=0 l=1 rev1=1 crypto rx=0x7fe0d0013fd0 tx=0x7fe0d0005970 comp rx=0 tx=0).ready entity=mon.0 client_cookie=eb8236f0b5c4b6ba server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:09.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.113+0000 7fe0c77fe640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:33:09.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.113+0000 7fe0c77fe640 1 -- 172.21.15.32:0/236948879 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe0d0013020 con 0x565336760fa0 2024-04-25T00:33:09.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.113+0000 7fe0c77fe640 1 -- 172.21.15.32:0/236948879 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7fe0d00131c0 con 0x565336760fa0 2024-04-25T00:33:09.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.113+0000 7fe0f1378740 1 -- 172.21.15.32:0/236948879 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe0d001a040 con 0x565336760fa0 2024-04-25T00:33:09.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.113+0000 7fe0c77fe640 1 -- 172.21.15.32:0/236948879 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) ==== 24+0+0 (secure 0 0 0) 0x7fe0d002a390 con 0x565336760fa0 2024-04-25T00:33:09.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.113+0000 7fe0f1378740 10 client.15423 fetch_fsmap learned FSMap version 24 2024-04-25T00:33:09.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.113+0000 7fe0f1378740 1 -- 172.21.15.32:0/236948879 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) -- 0x5653367d8750 con 0x565336760fa0 2024-04-25T00:33:09.125 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.113+0000 7fe0c77fe640 1 -- 172.21.15.32:0/236948879 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe0d001a040 con 0x565336760fa0 2024-04-25T00:33:09.125 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.113+0000 7fe0c77fe640 1 -- 172.21.15.32:0/236948879 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 24) ==== 8346+0+0 (secure 0 0 0) 0x7fe0d001e460 con 0x565336760fa0 2024-04-25T00:33:09.125 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.113+0000 7fe0f1378740 10 client.15423 fetch_fsmap finished waiting for FSMap version 24 2024-04-25T00:33:09.125 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.113+0000 7fe0f1378740 10 client.15423 resolve_mds: resolved name 'h' to daemon mds.h 2024-04-25T00:33:09.125 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.113+0000 7fe0f1378740 20 client.15423 populate_metadata read hostname 'smithi032' 2024-04-25T00:33:09.125 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.113+0000 7fe0f1378740 1 --2- 172.21.15.32:0/236948879 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x5653367dcbd0 0x5653367db8e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.125 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.113+0000 7fe0f1378740 4 client.15423 mds_command: new command op to 24476 tid=0 multi_id=0 [{"prefix": "status"}] 2024-04-25T00:33:09.125 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.113+0000 7fe0f1378740 1 -- 172.21.15.32:0/236948879 --> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] -- command(tid 0: {"prefix": "status"}) -- 0x5653366c6790 con 0x5653367dcbd0 2024-04-25T00:33:09.125 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.114+0000 7fe0e7fff640 1 --2- 172.21.15.32:0/236948879 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x5653367dcbd0 0x5653367db8e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.125 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.114+0000 7fe0e7fff640 1 --2- 172.21.15.32:0/236948879 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x5653367dcbd0 0x5653367db8e0 crc :-1 s=READY pgs=19 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=fce781a31245f951 server_cookie=e00c3c84868503c1 in_seq=0 out_seq=0 2024-04-25T00:33:09.125 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.114+0000 7fe0c77fe640 10 client.15423 ms_handle_connect on v2:172.21.15.121:6836/3456022860 2024-04-25T00:33:09.125 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.114+0000 7fe0c77fe640 1 -- 172.21.15.32:0/236948879 <== mds.2 v2:172.21.15.121:6836/3456022860 1 ==== command_reply(tid 0: 0 ) ==== 8+0+332 (crc 0 0 0) 0x5653366c6790 con 0x5653367dcbd0 2024-04-25T00:33:09.125 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.114+0000 7fe0c77fe640 10 client.15423 handle_command_reply: tid=0 2024-04-25T00:33:09.125 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.114+0000 7fe0f1378740 1 client.15423 shutdown 2024-04-25T00:33:09.125 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.114+0000 7fe0f1378740 1 -- 172.21.15.32:0/236948879 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x565336760fa0 msgr2=0x56533676bd90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.126 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.114+0000 7fe0f1378740 1 --2- 172.21.15.32:0/236948879 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x565336760fa0 0x56533676bd90 secure :-1 s=READY pgs=433 cs=0 l=1 rev1=1 crypto rx=0x7fe0d0013fd0 tx=0x7fe0d0005970 comp rx=0 tx=0).stop 2024-04-25T00:33:09.126 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.114+0000 7fe0f1378740 1 -- 172.21.15.32:0/236948879 shutdown_connections 2024-04-25T00:33:09.126 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.114+0000 7fe0f1378740 1 --2- 172.21.15.32:0/236948879 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x5653367dcbd0 0x5653367db8e0 crc :-1 s=READY pgs=19 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.126 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.114+0000 7fe0f1378740 1 --2- 172.21.15.32:0/236948879 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x565336760fa0 0x56533676bd90 unknown :-1 s=CLOSED pgs=433 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.126 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.114+0000 7fe0f1378740 1 --2- 172.21.15.32:0/236948879 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56533675be30 0x5653367694c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.126 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.114+0000 7fe0c77fe640 0 client.15423 ms_handle_reset on v2:172.21.15.121:6836/3456022860 2024-04-25T00:33:09.126 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.114+0000 7fe0f1378740 1 --2- 172.21.15.32:0/236948879 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56533675b440 0x565336768f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.126 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.114+0000 7fe0f1378740 1 -- 172.21.15.32:0/236948879 >> 172.21.15.32:0/236948879 conn(0x565336756ab0 msgr2=0x56533675c6c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:09.126 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.115+0000 7fe0f1378740 1 -- 172.21.15.32:0/236948879 shutdown_connections 2024-04-25T00:33:09.126 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.115+0000 7fe0f1378740 1 -- 172.21.15.32:0/236948879 wait complete. 2024-04-25T00:33:09.126 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.115+0000 7fe0f1378740 20 client.15423 trim_cache size 0 max 16384 2024-04-25T00:33:09.127 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:33:09.127 INFO:teuthology.orchestra.run.smithi032.stdout: "cluster_fsid": "cb997156-0299-11ef-bc93-c7b262605968", 2024-04-25T00:33:09.127 INFO:teuthology.orchestra.run.smithi032.stdout: "whoami": 2, 2024-04-25T00:33:09.127 INFO:teuthology.orchestra.run.smithi032.stdout: "id": 24476, 2024-04-25T00:33:09.127 INFO:teuthology.orchestra.run.smithi032.stdout: "want_state": "up:active", 2024-04-25T00:33:09.127 INFO:teuthology.orchestra.run.smithi032.stdout: "state": "up:active", 2024-04-25T00:33:09.127 INFO:teuthology.orchestra.run.smithi032.stdout: "fs_name": "cephfs", 2024-04-25T00:33:09.127 INFO:teuthology.orchestra.run.smithi032.stdout: "rank_uptime": 115.15014493699999, 2024-04-25T00:33:09.127 INFO:teuthology.orchestra.run.smithi032.stdout: "mdsmap_epoch": 24, 2024-04-25T00:33:09.127 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_epoch": 93, 2024-04-25T00:33:09.127 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_epoch_barrier": 89, 2024-04-25T00:33:09.127 INFO:teuthology.orchestra.run.smithi032.stdout: "uptime": 158.50719177100001 2024-04-25T00:33:09.127 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:33:09.129 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.115+0000 7fe0ee590640 1 -- 172.21.15.32:0/2872124476 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe0d40a2b90 msgr2=0x7fe0d40a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.129 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.115+0000 7fe0ee590640 1 --2- 172.21.15.32:0/2872124476 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe0d40a2b90 0x7fe0d40a5050 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7fe0e00cd1b0 tx=0x7fe0e006ec60 comp rx=0 tx=0).stop 2024-04-25T00:33:09.129 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.115+0000 7fe0ee590640 1 -- 172.21.15.32:0/2872124476 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe0e81074a0 msgr2=0x7fe0e819f0d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.129 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.115+0000 7fe0ee590640 1 --2- 172.21.15.32:0/2872124476 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe0e81074a0 0x7fe0e819f0d0 secure :-1 s=READY pgs=429 cs=0 l=1 rev1=1 crypto rx=0x7fe0e000eaa0 tx=0x7fe0e000ef60 comp rx=0 tx=0).stop 2024-04-25T00:33:09.129 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.115+0000 7fe0ee590640 1 -- 172.21.15.32:0/2872124476 shutdown_connections 2024-04-25T00:33:09.129 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.115+0000 7fe0ee590640 1 --2- 172.21.15.32:0/2872124476 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fe0d40a2b90 0x7fe0d40a5050 unknown :-1 s=CLOSED pgs=259 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.129 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.115+0000 7fe0ee590640 1 --2- 172.21.15.32:0/2872124476 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe0e81074a0 0x7fe0e819f0d0 unknown :-1 s=CLOSED pgs=429 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.129 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.115+0000 7fe0ee590640 1 --2- 172.21.15.32:0/2872124476 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe0e8104b70 0x7fe0e8198060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.129 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.115+0000 7fe0ee590640 1 --2- 172.21.15.32:0/2872124476 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe0e8102240 0x7fe0e8197b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.130 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.115+0000 7fe0ee590640 1 -- 172.21.15.32:0/2872124476 >> 172.21.15.32:0/2872124476 conn(0x7fe0e80fbbf0 msgr2=0x7fe0e8104090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:09.130 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.115+0000 7fe0ee590640 1 -- 172.21.15.32:0/2872124476 shutdown_connections 2024-04-25T00:33:09.130 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.115+0000 7fe0ee590640 1 -- 172.21.15.32:0/2872124476 wait complete. 2024-04-25T00:33:09.130 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.i status 2024-04-25T00:33:09.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.232+0000 7fd25c93b640 1 -- 172.21.15.32:0/550872709 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd2581074c0 msgr2=0x7fd2581098b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.232+0000 7fd25c93b640 1 --2- 172.21.15.32:0/550872709 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd2581074c0 0x7fd2581098b0 secure :-1 s=READY pgs=434 cs=0 l=1 rev1=1 crypto rx=0x7fd250009ca0 tx=0x7fd25002a8c0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.232+0000 7fd25c93b640 1 -- 172.21.15.32:0/550872709 shutdown_connections 2024-04-25T00:33:09.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.232+0000 7fd25c93b640 1 --2- 172.21.15.32:0/550872709 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd2581074c0 0x7fd2581098b0 unknown :-1 s=CLOSED pgs=434 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.232+0000 7fd25c93b640 1 --2- 172.21.15.32:0/550872709 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd258104b90 0x7fd258106f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.232+0000 7fd25c93b640 1 --2- 172.21.15.32:0/550872709 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd258102260 0x7fd258104650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.232+0000 7fd25c93b640 1 -- 172.21.15.32:0/550872709 >> 172.21.15.32:0/550872709 conn(0x7fd2580fbbf0 msgr2=0x7fd2580fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:09.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.232+0000 7fd25c93b640 1 -- 172.21.15.32:0/550872709 shutdown_connections 2024-04-25T00:33:09.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.232+0000 7fd25c93b640 1 -- 172.21.15.32:0/550872709 wait complete. 2024-04-25T00:33:09.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.233+0000 7fd25c93b640 1 Processor -- start 2024-04-25T00:33:09.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.233+0000 7fd25c93b640 1 -- start start 2024-04-25T00:33:09.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.233+0000 7fd25c93b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd258102260 0x7fd258197a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.233+0000 7fd25c93b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd258104b90 0x7fd258197f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.233+0000 7fd25c93b640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd2581074c0 0x7fd25819efd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.233+0000 7fd25c93b640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fd25810d400 con 0x7fd258102260 2024-04-25T00:33:09.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.233+0000 7fd25c93b640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fd25810d280 con 0x7fd258104b90 2024-04-25T00:33:09.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.233+0000 7fd25c93b640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fd25810d580 con 0x7fd2581074c0 2024-04-25T00:33:09.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.233+0000 7fd25659c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd258102260 0x7fd258197a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.233+0000 7fd25659c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd258102260 0x7fd258197a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44254/0 (socket says 172.21.15.32:44254) 2024-04-25T00:33:09.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.233+0000 7fd25659c640 1 -- 172.21.15.32:0/560890810 learned_addr learned my addr 172.21.15.32:0/560890810 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:09.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.233+0000 7fd256d9d640 1 --2- 172.21.15.32:0/560890810 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd2581074c0 0x7fd25819efd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.233+0000 7fd25659c640 1 -- 172.21.15.32:0/560890810 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd2581074c0 msgr2=0x7fd25819efd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.233+0000 7fd25659c640 1 --2- 172.21.15.32:0/560890810 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd2581074c0 0x7fd25819efd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.233+0000 7fd25659c640 1 -- 172.21.15.32:0/560890810 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd258104b90 msgr2=0x7fd258197f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.233+0000 7fd255d9b640 1 --2- 172.21.15.32:0/560890810 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd258104b90 0x7fd258197f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.233+0000 7fd25659c640 1 --2- 172.21.15.32:0/560890810 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd258104b90 0x7fd258197f60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.233+0000 7fd25659c640 1 -- 172.21.15.32:0/560890810 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd25819f640 con 0x7fd258102260 2024-04-25T00:33:09.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.234+0000 7fd25659c640 1 --2- 172.21.15.32:0/560890810 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd258102260 0x7fd258197a20 secure :-1 s=READY pgs=435 cs=0 l=1 rev1=1 crypto rx=0x7fd24800c920 tx=0x7fd24800cde0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:09.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.234+0000 7fd2477fe640 1 -- 172.21.15.32:0/560890810 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd248007c70 con 0x7fd258102260 2024-04-25T00:33:09.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.234+0000 7fd2477fe640 1 -- 172.21.15.32:0/560890810 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd248007e10 con 0x7fd258102260 2024-04-25T00:33:09.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.234+0000 7fd25c93b640 1 -- 172.21.15.32:0/560890810 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd25819f930 con 0x7fd258102260 2024-04-25T00:33:09.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.234+0000 7fd2477fe640 1 -- 172.21.15.32:0/560890810 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd248018890 con 0x7fd258102260 2024-04-25T00:33:09.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.234+0000 7fd25c93b640 1 -- 172.21.15.32:0/560890810 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd25819fe80 con 0x7fd258102260 2024-04-25T00:33:09.243 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.236+0000 7fd2477fe640 1 -- 172.21.15.32:0/560890810 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 18) ==== 137833+0+0 (secure 0 0 0) 0x7fd248018a70 con 0x7fd258102260 2024-04-25T00:33:09.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.236+0000 7fd25f646740 1 Processor -- start 2024-04-25T00:33:09.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.237+0000 7fd25f646740 1 -- start start 2024-04-25T00:33:09.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.237+0000 7fd25f646740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5571e6a43440 0x5571e6a43860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.237+0000 7fd25f646740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5571e6a43e30 0x5571e6a48930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.237+0000 7fd25f646740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a48fa0 0x5571e6a4b430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.237+0000 7fd25f646740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5571e69a6f00 con 0x5571e6a48fa0 2024-04-25T00:33:09.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.237+0000 7fd25f646740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5571e694f750 con 0x5571e6a43e30 2024-04-25T00:33:09.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.237+0000 7fd25f646740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5571e69a2ec0 con 0x5571e6a43440 2024-04-25T00:33:09.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.237+0000 7fd2477fe640 1 --2- 172.21.15.32:0/560890810 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fd2400a2b90 0x7fd2400a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.237+0000 7fd255d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a48fa0 0x5571e6a4b430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.237+0000 7fd255d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a48fa0 0x5571e6a4b430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44268/0 (socket says 172.21.15.32:44268) 2024-04-25T00:33:09.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.237+0000 7fd255d9b640 1 -- 172.21.15.32:0/1405036885 learned_addr learned my addr 172.21.15.32:0/1405036885 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:09.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.237+0000 7fd25659c640 1 --2- 172.21.15.32:0/1405036885 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5571e6a43e30 0x5571e6a48930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.237+0000 7fd256d9d640 1 --2- 172.21.15.32:0/560890810 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fd2400a2b90 0x7fd2400a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.237+0000 7fd256d9d640 1 --2- 172.21.15.32:0/1405036885 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5571e6a43440 0x5571e6a43860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.237+0000 7fd2477fe640 1 -- 172.21.15.32:0/560890810 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7fd2480cf670 con 0x7fd258102260 2024-04-25T00:33:09.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.237+0000 7fd255d9b640 1 -- 172.21.15.32:0/1405036885 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5571e6a43440 msgr2=0x5571e6a43860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:09.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.237+0000 7fd255d9b640 1 --2- 172.21.15.32:0/1405036885 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5571e6a43440 0x5571e6a43860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.237+0000 7fd255d9b640 1 -- 172.21.15.32:0/1405036885 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5571e6a43e30 msgr2=0x5571e6a48930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:09.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.237+0000 7fd255d9b640 1 --2- 172.21.15.32:0/1405036885 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5571e6a43e30 0x5571e6a48930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.237+0000 7fd255d9b640 1 -- 172.21.15.32:0/1405036885 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5571e66bb2d0 con 0x5571e6a48fa0 2024-04-25T00:33:09.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.237+0000 7fd25659c640 1 --2- 172.21.15.32:0/1405036885 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5571e6a43e30 0x5571e6a48930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:09.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.238+0000 7fd255d9b640 1 --2- 172.21.15.32:0/1405036885 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a48fa0 0x5571e6a4b430 secure :-1 s=READY pgs=436 cs=0 l=1 rev1=1 crypto rx=0x7fd24c009b10 tx=0x7fd24c00e2f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9741315b151c3bff server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:09.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.238+0000 7fd256d9d640 1 --2- 172.21.15.32:0/560890810 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fd2400a2b90 0x7fd2400a5050 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x5571e6a44690 tx=0x7fd250004a80 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:09.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.238+0000 7fd244ff9640 1 -- 172.21.15.32:0/1405036885 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd24c016070 con 0x5571e6a48fa0 2024-04-25T00:33:09.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.238+0000 7fd244ff9640 1 -- 172.21.15.32:0/1405036885 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd24c005500 con 0x5571e6a48fa0 2024-04-25T00:33:09.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.238+0000 7fd244ff9640 1 -- 172.21.15.32:0/1405036885 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd24c011470 con 0x5571e6a48fa0 2024-04-25T00:33:09.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.238+0000 7fd25f646740 1 -- 172.21.15.32:0/1405036885 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a48fa0 msgr2=0x5571e6a4b430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.238+0000 7fd25f646740 1 --2- 172.21.15.32:0/1405036885 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a48fa0 0x5571e6a4b430 secure :-1 s=READY pgs=436 cs=0 l=1 rev1=1 crypto rx=0x7fd24c009b10 tx=0x7fd24c00e2f0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.239+0000 7fd25f646740 1 -- 172.21.15.32:0/1405036885 shutdown_connections 2024-04-25T00:33:09.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.239+0000 7fd25f646740 1 --2- 172.21.15.32:0/1405036885 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a48fa0 0x5571e6a4b430 unknown :-1 s=CLOSED pgs=436 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.239+0000 7fd25f646740 1 --2- 172.21.15.32:0/1405036885 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5571e6a43e30 0x5571e6a48930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.239+0000 7fd25f646740 1 --2- 172.21.15.32:0/1405036885 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5571e6a43440 0x5571e6a43860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.239+0000 7fd25f646740 1 -- 172.21.15.32:0/1405036885 >> 172.21.15.32:0/1405036885 conn(0x5571e6a3eab0 msgr2=0x5571e6a3eeb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:09.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.239+0000 7fd25f646740 1 -- 172.21.15.32:0/1405036885 shutdown_connections 2024-04-25T00:33:09.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.239+0000 7fd25f646740 1 -- 172.21.15.32:0/1405036885 wait complete. 2024-04-25T00:33:09.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.239+0000 7fd25f646740 1 -- start start 2024-04-25T00:33:09.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.239+0000 7fd25f646740 1 Processor -- start 2024-04-25T00:33:09.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.239+0000 7fd25f646740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a43440 0x5571e6a4d8c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.239+0000 7fd25f646740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5571e6a43e30 0x5571e6a4de00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.239+0000 7fd25f646740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5571e6a48fa0 0x5571e6a54e40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.239+0000 7fd25f646740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5571e6a4bf50 con 0x5571e6a43440 2024-04-25T00:33:09.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.239+0000 7fd25f646740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5571e6962cf0 con 0x5571e6a48fa0 2024-04-25T00:33:09.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.239+0000 7fd25f646740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5571e6a4c0d0 con 0x5571e6a43e30 2024-04-25T00:33:09.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.239+0000 7fd25f646740 10 client.0 _ll_register_callbacks cb 0x5571e69626e0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:33:09.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.239+0000 7fd25659c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a43440 0x5571e6a4d8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.239+0000 7fd25659c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a43440 0x5571e6a4d8c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44272/0 (socket says 172.21.15.32:44272) 2024-04-25T00:33:09.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.239+0000 7fd25659c640 1 -- 172.21.15.32:0/1449809598 learned_addr learned my addr 172.21.15.32:0/1449809598 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:09.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.240+0000 7fd255d9b640 1 --2- 172.21.15.32:0/1449809598 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5571e6a43e30 0x5571e6a4de00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.240+0000 7fd256d9d640 1 --2- 172.21.15.32:0/1449809598 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5571e6a48fa0 0x5571e6a54e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.240+0000 7fd25659c640 1 -- 172.21.15.32:0/1449809598 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5571e6a43e30 msgr2=0x5571e6a4de00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:09.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.240+0000 7fd25659c640 1 --2- 172.21.15.32:0/1449809598 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5571e6a43e30 0x5571e6a4de00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.240+0000 7fd25659c640 1 -- 172.21.15.32:0/1449809598 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5571e6a48fa0 msgr2=0x5571e6a54e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:09.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.240+0000 7fd25659c640 1 --2- 172.21.15.32:0/1449809598 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5571e6a48fa0 0x5571e6a54e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.240+0000 7fd25659c640 1 -- 172.21.15.32:0/1449809598 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5571e67856b0 con 0x5571e6a43440 2024-04-25T00:33:09.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.240+0000 7fd255d9b640 1 --2- 172.21.15.32:0/1449809598 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5571e6a43e30 0x5571e6a4de00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:33:09.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.240+0000 7fd256d9d640 1 --2- 172.21.15.32:0/1449809598 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5571e6a48fa0 0x5571e6a54e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:33:09.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.240+0000 7fd25659c640 1 --2- 172.21.15.32:0/1449809598 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a43440 0x5571e6a4d8c0 secure :-1 s=READY pgs=437 cs=0 l=1 rev1=1 crypto rx=0x7fd248098470 tx=0x7fd24806aa20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5956ab2a2ca59c06 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:09.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.240+0000 7fd23dffb640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:33:09.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.240+0000 7fd23dffb640 1 -- 172.21.15.32:0/1449809598 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd24800de20 con 0x5571e6a43440 2024-04-25T00:33:09.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.240+0000 7fd23dffb640 1 -- 172.21.15.32:0/1449809598 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7fd24800fb30 con 0x5571e6a43440 2024-04-25T00:33:09.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.240+0000 7fd25f646740 1 -- 172.21.15.32:0/1449809598 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd24c005500 con 0x5571e6a43440 2024-04-25T00:33:09.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.240+0000 7fd23dffb640 1 -- 172.21.15.32:0/1449809598 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) ==== 24+0+0 (secure 0 0 0) 0x7fd24809baa0 con 0x5571e6a43440 2024-04-25T00:33:09.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.240+0000 7fd25f646740 10 client.15435 fetch_fsmap learned FSMap version 24 2024-04-25T00:33:09.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.240+0000 7fd25f646740 1 -- 172.21.15.32:0/1449809598 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) -- 0x5571e6a57e60 con 0x5571e6a43440 2024-04-25T00:33:09.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.240+0000 7fd23dffb640 1 -- 172.21.15.32:0/1449809598 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd24c005500 con 0x5571e6a43440 2024-04-25T00:33:09.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.240+0000 7fd23dffb640 1 -- 172.21.15.32:0/1449809598 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 24) ==== 8346+0+0 (secure 0 0 0) 0x7fd24809bd00 con 0x5571e6a43440 2024-04-25T00:33:09.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.240+0000 7fd25f646740 10 client.15435 fetch_fsmap finished waiting for FSMap version 24 2024-04-25T00:33:09.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.240+0000 7fd25f646740 10 client.15435 resolve_mds: resolved name 'i' to daemon mds.i 2024-04-25T00:33:09.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.240+0000 7fd25f646740 20 client.15435 populate_metadata read hostname 'smithi032' 2024-04-25T00:33:09.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.241+0000 7fd25f646740 1 --2- 172.21.15.32:0/1449809598 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x5571e6a5a530 0x5571e6a5b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.241+0000 7fd25f646740 4 client.15435 mds_command: new command op to 24481 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:33:09.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.241+0000 7fd25f646740 1 -- 172.21.15.32:0/1449809598 --> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5571e69b4440 con 0x5571e6a5a530 2024-04-25T00:33:09.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.241+0000 7fd255d9b640 1 --2- 172.21.15.32:0/1449809598 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x5571e6a5a530 0x5571e6a5b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.241+0000 7fd23dffb640 10 client.15435 ms_handle_connect on v2:172.21.15.130:6838/4267125809 2024-04-25T00:33:09.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.241+0000 7fd255d9b640 1 --2- 172.21.15.32:0/1449809598 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x5571e6a5a530 0x5571e6a5b9a0 crc :-1 s=READY pgs=17 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=f045c78fa94de693 server_cookie=d59273f6adb055f6 in_seq=0 out_seq=0 2024-04-25T00:33:09.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.242+0000 7fd23dffb640 1 -- 172.21.15.32:0/1449809598 <== mds.1 v2:172.21.15.130:6838/4267125809 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x5571e69b4440 con 0x5571e6a5a530 2024-04-25T00:33:09.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.242+0000 7fd23dffb640 10 client.15435 handle_command_reply: tid=0 2024-04-25T00:33:09.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.242+0000 7fd25f646740 1 client.15435 shutdown 2024-04-25T00:33:09.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.242+0000 7fd25f646740 1 -- 172.21.15.32:0/1449809598 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a43440 msgr2=0x5571e6a4d8c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.242+0000 7fd25f646740 1 --2- 172.21.15.32:0/1449809598 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a43440 0x5571e6a4d8c0 secure :-1 s=READY pgs=437 cs=0 l=1 rev1=1 crypto rx=0x7fd248098470 tx=0x7fd24806aa20 comp rx=0 tx=0).stop 2024-04-25T00:33:09.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.242+0000 7fd25f646740 1 -- 172.21.15.32:0/1449809598 shutdown_connections 2024-04-25T00:33:09.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.242+0000 7fd25f646740 1 --2- 172.21.15.32:0/1449809598 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5571e6a48fa0 0x5571e6a54e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.242+0000 7fd25f646740 1 --2- 172.21.15.32:0/1449809598 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5571e6a43e30 0x5571e6a4de00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.242+0000 7fd25f646740 1 --2- 172.21.15.32:0/1449809598 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x5571e6a5a530 0x5571e6a5b9a0 crc :-1 s=READY pgs=17 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.242+0000 7fd25f646740 1 --2- 172.21.15.32:0/1449809598 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a43440 0x5571e6a4d8c0 unknown :-1 s=CLOSED pgs=437 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.242+0000 7fd23dffb640 0 client.15435 ms_handle_reset on v2:172.21.15.130:6838/4267125809 2024-04-25T00:33:09.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.242+0000 7fd25f646740 1 -- 172.21.15.32:0/1449809598 >> 172.21.15.32:0/1449809598 conn(0x5571e6a3eab0 msgr2=0x5571e6a3dae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:09.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.242+0000 7fd25f646740 1 -- 172.21.15.32:0/1449809598 shutdown_connections 2024-04-25T00:33:09.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.242+0000 7fd25f646740 1 -- 172.21.15.32:0/1449809598 wait complete. 2024-04-25T00:33:09.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.242+0000 7fd25f646740 20 client.15435 trim_cache size 0 max 16384 2024-04-25T00:33:09.266 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.260+0000 7fd25f646740 1 Processor -- start 2024-04-25T00:33:09.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.261+0000 7fd25f646740 1 -- start start 2024-04-25T00:33:09.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.261+0000 7fd25f646740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a43440 0x5571e6a39690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.261+0000 7fd25f646740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5571e6a43e30 0x5571e6a54e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.261+0000 7fd25f646740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5571e6a48fa0 0x5571e6a4d4d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.261+0000 7fd25f646740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5571e6a59ce0 con 0x5571e6a43440 2024-04-25T00:33:09.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.261+0000 7fd25f646740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5571e6a5a3b0 con 0x5571e6a43e30 2024-04-25T00:33:09.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.261+0000 7fd25f646740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5571e6a5aa80 con 0x5571e6a48fa0 2024-04-25T00:33:09.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.261+0000 7fd25659c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a43440 0x5571e6a39690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.261+0000 7fd25659c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a43440 0x5571e6a39690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44282/0 (socket says 172.21.15.32:44282) 2024-04-25T00:33:09.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.261+0000 7fd25659c640 1 -- 172.21.15.32:0/3570484226 learned_addr learned my addr 172.21.15.32:0/3570484226 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:09.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.261+0000 7fd256d9d640 1 --2- 172.21.15.32:0/3570484226 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5571e6a48fa0 0x5571e6a4d4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.261+0000 7fd255d9b640 1 --2- 172.21.15.32:0/3570484226 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5571e6a43e30 0x5571e6a54e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.261+0000 7fd25659c640 1 -- 172.21.15.32:0/3570484226 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5571e6a48fa0 msgr2=0x5571e6a4d4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:09.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.261+0000 7fd25659c640 1 --2- 172.21.15.32:0/3570484226 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5571e6a48fa0 0x5571e6a4d4d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.261+0000 7fd25659c640 1 -- 172.21.15.32:0/3570484226 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5571e6a43e30 msgr2=0x5571e6a54e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:09.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.261+0000 7fd25659c640 1 --2- 172.21.15.32:0/3570484226 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5571e6a43e30 0x5571e6a54e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.261+0000 7fd25659c640 1 -- 172.21.15.32:0/3570484226 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5571e6a4be30 con 0x5571e6a43440 2024-04-25T00:33:09.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.261+0000 7fd255d9b640 1 --2- 172.21.15.32:0/3570484226 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5571e6a43e30 0x5571e6a54e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:33:09.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.261+0000 7fd256d9d640 1 --2- 172.21.15.32:0/3570484226 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5571e6a48fa0 0x5571e6a4d4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:33:09.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.261+0000 7fd25659c640 1 --2- 172.21.15.32:0/3570484226 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a43440 0x5571e6a39690 secure :-1 s=READY pgs=438 cs=0 l=1 rev1=1 crypto rx=0x7fd248096540 tx=0x7fd248000bc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4e145df67df9a371 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:09.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.261+0000 7fd23ffff640 1 -- 172.21.15.32:0/3570484226 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd2480a4710 con 0x5571e6a43440 2024-04-25T00:33:09.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.261+0000 7fd23ffff640 1 -- 172.21.15.32:0/3570484226 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd24809fe50 con 0x5571e6a43440 2024-04-25T00:33:09.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.261+0000 7fd23ffff640 1 -- 172.21.15.32:0/3570484226 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd24806ebd0 con 0x5571e6a43440 2024-04-25T00:33:09.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.262+0000 7fd25f646740 1 -- 172.21.15.32:0/3570484226 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a43440 msgr2=0x5571e6a39690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.262+0000 7fd25f646740 1 --2- 172.21.15.32:0/3570484226 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a43440 0x5571e6a39690 secure :-1 s=READY pgs=438 cs=0 l=1 rev1=1 crypto rx=0x7fd248096540 tx=0x7fd248000bc0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.269 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.262+0000 7fd25f646740 1 -- 172.21.15.32:0/3570484226 shutdown_connections 2024-04-25T00:33:09.269 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.262+0000 7fd25f646740 1 --2- 172.21.15.32:0/3570484226 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5571e6a48fa0 0x5571e6a4d4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.269 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.262+0000 7fd25f646740 1 --2- 172.21.15.32:0/3570484226 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5571e6a43e30 0x5571e6a54e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.269 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.262+0000 7fd25f646740 1 --2- 172.21.15.32:0/3570484226 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a43440 0x5571e6a39690 unknown :-1 s=CLOSED pgs=438 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.269 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.262+0000 7fd25f646740 1 -- 172.21.15.32:0/3570484226 >> 172.21.15.32:0/3570484226 conn(0x5571e6a3eab0 msgr2=0x5571e6a3ee90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:09.269 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.262+0000 7fd25f646740 1 -- 172.21.15.32:0/3570484226 shutdown_connections 2024-04-25T00:33:09.269 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.262+0000 7fd25f646740 1 -- 172.21.15.32:0/3570484226 wait complete. 2024-04-25T00:33:09.269 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.262+0000 7fd25f646740 1 -- start start 2024-04-25T00:33:09.269 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.262+0000 7fd25f646740 1 Processor -- start 2024-04-25T00:33:09.269 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.263+0000 7fd25f646740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5571e6a43440 0x5571e6a52df0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.269 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.263+0000 7fd25f646740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5571e6a43e30 0x5571e6abc4c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.270 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.263+0000 7fd25f646740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a48fa0 0x5571e6abeae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.270 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.263+0000 7fd25f646740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5571e6a5b820 con 0x5571e6a48fa0 2024-04-25T00:33:09.270 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.263+0000 7fd25f646740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5571e6a5b150 con 0x5571e6a43e30 2024-04-25T00:33:09.270 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.263+0000 7fd25f646740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5571e6a5bef0 con 0x5571e6a43440 2024-04-25T00:33:09.270 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.263+0000 7fd25f646740 10 client.0 _ll_register_callbacks cb 0x7fd248097d40 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:33:09.270 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.263+0000 7fd256d9d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a48fa0 0x5571e6abeae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.270 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.263+0000 7fd256d9d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a48fa0 0x5571e6abeae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44298/0 (socket says 172.21.15.32:44298) 2024-04-25T00:33:09.270 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.263+0000 7fd256d9d640 1 -- 172.21.15.32:0/1246349261 learned_addr learned my addr 172.21.15.32:0/1246349261 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:09.270 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.263+0000 7fd255d9b640 1 --2- 172.21.15.32:0/1246349261 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5571e6a43e30 0x5571e6abc4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.270 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.263+0000 7fd25659c640 1 --2- 172.21.15.32:0/1246349261 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5571e6a43440 0x5571e6a52df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.270 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.263+0000 7fd256d9d640 1 -- 172.21.15.32:0/1246349261 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5571e6a43440 msgr2=0x5571e6a52df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:09.270 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.263+0000 7fd256d9d640 1 --2- 172.21.15.32:0/1246349261 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5571e6a43440 0x5571e6a52df0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.270 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.263+0000 7fd256d9d640 1 -- 172.21.15.32:0/1246349261 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5571e6a43e30 msgr2=0x5571e6abc4c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:09.270 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.263+0000 7fd256d9d640 1 --2- 172.21.15.32:0/1246349261 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5571e6a43e30 0x5571e6abc4c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.271 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.263+0000 7fd256d9d640 1 -- 172.21.15.32:0/1246349261 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5571e6abf310 con 0x5571e6a48fa0 2024-04-25T00:33:09.271 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.263+0000 7fd256d9d640 1 --2- 172.21.15.32:0/1246349261 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a48fa0 0x5571e6abeae0 secure :-1 s=READY pgs=439 cs=0 l=1 rev1=1 crypto rx=0x7fd250031ae0 tx=0x7fd2500095c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=922c6979fad24cbf server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:09.271 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.263+0000 7fd23dffb640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:33:09.271 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.263+0000 7fd23dffb640 1 -- 172.21.15.32:0/1246349261 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd2500305c0 con 0x5571e6a48fa0 2024-04-25T00:33:09.271 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.263+0000 7fd25f646740 1 -- 172.21.15.32:0/1246349261 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd24809fe50 con 0x5571e6a48fa0 2024-04-25T00:33:09.271 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.263+0000 7fd23dffb640 1 -- 172.21.15.32:0/1246349261 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7fd250030760 con 0x5571e6a48fa0 2024-04-25T00:33:09.271 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.264+0000 7fd23dffb640 1 -- 172.21.15.32:0/1246349261 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) ==== 24+0+0 (secure 0 0 0) 0x7fd25005e050 con 0x5571e6a48fa0 2024-04-25T00:33:09.271 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.264+0000 7fd25f646740 10 client.15441 fetch_fsmap learned FSMap version 24 2024-04-25T00:33:09.271 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.264+0000 7fd25f646740 1 -- 172.21.15.32:0/1246349261 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) -- 0x5571e6ac05f0 con 0x5571e6a48fa0 2024-04-25T00:33:09.271 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.264+0000 7fd23dffb640 1 -- 172.21.15.32:0/1246349261 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd24809fe50 con 0x5571e6a48fa0 2024-04-25T00:33:09.271 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.264+0000 7fd23dffb640 1 -- 172.21.15.32:0/1246349261 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 24) ==== 8346+0+0 (secure 0 0 0) 0x7fd250055b30 con 0x5571e6a48fa0 2024-04-25T00:33:09.271 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.264+0000 7fd25f646740 10 client.15441 fetch_fsmap finished waiting for FSMap version 24 2024-04-25T00:33:09.271 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.264+0000 7fd25f646740 10 client.15441 resolve_mds: resolved name 'i' to daemon mds.i 2024-04-25T00:33:09.271 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.264+0000 7fd25f646740 20 client.15441 populate_metadata read hostname 'smithi032' 2024-04-25T00:33:09.272 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.264+0000 7fd25f646740 1 --2- 172.21.15.32:0/1246349261 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x5571e6ac51e0 0x5571e6ac3ef0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.272 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.264+0000 7fd25f646740 4 client.15441 mds_command: new command op to 24481 tid=0 multi_id=0 [{"prefix": "status"}] 2024-04-25T00:33:09.272 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.264+0000 7fd25f646740 1 -- 172.21.15.32:0/1246349261 --> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] -- command(tid 0: {"prefix": "status"}) -- 0x5571e69ae790 con 0x5571e6ac51e0 2024-04-25T00:33:09.272 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.264+0000 7fd25659c640 1 --2- 172.21.15.32:0/1246349261 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x5571e6ac51e0 0x5571e6ac3ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.272 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.265+0000 7fd25659c640 1 --2- 172.21.15.32:0/1246349261 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x5571e6ac51e0 0x5571e6ac3ef0 crc :-1 s=READY pgs=18 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=9dae01549efd4edb server_cookie=7995ef5f5ed6d511 in_seq=0 out_seq=0 2024-04-25T00:33:09.272 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.265+0000 7fd23dffb640 10 client.15441 ms_handle_connect on v2:172.21.15.130:6838/4267125809 2024-04-25T00:33:09.272 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.265+0000 7fd23dffb640 1 -- 172.21.15.32:0/1246349261 <== mds.1 v2:172.21.15.130:6838/4267125809 1 ==== command_reply(tid 0: 0 ) ==== 8+0+327 (crc 0 0 0) 0x5571e69ae790 con 0x5571e6ac51e0 2024-04-25T00:33:09.272 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.265+0000 7fd23dffb640 10 client.15441 handle_command_reply: tid=0 2024-04-25T00:33:09.272 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.265+0000 7fd25f646740 1 client.15441 shutdown 2024-04-25T00:33:09.272 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.265+0000 7fd25f646740 1 -- 172.21.15.32:0/1246349261 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a48fa0 msgr2=0x5571e6abeae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.272 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.265+0000 7fd25f646740 1 --2- 172.21.15.32:0/1246349261 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a48fa0 0x5571e6abeae0 secure :-1 s=READY pgs=439 cs=0 l=1 rev1=1 crypto rx=0x7fd250031ae0 tx=0x7fd2500095c0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.272 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.265+0000 7fd25f646740 1 -- 172.21.15.32:0/1246349261 shutdown_connections 2024-04-25T00:33:09.272 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.265+0000 7fd25f646740 1 --2- 172.21.15.32:0/1246349261 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x5571e6ac51e0 0x5571e6ac3ef0 crc :-1 s=READY pgs=18 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.272 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.265+0000 7fd25f646740 1 --2- 172.21.15.32:0/1246349261 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5571e6a48fa0 0x5571e6abeae0 unknown :-1 s=CLOSED pgs=439 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.272 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.265+0000 7fd25f646740 1 --2- 172.21.15.32:0/1246349261 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5571e6a43e30 0x5571e6abc4c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.273 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.265+0000 7fd23dffb640 0 client.15441 ms_handle_reset on v2:172.21.15.130:6838/4267125809 2024-04-25T00:33:09.273 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.265+0000 7fd25f646740 1 --2- 172.21.15.32:0/1246349261 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5571e6a43440 0x5571e6a52df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.273 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.265+0000 7fd25f646740 1 -- 172.21.15.32:0/1246349261 >> 172.21.15.32:0/1246349261 conn(0x5571e6a3eab0 msgr2=0x5571e6a39690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:09.273 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.265+0000 7fd25f646740 1 -- 172.21.15.32:0/1246349261 shutdown_connections 2024-04-25T00:33:09.273 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.265+0000 7fd25f646740 1 -- 172.21.15.32:0/1246349261 wait complete. 2024-04-25T00:33:09.273 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.265+0000 7fd25f646740 20 client.15441 trim_cache size 0 max 16384 2024-04-25T00:33:09.273 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:33:09.273 INFO:teuthology.orchestra.run.smithi032.stdout: "cluster_fsid": "cb997156-0299-11ef-bc93-c7b262605968", 2024-04-25T00:33:09.273 INFO:teuthology.orchestra.run.smithi032.stdout: "whoami": 1, 2024-04-25T00:33:09.273 INFO:teuthology.orchestra.run.smithi032.stdout: "id": 24481, 2024-04-25T00:33:09.274 INFO:teuthology.orchestra.run.smithi032.stdout: "want_state": "up:active", 2024-04-25T00:33:09.274 INFO:teuthology.orchestra.run.smithi032.stdout: "state": "up:active", 2024-04-25T00:33:09.274 INFO:teuthology.orchestra.run.smithi032.stdout: "fs_name": "cephfs", 2024-04-25T00:33:09.274 INFO:teuthology.orchestra.run.smithi032.stdout: "rank_uptime": 116.313119802, 2024-04-25T00:33:09.274 INFO:teuthology.orchestra.run.smithi032.stdout: "mdsmap_epoch": 24, 2024-04-25T00:33:09.274 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_epoch": 93, 2024-04-25T00:33:09.274 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_epoch_barrier": 89, 2024-04-25T00:33:09.274 INFO:teuthology.orchestra.run.smithi032.stdout: "uptime": 155.92351458799999 2024-04-25T00:33:09.274 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:33:09.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.266+0000 7fd25c93b640 1 -- 172.21.15.32:0/560890810 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fd2400a2b90 msgr2=0x7fd2400a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.266+0000 7fd25c93b640 1 --2- 172.21.15.32:0/560890810 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fd2400a2b90 0x7fd2400a5050 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x5571e6a44690 tx=0x7fd250004a80 comp rx=0 tx=0).stop 2024-04-25T00:33:09.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.266+0000 7fd25c93b640 1 -- 172.21.15.32:0/560890810 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd258102260 msgr2=0x7fd258197a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.266+0000 7fd25c93b640 1 --2- 172.21.15.32:0/560890810 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd258102260 0x7fd258197a20 secure :-1 s=READY pgs=435 cs=0 l=1 rev1=1 crypto rx=0x7fd24800c920 tx=0x7fd24800cde0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.266+0000 7fd25c93b640 1 -- 172.21.15.32:0/560890810 shutdown_connections 2024-04-25T00:33:09.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.266+0000 7fd25c93b640 1 --2- 172.21.15.32:0/560890810 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fd2400a2b90 0x7fd2400a5050 unknown :-1 s=CLOSED pgs=260 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.266+0000 7fd25c93b640 1 --2- 172.21.15.32:0/560890810 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd2581074c0 0x7fd25819efd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.266+0000 7fd25c93b640 1 --2- 172.21.15.32:0/560890810 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd258104b90 0x7fd258197f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.266+0000 7fd25c93b640 1 --2- 172.21.15.32:0/560890810 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd258102260 0x7fd258197a20 unknown :-1 s=CLOSED pgs=435 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.266+0000 7fd25c93b640 1 -- 172.21.15.32:0/560890810 >> 172.21.15.32:0/560890810 conn(0x7fd2580fbbf0 msgr2=0x7fd258102b60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:09.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.266+0000 7fd25c93b640 1 -- 172.21.15.32:0/560890810 shutdown_connections 2024-04-25T00:33:09.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.266+0000 7fd25c93b640 1 -- 172.21.15.32:0/560890810 wait complete. 2024-04-25T00:33:09.277 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.b status 2024-04-25T00:33:09.382 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.379+0000 7fb19ed90640 1 -- 172.21.15.32:0/1631402004 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb198101870 msgr2=0x7fb198101cf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.382 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.379+0000 7fb19ed90640 1 --2- 172.21.15.32:0/1631402004 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb198101870 0x7fb198101cf0 secure :-1 s=READY pgs=440 cs=0 l=1 rev1=1 crypto rx=0x7fb18c009900 tx=0x7fb18c02a410 comp rx=0 tx=0).stop 2024-04-25T00:33:09.382 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.379+0000 7fb19ed90640 1 -- 172.21.15.32:0/1631402004 shutdown_connections 2024-04-25T00:33:09.382 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.379+0000 7fb19ed90640 1 --2- 172.21.15.32:0/1631402004 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb198102230 0x7fb198108ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.382 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.379+0000 7fb19ed90640 1 --2- 172.21.15.32:0/1631402004 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb198101870 0x7fb198101cf0 unknown :-1 s=CLOSED pgs=440 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.382 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.379+0000 7fb19ed90640 1 --2- 172.21.15.32:0/1631402004 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb198100680 0x7fb198100aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.382 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.379+0000 7fb19ed90640 1 -- 172.21.15.32:0/1631402004 >> 172.21.15.32:0/1631402004 conn(0x7fb1980fbbf0 msgr2=0x7fb1980fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:09.383 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.379+0000 7fb19ed90640 1 -- 172.21.15.32:0/1631402004 shutdown_connections 2024-04-25T00:33:09.383 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.379+0000 7fb19ed90640 1 -- 172.21.15.32:0/1631402004 wait complete. 2024-04-25T00:33:09.383 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.380+0000 7fb19ed90640 1 Processor -- start 2024-04-25T00:33:09.383 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.380+0000 7fb19ed90640 1 -- start start 2024-04-25T00:33:09.383 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.380+0000 7fb19ed90640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb198100680 0x7fb1980694a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.383 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.380+0000 7fb19ed90640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb198101870 0x7fb19819c770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.383 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.380+0000 7fb19ed90640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb198102230 0x7fb19819ede0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.383 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.380+0000 7fb19ed90640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fb19810c2e0 con 0x7fb198100680 2024-04-25T00:33:09.383 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.380+0000 7fb19ed90640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fb19810c160 con 0x7fb198101870 2024-04-25T00:33:09.383 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.380+0000 7fb19ed90640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fb19810c460 con 0x7fb198102230 2024-04-25T00:33:09.383 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.380+0000 7fb19cb2c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb198100680 0x7fb1980694a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.383 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.380+0000 7fb19cb2c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb198100680 0x7fb1980694a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44316/0 (socket says 172.21.15.32:44316) 2024-04-25T00:33:09.383 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.380+0000 7fb19cb2c640 1 -- 172.21.15.32:0/705257155 learned_addr learned my addr 172.21.15.32:0/705257155 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:09.383 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.380+0000 7fb19d32d640 1 --2- 172.21.15.32:0/705257155 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb198102230 0x7fb19819ede0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.384 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.380+0000 7fb197fff640 1 --2- 172.21.15.32:0/705257155 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb198101870 0x7fb19819c770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.384 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.381+0000 7fb19cb2c640 1 -- 172.21.15.32:0/705257155 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb198102230 msgr2=0x7fb19819ede0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.384 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.381+0000 7fb19cb2c640 1 --2- 172.21.15.32:0/705257155 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb198102230 0x7fb19819ede0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.384 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.381+0000 7fb19cb2c640 1 -- 172.21.15.32:0/705257155 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb198101870 msgr2=0x7fb19819c770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.384 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.381+0000 7fb19cb2c640 1 --2- 172.21.15.32:0/705257155 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb198101870 0x7fb19819c770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.384 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.381+0000 7fb19cb2c640 1 -- 172.21.15.32:0/705257155 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb19819f6a0 con 0x7fb198100680 2024-04-25T00:33:09.384 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.381+0000 7fb197fff640 1 --2- 172.21.15.32:0/705257155 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb198101870 0x7fb19819c770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:33:09.384 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.381+0000 7fb19cb2c640 1 --2- 172.21.15.32:0/705257155 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb198100680 0x7fb1980694a0 secure :-1 s=READY pgs=441 cs=0 l=1 rev1=1 crypto rx=0x7fb18800d770 tx=0x7fb18800dc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:09.384 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.381+0000 7fb195ffb640 1 -- 172.21.15.32:0/705257155 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb188017070 con 0x7fb198100680 2024-04-25T00:33:09.384 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.381+0000 7fb19ed90640 1 -- 172.21.15.32:0/705257155 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb19819f990 con 0x7fb198100680 2024-04-25T00:33:09.384 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.381+0000 7fb195ffb640 1 -- 172.21.15.32:0/705257155 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb188009c30 con 0x7fb198100680 2024-04-25T00:33:09.384 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.381+0000 7fb195ffb640 1 -- 172.21.15.32:0/705257155 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb188003dd0 con 0x7fb198100680 2024-04-25T00:33:09.384 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.381+0000 7fb19ed90640 1 -- 172.21.15.32:0/705257155 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb19819ff30 con 0x7fb198100680 2024-04-25T00:33:09.391 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.383+0000 7fb195ffb640 1 -- 172.21.15.32:0/705257155 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 18) ==== 137833+0+0 (secure 0 0 0) 0x7fb188003f70 con 0x7fb198100680 2024-04-25T00:33:09.391 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.384+0000 7fb1a1b7a740 1 Processor -- start 2024-04-25T00:33:09.392 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.384+0000 7fb1a1b7a740 1 -- start start 2024-04-25T00:33:09.392 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.384+0000 7fb195ffb640 1 --2- 172.21.15.32:0/705257155 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb1800a2b90 0x7fb1800a5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.392 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.384+0000 7fb1a1b7a740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x558116f71440 0x558116f71860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.392 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.384+0000 7fb1a1b7a740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558116f71e30 0x558116f76930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.392 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.384+0000 7fb1a1b7a740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f76fa0 0x558116f79430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.392 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.384+0000 7fb1a1b7a740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x558116ed4f00 con 0x558116f76fa0 2024-04-25T00:33:09.392 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.384+0000 7fb1a1b7a740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x558116e7d750 con 0x558116f71440 2024-04-25T00:33:09.392 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.384+0000 7fb1a1b7a740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x558116ed0ec0 con 0x558116f71e30 2024-04-25T00:33:09.392 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.384+0000 7fb19d32d640 1 --2- 172.21.15.32:0/705257155 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb1800a2b90 0x7fb1800a5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.392 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.384+0000 7fb195ffb640 1 -- 172.21.15.32:0/705257155 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7fb1880cee30 con 0x7fb198100680 2024-04-25T00:33:09.392 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.384+0000 7fb19cb2c640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x558116f71440 0x558116f71860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.392 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.384+0000 7fb19cb2c640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x558116f71440 0x558116f71860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:48046/0 (socket says 172.21.15.32:48046) 2024-04-25T00:33:09.392 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.384+0000 7fb19cb2c640 1 -- 172.21.15.32:0/3861739562 learned_addr learned my addr 172.21.15.32:0/3861739562 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:09.392 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.384+0000 7fb19d32d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f76fa0 0x558116f79430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.393 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.384+0000 7fb197fff640 1 --2- 172.21.15.32:0/3861739562 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558116f71e30 0x558116f76930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.393 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.384+0000 7fb19cb2c640 1 -- 172.21.15.32:0/3861739562 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558116f71e30 msgr2=0x558116f76930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:09.393 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.384+0000 7fb19cb2c640 1 --2- 172.21.15.32:0/3861739562 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558116f71e30 0x558116f76930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.393 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.384+0000 7fb19cb2c640 1 -- 172.21.15.32:0/3861739562 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f76fa0 msgr2=0x558116f79430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:09.393 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.384+0000 7fb19cb2c640 1 --2- 172.21.15.32:0/3861739562 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f76fa0 0x558116f79430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.393 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.384+0000 7fb19cb2c640 1 -- 172.21.15.32:0/3861739562 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558116be92d0 con 0x558116f71440 2024-04-25T00:33:09.393 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.384+0000 7fb19d32d640 1 --2- 172.21.15.32:0/3861739562 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f76fa0 0x558116f79430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:09.393 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.384+0000 7fb19d32d640 1 --2- 172.21.15.32:0/705257155 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb1800a2b90 0x7fb1800a5050 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7fb1900098b0 tx=0x7fb190009e20 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:09.393 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.385+0000 7fb19cb2c640 1 --2- 172.21.15.32:0/3861739562 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x558116f71440 0x558116f71860 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7fb1880976c0 tx=0x7fb188097630 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2fd474cf0b306532 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:09.393 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.385+0000 7fb17effd640 1 -- 172.21.15.32:0/3861739562 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb188020040 con 0x558116f71440 2024-04-25T00:33:09.393 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.385+0000 7fb17effd640 1 -- 172.21.15.32:0/3861739562 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb18809b960 con 0x558116f71440 2024-04-25T00:33:09.393 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.385+0000 7fb17effd640 1 -- 172.21.15.32:0/3861739562 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb1880cc610 con 0x558116f71440 2024-04-25T00:33:09.393 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.385+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/3861739562 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x558116f71440 msgr2=0x558116f71860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.393 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.385+0000 7fb1a1b7a740 1 --2- 172.21.15.32:0/3861739562 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x558116f71440 0x558116f71860 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7fb1880976c0 tx=0x7fb188097630 comp rx=0 tx=0).stop 2024-04-25T00:33:09.394 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.385+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/3861739562 shutdown_connections 2024-04-25T00:33:09.394 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.385+0000 7fb1a1b7a740 1 --2- 172.21.15.32:0/3861739562 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f76fa0 0x558116f79430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.394 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.385+0000 7fb1a1b7a740 1 --2- 172.21.15.32:0/3861739562 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558116f71e30 0x558116f76930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.394 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.385+0000 7fb1a1b7a740 1 --2- 172.21.15.32:0/3861739562 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x558116f71440 0x558116f71860 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.394 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.385+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/3861739562 >> 172.21.15.32:0/3861739562 conn(0x558116f6cab0 msgr2=0x558116f6ceb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:09.394 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.386+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/3861739562 shutdown_connections 2024-04-25T00:33:09.394 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.386+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/3861739562 wait complete. 2024-04-25T00:33:09.394 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.386+0000 7fb1a1b7a740 1 -- start start 2024-04-25T00:33:09.394 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.386+0000 7fb1a1b7a740 1 Processor -- start 2024-04-25T00:33:09.394 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.386+0000 7fb1a1b7a740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f71440 0x558116f7b780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.394 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.386+0000 7fb1a1b7a740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558116f71e30 0x558116f7bcc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.394 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.386+0000 7fb1a1b7a740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x558116f76fa0 0x558116f82d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.394 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.386+0000 7fb1a1b7a740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x558116f79e90 con 0x558116f71440 2024-04-25T00:33:09.394 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.386+0000 7fb1a1b7a740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x558116e90cf0 con 0x558116f76fa0 2024-04-25T00:33:09.395 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.386+0000 7fb1a1b7a740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x558116f7a010 con 0x558116f71e30 2024-04-25T00:33:09.395 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.386+0000 7fb1a1b7a740 10 client.0 _ll_register_callbacks cb 0x558116e906e0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:33:09.395 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.386+0000 7fb19cb2c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f71440 0x558116f7b780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.395 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.386+0000 7fb19cb2c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f71440 0x558116f7b780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44332/0 (socket says 172.21.15.32:44332) 2024-04-25T00:33:09.395 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.386+0000 7fb19cb2c640 1 -- 172.21.15.32:0/3675419403 learned_addr learned my addr 172.21.15.32:0/3675419403 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:09.395 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.387+0000 7fb197fff640 1 --2- 172.21.15.32:0/3675419403 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558116f71e30 0x558116f7bcc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.395 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.387+0000 7fb19d32d640 1 --2- 172.21.15.32:0/3675419403 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x558116f76fa0 0x558116f82d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.395 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.387+0000 7fb19cb2c640 1 -- 172.21.15.32:0/3675419403 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558116f71e30 msgr2=0x558116f7bcc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:09.395 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.387+0000 7fb19cb2c640 1 --2- 172.21.15.32:0/3675419403 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558116f71e30 0x558116f7bcc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.395 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.387+0000 7fb19cb2c640 1 -- 172.21.15.32:0/3675419403 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x558116f76fa0 msgr2=0x558116f82d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:09.395 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.387+0000 7fb19cb2c640 1 --2- 172.21.15.32:0/3675419403 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x558116f76fa0 0x558116f82d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.395 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.387+0000 7fb19cb2c640 1 -- 172.21.15.32:0/3675419403 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558116cb36b0 con 0x558116f71440 2024-04-25T00:33:09.395 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.387+0000 7fb19cb2c640 1 --2- 172.21.15.32:0/3675419403 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f71440 0x558116f7b780 secure :-1 s=READY pgs=442 cs=0 l=1 rev1=1 crypto rx=0x7fb188000c00 tx=0x7fb18800de80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=90cd6bbef9864249 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:09.395 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.387+0000 7fb177fff640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:33:09.396 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.387+0000 7fb177fff640 1 -- 172.21.15.32:0/3675419403 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb188020030 con 0x558116f71440 2024-04-25T00:33:09.396 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.387+0000 7fb177fff640 1 -- 172.21.15.32:0/3675419403 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7fb1880978d0 con 0x558116f71440 2024-04-25T00:33:09.396 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.387+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/3675419403 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb18809b960 con 0x558116f71440 2024-04-25T00:33:09.396 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.387+0000 7fb177fff640 1 -- 172.21.15.32:0/3675419403 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) ==== 24+0+0 (secure 0 0 0) 0x7fb18809b260 con 0x558116f71440 2024-04-25T00:33:09.396 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.387+0000 7fb1a1b7a740 10 client.15453 fetch_fsmap learned FSMap version 24 2024-04-25T00:33:09.396 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.387+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/3675419403 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) -- 0x558116f85ce0 con 0x558116f71440 2024-04-25T00:33:09.396 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.387+0000 7fb177fff640 1 -- 172.21.15.32:0/3675419403 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb18809b960 con 0x558116f71440 2024-04-25T00:33:09.396 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.388+0000 7fb177fff640 1 -- 172.21.15.32:0/3675419403 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 24) ==== 8346+0+0 (secure 0 0 0) 0x7fb1880ba100 con 0x558116f71440 2024-04-25T00:33:09.396 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.388+0000 7fb1a1b7a740 10 client.15453 fetch_fsmap finished waiting for FSMap version 24 2024-04-25T00:33:09.396 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.388+0000 7fb1a1b7a740 10 client.15453 resolve_mds: resolved name 'b' to daemon mds.b 2024-04-25T00:33:09.396 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.388+0000 7fb1a1b7a740 20 client.15453 populate_metadata read hostname 'smithi032' 2024-04-25T00:33:09.396 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.388+0000 7fb1a1b7a740 1 --2- 172.21.15.32:0/3675419403 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x558116f87e60 0x558116f892d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.396 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.388+0000 7fb1a1b7a740 4 client.15453 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:33:09.396 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.388+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/3675419403 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x558116ee2440 con 0x558116f87e60 2024-04-25T00:33:09.396 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.388+0000 7fb197fff640 1 --2- 172.21.15.32:0/3675419403 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x558116f87e60 0x558116f892d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.397 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.388+0000 7fb177fff640 10 client.15453 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:09.397 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.388+0000 7fb197fff640 1 --2- 172.21.15.32:0/3675419403 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x558116f87e60 0x558116f892d0 crc :-1 s=READY pgs=27 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=41e7042ec3ec4b23 server_cookie=c4ae475108e26aa6 in_seq=0 out_seq=0 2024-04-25T00:33:09.397 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.389+0000 7fb177fff640 1 -- 172.21.15.32:0/3675419403 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x558116ee2440 con 0x558116f87e60 2024-04-25T00:33:09.397 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.389+0000 7fb177fff640 10 client.15453 handle_command_reply: tid=0 2024-04-25T00:33:09.397 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.389+0000 7fb1a1b7a740 1 client.15453 shutdown 2024-04-25T00:33:09.397 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.389+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/3675419403 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f71440 msgr2=0x558116f7b780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.397 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.389+0000 7fb1a1b7a740 1 --2- 172.21.15.32:0/3675419403 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f71440 0x558116f7b780 secure :-1 s=READY pgs=442 cs=0 l=1 rev1=1 crypto rx=0x7fb188000c00 tx=0x7fb18800de80 comp rx=0 tx=0).stop 2024-04-25T00:33:09.397 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.389+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/3675419403 shutdown_connections 2024-04-25T00:33:09.397 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.389+0000 7fb1a1b7a740 1 --2- 172.21.15.32:0/3675419403 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x558116f87e60 0x558116f892d0 crc :-1 s=READY pgs=27 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.397 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.389+0000 7fb1a1b7a740 1 --2- 172.21.15.32:0/3675419403 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x558116f76fa0 0x558116f82d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.397 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.389+0000 7fb1a1b7a740 1 --2- 172.21.15.32:0/3675419403 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558116f71e30 0x558116f7bcc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.397 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.389+0000 7fb177fff640 0 client.15453 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:09.397 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.389+0000 7fb1a1b7a740 1 --2- 172.21.15.32:0/3675419403 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f71440 0x558116f7b780 unknown :-1 s=CLOSED pgs=442 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.397 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.389+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/3675419403 >> 172.21.15.32:0/3675419403 conn(0x558116f6cab0 msgr2=0x558116f6bae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:09.397 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.389+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/3675419403 shutdown_connections 2024-04-25T00:33:09.398 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.390+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/3675419403 wait complete. 2024-04-25T00:33:09.398 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.390+0000 7fb1a1b7a740 20 client.15453 trim_cache size 0 max 16384 2024-04-25T00:33:09.413 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.408+0000 7fb1a1b7a740 1 Processor -- start 2024-04-25T00:33:09.413 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.408+0000 7fb1a1b7a740 1 -- start start 2024-04-25T00:33:09.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.408+0000 7fb1a1b7a740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558116f71440 0x558116f79390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.408+0000 7fb1a1b7a740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x558116f71e30 0x558116f82c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.408+0000 7fb1a1b7a740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f76fa0 0x558116f67690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.408+0000 7fb1a1b7a740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x558116f87610 con 0x558116f76fa0 2024-04-25T00:33:09.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.408+0000 7fb1a1b7a740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x558116f87ce0 con 0x558116f71e30 2024-04-25T00:33:09.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.408+0000 7fb1a1b7a740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x558116f883b0 con 0x558116f71440 2024-04-25T00:33:09.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.408+0000 7fb19d32d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f76fa0 0x558116f67690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.408+0000 7fb19d32d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f76fa0 0x558116f67690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44338/0 (socket says 172.21.15.32:44338) 2024-04-25T00:33:09.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.408+0000 7fb19d32d640 1 -- 172.21.15.32:0/1395087966 learned_addr learned my addr 172.21.15.32:0/1395087966 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:09.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.408+0000 7fb197fff640 1 --2- 172.21.15.32:0/1395087966 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x558116f71e30 0x558116f82c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.408+0000 7fb19d32d640 1 -- 172.21.15.32:0/1395087966 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558116f71440 msgr2=0x558116f79390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:09.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.408+0000 7fb19d32d640 1 --2- 172.21.15.32:0/1395087966 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558116f71440 0x558116f79390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.408+0000 7fb19d32d640 1 -- 172.21.15.32:0/1395087966 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x558116f71e30 msgr2=0x558116f82c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:09.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.408+0000 7fb19d32d640 1 --2- 172.21.15.32:0/1395087966 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x558116f71e30 0x558116f82c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.408+0000 7fb19d32d640 1 -- 172.21.15.32:0/1395087966 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558116f88860 con 0x558116f76fa0 2024-04-25T00:33:09.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.408+0000 7fb197fff640 1 --2- 172.21.15.32:0/1395087966 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x558116f71e30 0x558116f82c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:33:09.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.408+0000 7fb19d32d640 1 --2- 172.21.15.32:0/1395087966 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f76fa0 0x558116f67690 secure :-1 s=READY pgs=443 cs=0 l=1 rev1=1 crypto rx=0x7fb190010c50 tx=0x7fb190004ab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=76933af89dbdf734 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:09.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.409+0000 7fb17e7fc640 1 -- 172.21.15.32:0/1395087966 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb190010d40 con 0x558116f76fa0 2024-04-25T00:33:09.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.409+0000 7fb17e7fc640 1 -- 172.21.15.32:0/1395087966 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb190004030 con 0x558116f76fa0 2024-04-25T00:33:09.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.409+0000 7fb17e7fc640 1 -- 172.21.15.32:0/1395087966 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb190005280 con 0x558116f76fa0 2024-04-25T00:33:09.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.409+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/1395087966 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f76fa0 msgr2=0x558116f67690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.409+0000 7fb1a1b7a740 1 --2- 172.21.15.32:0/1395087966 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f76fa0 0x558116f67690 secure :-1 s=READY pgs=443 cs=0 l=1 rev1=1 crypto rx=0x7fb190010c50 tx=0x7fb190004ab0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.409+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/1395087966 shutdown_connections 2024-04-25T00:33:09.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.409+0000 7fb1a1b7a740 1 --2- 172.21.15.32:0/1395087966 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f76fa0 0x558116f67690 unknown :-1 s=CLOSED pgs=443 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.409+0000 7fb1a1b7a740 1 --2- 172.21.15.32:0/1395087966 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x558116f71e30 0x558116f82c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.409+0000 7fb1a1b7a740 1 --2- 172.21.15.32:0/1395087966 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558116f71440 0x558116f79390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.409+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/1395087966 >> 172.21.15.32:0/1395087966 conn(0x558116f6cab0 msgr2=0x558116f6ce90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:09.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.409+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/1395087966 shutdown_connections 2024-04-25T00:33:09.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.409+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/1395087966 wait complete. 2024-04-25T00:33:09.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.409+0000 7fb1a1b7a740 1 -- start start 2024-04-25T00:33:09.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.410+0000 7fb1a1b7a740 1 Processor -- start 2024-04-25T00:33:09.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.410+0000 7fb1a1b7a740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x558116f71440 0x558116f7ee50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.410+0000 7fb1a1b7a740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f71e30 0x558116f7f390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.410+0000 7fb1a1b7a740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558116f76fa0 0x558116f82000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.410+0000 7fb1a1b7a740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x558116f89150 con 0x558116f71e30 2024-04-25T00:33:09.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.410+0000 7fb1a1b7a740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x558116f88a80 con 0x558116f71440 2024-04-25T00:33:09.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.410+0000 7fb1a1b7a740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x558116f89820 con 0x558116f76fa0 2024-04-25T00:33:09.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.410+0000 7fb1a1b7a740 10 client.0 _ll_register_callbacks cb 0x7fb188097500 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:33:09.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.410+0000 7fb197fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f71e30 0x558116f7f390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.410+0000 7fb197fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f71e30 0x558116f7f390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44348/0 (socket says 172.21.15.32:44348) 2024-04-25T00:33:09.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.410+0000 7fb197fff640 1 -- 172.21.15.32:0/3663099721 learned_addr learned my addr 172.21.15.32:0/3663099721 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:09.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.410+0000 7fb19cb2c640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x558116f71440 0x558116f7ee50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.410+0000 7fb19d32d640 1 --2- 172.21.15.32:0/3663099721 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558116f76fa0 0x558116f82000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.410+0000 7fb197fff640 1 -- 172.21.15.32:0/3663099721 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558116f76fa0 msgr2=0x558116f82000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:09.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.410+0000 7fb197fff640 1 --2- 172.21.15.32:0/3663099721 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558116f76fa0 0x558116f82000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.410+0000 7fb197fff640 1 -- 172.21.15.32:0/3663099721 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x558116f71440 msgr2=0x558116f7ee50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:09.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.410+0000 7fb197fff640 1 --2- 172.21.15.32:0/3663099721 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x558116f71440 0x558116f7ee50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.410+0000 7fb197fff640 1 -- 172.21.15.32:0/3663099721 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558116f825f0 con 0x558116f71e30 2024-04-25T00:33:09.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.410+0000 7fb197fff640 1 --2- 172.21.15.32:0/3663099721 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f71e30 0x558116f7f390 secure :-1 s=READY pgs=444 cs=0 l=1 rev1=1 crypto rx=0x7fb18c0314e0 tx=0x7fb18c0040b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=eb9da340461eb451 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:09.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.410+0000 7fb177fff640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:33:09.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.411+0000 7fb177fff640 1 -- 172.21.15.32:0/3663099721 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb18c035040 con 0x558116f71e30 2024-04-25T00:33:09.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.411+0000 7fb177fff640 1 -- 172.21.15.32:0/3663099721 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7fb18c004470 con 0x558116f71e30 2024-04-25T00:33:09.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.411+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/3663099721 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb190004030 con 0x558116f71e30 2024-04-25T00:33:09.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.411+0000 7fb177fff640 1 -- 172.21.15.32:0/3663099721 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) ==== 24+0+0 (secure 0 0 0) 0x7fb18c052050 con 0x558116f71e30 2024-04-25T00:33:09.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.411+0000 7fb1a1b7a740 10 client.15459 fetch_fsmap learned FSMap version 24 2024-04-25T00:33:09.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.411+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/3663099721 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) -- 0x558116fee420 con 0x558116f71e30 2024-04-25T00:33:09.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.411+0000 7fb177fff640 1 -- 172.21.15.32:0/3663099721 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb190004030 con 0x558116f71e30 2024-04-25T00:33:09.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.411+0000 7fb177fff640 1 -- 172.21.15.32:0/3663099721 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 24) ==== 8346+0+0 (secure 0 0 0) 0x7fb18c0444b0 con 0x558116f71e30 2024-04-25T00:33:09.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.411+0000 7fb1a1b7a740 10 client.15459 fetch_fsmap finished waiting for FSMap version 24 2024-04-25T00:33:09.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.411+0000 7fb1a1b7a740 10 client.15459 resolve_mds: resolved name 'b' to daemon mds.b 2024-04-25T00:33:09.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.411+0000 7fb1a1b7a740 20 client.15459 populate_metadata read hostname 'smithi032' 2024-04-25T00:33:09.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.411+0000 7fb1a1b7a740 1 --2- 172.21.15.32:0/3663099721 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x558116ff30c0 0x558116ff1dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.411+0000 7fb1a1b7a740 4 client.15459 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "status"}] 2024-04-25T00:33:09.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.411+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/3663099721 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "status"}) -- 0x558116edc790 con 0x558116ff30c0 2024-04-25T00:33:09.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.412+0000 7fb19cb2c640 1 --2- 172.21.15.32:0/3663099721 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x558116ff30c0 0x558116ff1dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.412+0000 7fb19cb2c640 1 --2- 172.21.15.32:0/3663099721 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x558116ff30c0 0x558116ff1dd0 crc :-1 s=READY pgs=28 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=42e460b0b37f8fbb server_cookie=508c9a31bb8d4007 in_seq=0 out_seq=0 2024-04-25T00:33:09.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.412+0000 7fb177fff640 10 client.15459 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:09.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.412+0000 7fb177fff640 1 -- 172.21.15.32:0/3663099721 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+332 (crc 0 0 0) 0x558116edc790 con 0x558116ff30c0 2024-04-25T00:33:09.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.412+0000 7fb177fff640 10 client.15459 handle_command_reply: tid=0 2024-04-25T00:33:09.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.412+0000 7fb1a1b7a740 1 client.15459 shutdown 2024-04-25T00:33:09.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.412+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/3663099721 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f71e30 msgr2=0x558116f7f390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.412+0000 7fb1a1b7a740 1 --2- 172.21.15.32:0/3663099721 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f71e30 0x558116f7f390 secure :-1 s=READY pgs=444 cs=0 l=1 rev1=1 crypto rx=0x7fb18c0314e0 tx=0x7fb18c0040b0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.412+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/3663099721 shutdown_connections 2024-04-25T00:33:09.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.412+0000 7fb1a1b7a740 1 --2- 172.21.15.32:0/3663099721 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x558116ff30c0 0x558116ff1dd0 crc :-1 s=READY pgs=28 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.412+0000 7fb1a1b7a740 1 --2- 172.21.15.32:0/3663099721 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558116f76fa0 0x558116f82000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.412+0000 7fb1a1b7a740 1 --2- 172.21.15.32:0/3663099721 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x558116f71e30 0x558116f7f390 unknown :-1 s=CLOSED pgs=444 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.412+0000 7fb1a1b7a740 1 --2- 172.21.15.32:0/3663099721 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x558116f71440 0x558116f7ee50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.412+0000 7fb177fff640 0 client.15459 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:09.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.412+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/3663099721 >> 172.21.15.32:0/3663099721 conn(0x558116f6cab0 msgr2=0x558116f67690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:09.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.412+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/3663099721 shutdown_connections 2024-04-25T00:33:09.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.412+0000 7fb1a1b7a740 1 -- 172.21.15.32:0/3663099721 wait complete. 2024-04-25T00:33:09.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.412+0000 7fb1a1b7a740 20 client.15459 trim_cache size 0 max 16384 2024-04-25T00:33:09.420 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:33:09.420 INFO:teuthology.orchestra.run.smithi032.stdout: "cluster_fsid": "cb997156-0299-11ef-bc93-c7b262605968", 2024-04-25T00:33:09.420 INFO:teuthology.orchestra.run.smithi032.stdout: "whoami": 0, 2024-04-25T00:33:09.420 INFO:teuthology.orchestra.run.smithi032.stdout: "id": 24506, 2024-04-25T00:33:09.420 INFO:teuthology.orchestra.run.smithi032.stdout: "want_state": "up:active", 2024-04-25T00:33:09.420 INFO:teuthology.orchestra.run.smithi032.stdout: "state": "up:active", 2024-04-25T00:33:09.420 INFO:teuthology.orchestra.run.smithi032.stdout: "fs_name": "cephfs", 2024-04-25T00:33:09.420 INFO:teuthology.orchestra.run.smithi032.stdout: "rank_uptime": 140.67705790299999, 2024-04-25T00:33:09.420 INFO:teuthology.orchestra.run.smithi032.stdout: "mdsmap_epoch": 24, 2024-04-25T00:33:09.420 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_epoch": 93, 2024-04-25T00:33:09.421 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_epoch_barrier": 89, 2024-04-25T00:33:09.421 INFO:teuthology.orchestra.run.smithi032.stdout: "uptime": 147.08359580199999 2024-04-25T00:33:09.421 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:33:09.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.413+0000 7fb19ed90640 1 -- 172.21.15.32:0/705257155 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb1800a2b90 msgr2=0x7fb1800a5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.413+0000 7fb19ed90640 1 --2- 172.21.15.32:0/705257155 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb1800a2b90 0x7fb1800a5050 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7fb1900098b0 tx=0x7fb190009e20 comp rx=0 tx=0).stop 2024-04-25T00:33:09.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.413+0000 7fb19ed90640 1 -- 172.21.15.32:0/705257155 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb198100680 msgr2=0x7fb1980694a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.413+0000 7fb19ed90640 1 --2- 172.21.15.32:0/705257155 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb198100680 0x7fb1980694a0 secure :-1 s=READY pgs=441 cs=0 l=1 rev1=1 crypto rx=0x7fb18800d770 tx=0x7fb18800dc30 comp rx=0 tx=0).stop 2024-04-25T00:33:09.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.413+0000 7fb19ed90640 1 -- 172.21.15.32:0/705257155 shutdown_connections 2024-04-25T00:33:09.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.413+0000 7fb19ed90640 1 --2- 172.21.15.32:0/705257155 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fb1800a2b90 0x7fb1800a5050 unknown :-1 s=CLOSED pgs=261 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.413+0000 7fb19ed90640 1 --2- 172.21.15.32:0/705257155 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb198102230 0x7fb19819ede0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.413+0000 7fb19ed90640 1 --2- 172.21.15.32:0/705257155 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fb198101870 0x7fb19819c770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.413+0000 7fb19ed90640 1 --2- 172.21.15.32:0/705257155 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fb198100680 0x7fb1980694a0 unknown :-1 s=CLOSED pgs=441 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.413+0000 7fb19ed90640 1 -- 172.21.15.32:0/705257155 >> 172.21.15.32:0/705257155 conn(0x7fb1980fbbf0 msgr2=0x7fb198104c40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:09.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.413+0000 7fb19ed90640 1 -- 172.21.15.32:0/705257155 shutdown_connections 2024-04-25T00:33:09.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.413+0000 7fb19ed90640 1 -- 172.21.15.32:0/705257155 wait complete. 2024-04-25T00:33:09.424 INFO:teuthology.run_tasks:Running task sequential... 2024-04-25T00:33:09.434 INFO:teuthology.task.sequential:In sequential, running task sequential... 2024-04-25T00:33:09.434 INFO:teuthology.task.sequential:In sequential, running task print... 2024-04-25T00:33:09.434 INFO:teuthology.task.print:Enabling mgr modules 2024-04-25T00:33:09.434 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-04-25T00:33:09.435 INFO:teuthology.task.exec:Executing custom commands... 2024-04-25T00:33:09.435 INFO:teuthology.task.exec:Running commands on role mon.a host ubuntu@smithi032.front.sepia.ceph.com 2024-04-25T00:33:09.435 DEBUG:teuthology.orchestra.run.smithi032:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph mgr module enable snap_schedule' 2024-04-25T00:33:09.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.537+0000 7fbdcdd14640 1 -- 172.21.15.32:0/3228861986 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbdc8100750 msgr2=0x7fbdc8100b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.537+0000 7fbdcdd14640 1 --2- 172.21.15.32:0/3228861986 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbdc8100750 0x7fbdc8100b70 secure :-1 s=READY pgs=445 cs=0 l=1 rev1=1 crypto rx=0x7fbdbc009900 tx=0x7fbdbc02a410 comp rx=0 tx=0).stop 2024-04-25T00:33:09.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.537+0000 7fbdcdd14640 1 -- 172.21.15.32:0/3228861986 shutdown_connections 2024-04-25T00:33:09.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.537+0000 7fbdcdd14640 1 --2- 172.21.15.32:0/3228861986 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbdc8102300 0x7fbdc8108b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.537+0000 7fbdcdd14640 1 --2- 172.21.15.32:0/3228861986 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbdc8101940 0x7fbdc8101dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.537+0000 7fbdcdd14640 1 --2- 172.21.15.32:0/3228861986 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbdc8100750 0x7fbdc8100b70 unknown :-1 s=CLOSED pgs=445 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.537+0000 7fbdcdd14640 1 -- 172.21.15.32:0/3228861986 >> 172.21.15.32:0/3228861986 conn(0x7fbdc80fbcc0 msgr2=0x7fbdc80fe100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:09.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.537+0000 7fbdcdd14640 1 -- 172.21.15.32:0/3228861986 shutdown_connections 2024-04-25T00:33:09.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.537+0000 7fbdcdd14640 1 -- 172.21.15.32:0/3228861986 wait complete. 2024-04-25T00:33:09.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.538+0000 7fbdcdd14640 1 Processor -- start 2024-04-25T00:33:09.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.538+0000 7fbdcdd14640 1 -- start start 2024-04-25T00:33:09.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.538+0000 7fbdcdd14640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbdc8100750 0x7fbdc81979c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.538+0000 7fbdcdd14640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbdc8101940 0x7fbdc8197f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.538+0000 7fbdcdd14640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbdc8102300 0x7fbdc819ef70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.538+0000 7fbdcdd14640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fbdc810d440 con 0x7fbdc8102300 2024-04-25T00:33:09.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.538+0000 7fbdcdd14640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fbdc810d2c0 con 0x7fbdc8100750 2024-04-25T00:33:09.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.538+0000 7fbdcdd14640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fbdc810d5c0 con 0x7fbdc8101940 2024-04-25T00:33:09.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.538+0000 7fbdc7fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbdc8102300 0x7fbdc819ef70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.538+0000 7fbdc7fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbdc8102300 0x7fbdc819ef70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44356/0 (socket says 172.21.15.32:44356) 2024-04-25T00:33:09.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.538+0000 7fbdc7fff640 1 -- 172.21.15.32:0/2578818065 learned_addr learned my addr 172.21.15.32:0/2578818065 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:09.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.538+0000 7fbdc6ffd640 1 --2- 172.21.15.32:0/2578818065 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbdc8101940 0x7fbdc8197f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.538+0000 7fbdc77fe640 1 --2- 172.21.15.32:0/2578818065 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbdc8100750 0x7fbdc81979c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.538+0000 7fbdc7fff640 1 -- 172.21.15.32:0/2578818065 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbdc8101940 msgr2=0x7fbdc8197f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.538+0000 7fbdc7fff640 1 --2- 172.21.15.32:0/2578818065 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbdc8101940 0x7fbdc8197f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.538+0000 7fbdc7fff640 1 -- 172.21.15.32:0/2578818065 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbdc8100750 msgr2=0x7fbdc81979c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:09.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.539+0000 7fbdc7fff640 1 --2- 172.21.15.32:0/2578818065 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbdc8100750 0x7fbdc81979c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:09.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.539+0000 7fbdc7fff640 1 -- 172.21.15.32:0/2578818065 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbdc819f5e0 con 0x7fbdc8102300 2024-04-25T00:33:09.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.539+0000 7fbdc7fff640 1 --2- 172.21.15.32:0/2578818065 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbdc8102300 0x7fbdc819ef70 secure :-1 s=READY pgs=446 cs=0 l=1 rev1=1 crypto rx=0x7fbdc000cce0 tx=0x7fbdc000a620 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:09.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.539+0000 7fbdc4ff9640 1 -- 172.21.15.32:0/2578818065 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbdc000f450 con 0x7fbdc8102300 2024-04-25T00:33:09.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.539+0000 7fbdcdd14640 1 -- 172.21.15.32:0/2578818065 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbdc819f8d0 con 0x7fbdc8102300 2024-04-25T00:33:09.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.539+0000 7fbdc4ff9640 1 -- 172.21.15.32:0/2578818065 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbdc000fad0 con 0x7fbdc8102300 2024-04-25T00:33:09.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.539+0000 7fbdc4ff9640 1 -- 172.21.15.32:0/2578818065 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbdc0018c10 con 0x7fbdc8102300 2024-04-25T00:33:09.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.539+0000 7fbdcdd14640 1 -- 172.21.15.32:0/2578818065 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fbdc819fe20 con 0x7fbdc8102300 2024-04-25T00:33:09.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.540+0000 7fbdcdd14640 1 -- 172.21.15.32:0/2578818065 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fbdc81079a0 con 0x7fbdc8102300 2024-04-25T00:33:09.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.541+0000 7fbdc4ff9640 1 -- 172.21.15.32:0/2578818065 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 18) ==== 137833+0+0 (secure 0 0 0) 0x7fbdc000f5f0 con 0x7fbdc8102300 2024-04-25T00:33:09.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.542+0000 7fbdc4ff9640 1 --2- 172.21.15.32:0/2578818065 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fbdb00a2c30 0x7fbdb00a50f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:09.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.542+0000 7fbdc77fe640 1 --2- 172.21.15.32:0/2578818065 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fbdb00a2c30 0x7fbdb00a50f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:09.544 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.542+0000 7fbdc4ff9640 1 -- 172.21.15.32:0/2578818065 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7fbdc00cfb60 con 0x7fbdc8102300 2024-04-25T00:33:09.544 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.543+0000 7fbdc77fe640 1 --2- 172.21.15.32:0/2578818065 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fbdb00a2c30 0x7fbdb00a50f0 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7fbdbc006010 tx=0x7fbdbc000bc0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:09.546 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.546+0000 7fbdc4ff9640 1 -- 172.21.15.32:0/2578818065 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fbdc0014030 con 0x7fbdc8102300 2024-04-25T00:33:09.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:09 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3398744650' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:33:09.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:09 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/4231447488' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:33:09.782 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:09.782+0000 7fbdcdd14640 1 -- 172.21.15.32:0/2578818065 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "mgr module enable", "module": "snap_schedule"} v 0) -- 0x7fbdc8062340 con 0x7fbdc8102300 2024-04-25T00:33:09.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:09 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3398744650' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:33:09.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:09 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/4231447488' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:33:09.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:09 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3398744650' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:33:09.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:09 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/4231447488' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:33:10.524 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.523+0000 7fbdc4ff9640 1 -- 172.21.15.32:0/2578818065 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "mgr module enable", "module": "snap_schedule"}]=0 v19) ==== 92+0+0 (secure 0 0 0) 0x7fbdc009c690 con 0x7fbdc8102300 2024-04-25T00:33:10.527 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.526+0000 7fbdcdd14640 1 -- 172.21.15.32:0/2578818065 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fbdb00a2c30 msgr2=0x7fbdb00a50f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:10.527 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.526+0000 7fbdcdd14640 1 --2- 172.21.15.32:0/2578818065 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fbdb00a2c30 0x7fbdb00a50f0 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7fbdbc006010 tx=0x7fbdbc000bc0 comp rx=0 tx=0).stop 2024-04-25T00:33:10.527 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.526+0000 7fbdcdd14640 1 -- 172.21.15.32:0/2578818065 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbdc8102300 msgr2=0x7fbdc819ef70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:10.527 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.526+0000 7fbdcdd14640 1 --2- 172.21.15.32:0/2578818065 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbdc8102300 0x7fbdc819ef70 secure :-1 s=READY pgs=446 cs=0 l=1 rev1=1 crypto rx=0x7fbdc000cce0 tx=0x7fbdc000a620 comp rx=0 tx=0).stop 2024-04-25T00:33:10.527 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.526+0000 7fbdcdd14640 1 -- 172.21.15.32:0/2578818065 shutdown_connections 2024-04-25T00:33:10.527 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.526+0000 7fbdcdd14640 1 --2- 172.21.15.32:0/2578818065 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7fbdb00a2c30 0x7fbdb00a50f0 unknown :-1 s=CLOSED pgs=262 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:10.527 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.526+0000 7fbdcdd14640 1 --2- 172.21.15.32:0/2578818065 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbdc8102300 0x7fbdc819ef70 unknown :-1 s=CLOSED pgs=446 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:10.527 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.526+0000 7fbdcdd14640 1 --2- 172.21.15.32:0/2578818065 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbdc8101940 0x7fbdc8197f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:10.527 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.526+0000 7fbdcdd14640 1 --2- 172.21.15.32:0/2578818065 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbdc8100750 0x7fbdc81979c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:10.527 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.526+0000 7fbdcdd14640 1 -- 172.21.15.32:0/2578818065 >> 172.21.15.32:0/2578818065 conn(0x7fbdc80fbcc0 msgr2=0x7fbdc8104b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:10.527 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.526+0000 7fbdcdd14640 1 -- 172.21.15.32:0/2578818065 shutdown_connections 2024-04-25T00:33:10.529 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.529+0000 7fbdcdd14640 1 -- 172.21.15.32:0/2578818065 wait complete. 2024-04-25T00:33:10.547 DEBUG:teuthology.orchestra.run.smithi032:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph config set mgr mgr/snap_schedule/allow_m_granularity true' 2024-04-25T00:33:10.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.656+0000 7f57925a5640 1 -- 172.21.15.32:0/4263838597 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f578c102250 msgr2=0x7f578c108ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:10.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.656+0000 7f57925a5640 1 --2- 172.21.15.32:0/4263838597 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f578c102250 0x7f578c108ae0 secure :-1 s=READY pgs=449 cs=0 l=1 rev1=1 crypto rx=0x7f5784009ca0 tx=0x7f578402a8c0 comp rx=0 tx=0).stop 2024-04-25T00:33:10.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.656+0000 7f57925a5640 1 -- 172.21.15.32:0/4263838597 shutdown_connections 2024-04-25T00:33:10.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.656+0000 7f57925a5640 1 --2- 172.21.15.32:0/4263838597 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f578c102250 0x7f578c108ae0 unknown :-1 s=CLOSED pgs=449 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:10.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.656+0000 7f57925a5640 1 --2- 172.21.15.32:0/4263838597 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f578c101890 0x7f578c101d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:10.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.656+0000 7f57925a5640 1 --2- 172.21.15.32:0/4263838597 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f578c1006a0 0x7f578c100ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:10.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.656+0000 7f57925a5640 1 -- 172.21.15.32:0/4263838597 >> 172.21.15.32:0/4263838597 conn(0x7f578c0fbc10 msgr2=0x7f578c0fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:10.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.656+0000 7f57925a5640 1 -- 172.21.15.32:0/4263838597 shutdown_connections 2024-04-25T00:33:10.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.656+0000 7f57925a5640 1 -- 172.21.15.32:0/4263838597 wait complete. 2024-04-25T00:33:10.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.657+0000 7f57925a5640 1 Processor -- start 2024-04-25T00:33:10.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.657+0000 7f57925a5640 1 -- start start 2024-04-25T00:33:10.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.657+0000 7f57925a5640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f578c1006a0 0x7f578c197a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:10.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.657+0000 7f57925a5640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f578c101890 0x7f578c197fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:10.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.657+0000 7f57925a5640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f578c102250 0x7f578c19f030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:10.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.657+0000 7f57925a5640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f578c10c320 con 0x7f578c102250 2024-04-25T00:33:10.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.657+0000 7f57925a5640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f578c10c1a0 con 0x7f578c1006a0 2024-04-25T00:33:10.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.657+0000 7f57925a5640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f578c10c4a0 con 0x7f578c101890 2024-04-25T00:33:10.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.657+0000 7f5790b42640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f578c102250 0x7f578c19f030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:10.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.657+0000 7f5790b42640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f578c102250 0x7f578c19f030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44392/0 (socket says 172.21.15.32:44392) 2024-04-25T00:33:10.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.657+0000 7f578b7fe640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f578c101890 0x7f578c197fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:10.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.657+0000 7f5790b42640 1 -- 172.21.15.32:0/1159047617 learned_addr learned my addr 172.21.15.32:0/1159047617 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:10.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.657+0000 7f578bfff640 1 --2- 172.21.15.32:0/1159047617 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f578c1006a0 0x7f578c197a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:10.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.658+0000 7f578b7fe640 1 -- 172.21.15.32:0/1159047617 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f578c1006a0 msgr2=0x7f578c197a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:10.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.658+0000 7f578b7fe640 1 --2- 172.21.15.32:0/1159047617 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f578c1006a0 0x7f578c197a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:10.661 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.658+0000 7f578b7fe640 1 -- 172.21.15.32:0/1159047617 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f578c102250 msgr2=0x7f578c19f030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:10.661 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.658+0000 7f578b7fe640 1 --2- 172.21.15.32:0/1159047617 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f578c102250 0x7f578c19f030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:10.661 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.658+0000 7f578b7fe640 1 -- 172.21.15.32:0/1159047617 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f578c19f6a0 con 0x7f578c101890 2024-04-25T00:33:10.661 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.658+0000 7f578bfff640 1 --2- 172.21.15.32:0/1159047617 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f578c1006a0 0x7f578c197a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:10.661 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.658+0000 7f578b7fe640 1 --2- 172.21.15.32:0/1159047617 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f578c101890 0x7f578c197fc0 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f578000e9f0 tx=0x7f578000eeb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:10.661 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.658+0000 7f57897fa640 1 -- 172.21.15.32:0/1159047617 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f578000ccc0 con 0x7f578c101890 2024-04-25T00:33:10.661 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.658+0000 7f57925a5640 1 -- 172.21.15.32:0/1159047617 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f578c19f990 con 0x7f578c101890 2024-04-25T00:33:10.661 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.658+0000 7f57897fa640 1 -- 172.21.15.32:0/1159047617 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5780005020 con 0x7f578c101890 2024-04-25T00:33:10.661 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.658+0000 7f57897fa640 1 -- 172.21.15.32:0/1159047617 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f57800049e0 con 0x7f578c101890 2024-04-25T00:33:10.661 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.658+0000 7f57925a5640 1 -- 172.21.15.32:0/1159047617 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f578c19fee0 con 0x7f578c101890 2024-04-25T00:33:10.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.660+0000 7f57925a5640 1 -- 172.21.15.32:0/1159047617 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f578c1a0170 con 0x7f578c101890 2024-04-25T00:33:10.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.660+0000 7f57897fa640 1 -- 172.21.15.32:0/1159047617 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 19) ==== 137850+0+0 (secure 0 0 0) 0x7f5780005560 con 0x7f578c101890 2024-04-25T00:33:10.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.661+0000 7f57897fa640 1 --2- 172.21.15.32:0/1159047617 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f57740a2c80 0x7f57740a5140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:10.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.661+0000 7f578bfff640 1 -- 172.21.15.32:0/1159047617 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f57740a2c80 msgr2=0x7f57740a5140 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/585810931 2024-04-25T00:33:10.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.661+0000 7f57897fa640 1 -- 172.21.15.32:0/1159047617 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7f57800cf3e0 con 0x7f578c101890 2024-04-25T00:33:10.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.661+0000 7f578bfff640 1 --2- 172.21.15.32:0/1159047617 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f57740a2c80 0x7f57740a5140 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-25T00:33:10.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.665+0000 7f57897fa640 1 -- 172.21.15.32:0/1159047617 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f57800051d0 con 0x7f578c101890 2024-04-25T00:33:10.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:10 smithi121 ceph-mon[32231]: pgmap v366: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 7 op/s 2024-04-25T00:33:10.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:10 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2578818065' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "snap_schedule"}]: dispatch 2024-04-25T00:33:10.837 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:10 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: ignoring --setuser ceph since I am not root 2024-04-25T00:33:10.838 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:10 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: ignoring --setgroup ceph since I am not root 2024-04-25T00:33:10.838 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:10 smithi121 ceph-mgr[33626]: -- 172.21.15.121:0/3356204404 <== mon.2 v2:172.21.15.121:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ed35b0c5a0 con 0x55ed35ae0800 2024-04-25T00:33:10.838 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:10 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:10.706+0000 7f058f8e0200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-25T00:33:10.853 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.853+0000 7f57925a5640 1 -- 172.21.15.32:0/1159047617 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command([{prefix=config set, name=mgr/snap_schedule/allow_m_granularity}] v 0) -- 0x7f578c1a0340 con 0x7f578c101890 2024-04-25T00:33:10.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.858+0000 7f57897fa640 1 -- 172.21.15.32:0/1159047617 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/snap_schedule/allow_m_granularity}]=0 v29)=0 v29) ==== 138+0+0 (secure 0 0 0) 0x7f578009bf70 con 0x7f578c101890 2024-04-25T00:33:10.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.860+0000 7f57925a5640 1 -- 172.21.15.32:0/1159047617 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f57740a2c80 msgr2=0x7f57740a5140 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:33:10.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.860+0000 7f57925a5640 1 --2- 172.21.15.32:0/1159047617 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f57740a2c80 0x7f57740a5140 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:10.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.860+0000 7f57925a5640 1 -- 172.21.15.32:0/1159047617 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f578c101890 msgr2=0x7f578c197fc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:10.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.860+0000 7f57925a5640 1 --2- 172.21.15.32:0/1159047617 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f578c101890 0x7f578c197fc0 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f578000e9f0 tx=0x7f578000eeb0 comp rx=0 tx=0).stop 2024-04-25T00:33:10.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.860+0000 7f57925a5640 1 -- 172.21.15.32:0/1159047617 shutdown_connections 2024-04-25T00:33:10.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.860+0000 7f57925a5640 1 --2- 172.21.15.32:0/1159047617 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f57740a2c80 0x7f57740a5140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:10.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.860+0000 7f57925a5640 1 --2- 172.21.15.32:0/1159047617 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f578c102250 0x7f578c19f030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:10.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.860+0000 7f57925a5640 1 --2- 172.21.15.32:0/1159047617 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f578c101890 0x7f578c197fc0 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:10.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.860+0000 7f57925a5640 1 --2- 172.21.15.32:0/1159047617 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f578c1006a0 0x7f578c197a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:10.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.860+0000 7f57925a5640 1 -- 172.21.15.32:0/1159047617 >> 172.21.15.32:0/1159047617 conn(0x7f578c0fbc10 msgr2=0x7f578c0fd610 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:10.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.860+0000 7f57925a5640 1 -- 172.21.15.32:0/1159047617 shutdown_connections 2024-04-25T00:33:10.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.860+0000 7f57925a5640 1 -- 172.21.15.32:0/1159047617 wait complete. 2024-04-25T00:33:10.871 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:10 smithi032 ceph-mon[29357]: pgmap v366: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 7 op/s 2024-04-25T00:33:10.871 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:10 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2578818065' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "snap_schedule"}]: dispatch 2024-04-25T00:33:10.871 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:10 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: ignoring --setuser ceph since I am not root 2024-04-25T00:33:10.871 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:10 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: ignoring --setgroup ceph since I am not root 2024-04-25T00:33:10.871 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:10 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:10.720+0000 7fc561bbf200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-25T00:33:10.872 DEBUG:teuthology.orchestra.run.smithi032:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph config set mgr mgr/snap_schedule/dump_on_update true' 2024-04-25T00:33:10.887 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:10 smithi130 ceph-mon[32525]: pgmap v366: 129 pgs: 129 active+clean; 641 KiB data, 371 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 7 op/s 2024-04-25T00:33:10.887 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:10 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2578818065' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "snap_schedule"}]: dispatch 2024-04-25T00:33:10.888 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:10 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: ignoring --setuser ceph since I am not root 2024-04-25T00:33:10.888 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:10 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: ignoring --setgroup ceph since I am not root 2024-04-25T00:33:10.888 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:10 smithi130 ceph-mgr[34370]: -- 172.21.15.130:0/3065491884 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557d38e205a0 con 0x557d38df4800 2024-04-25T00:33:10.888 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:10 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:10.703+0000 7f651a1f7200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-25T00:33:10.888 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:10 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:10.887+0000 7f651a1f7200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-25T00:33:10.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.977+0000 7f02100cd640 1 -- 172.21.15.32:0/4095380563 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0208101890 msgr2=0x7f0208101d10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:10.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.977+0000 7f02100cd640 1 --2- 172.21.15.32:0/4095380563 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0208101890 0x7f0208101d10 secure :-1 s=READY pgs=450 cs=0 l=1 rev1=1 crypto rx=0x7f0204009900 tx=0x7f020402a410 comp rx=0 tx=0).stop 2024-04-25T00:33:10.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.977+0000 7f02100cd640 1 -- 172.21.15.32:0/4095380563 shutdown_connections 2024-04-25T00:33:10.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.977+0000 7f02100cd640 1 --2- 172.21.15.32:0/4095380563 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0208102250 0x7f0208108ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:10.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.977+0000 7f02100cd640 1 --2- 172.21.15.32:0/4095380563 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0208101890 0x7f0208101d10 unknown :-1 s=CLOSED pgs=450 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:10.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.977+0000 7f02100cd640 1 --2- 172.21.15.32:0/4095380563 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f02081006a0 0x7f0208100ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:10.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.977+0000 7f02100cd640 1 -- 172.21.15.32:0/4095380563 >> 172.21.15.32:0/4095380563 conn(0x7f02080fbc10 msgr2=0x7f02080fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:10.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.977+0000 7f02100cd640 1 -- 172.21.15.32:0/4095380563 shutdown_connections 2024-04-25T00:33:10.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.977+0000 7f02100cd640 1 -- 172.21.15.32:0/4095380563 wait complete. 2024-04-25T00:33:10.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.977+0000 7f02100cd640 1 Processor -- start 2024-04-25T00:33:10.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.977+0000 7f02100cd640 1 -- start start 2024-04-25T00:33:10.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.978+0000 7f02100cd640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f02081006a0 0x7f0208197a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:10.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.978+0000 7f02100cd640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0208101890 0x7f0208197f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:10.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.978+0000 7f02100cd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0208102250 0x7f020819efb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:10.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.978+0000 7f02100cd640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f020810c320 con 0x7f0208102250 2024-04-25T00:33:10.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.978+0000 7f02100cd640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f020810c1a0 con 0x7f02081006a0 2024-04-25T00:33:10.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.978+0000 7f02100cd640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f020810c4a0 con 0x7f0208101890 2024-04-25T00:33:10.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.978+0000 7f020e66a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0208102250 0x7f020819efb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:10.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.978+0000 7f020e66a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0208102250 0x7f020819efb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44406/0 (socket says 172.21.15.32:44406) 2024-04-25T00:33:10.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.978+0000 7f020e66a640 1 -- 172.21.15.32:0/1084979672 learned_addr learned my addr 172.21.15.32:0/1084979672 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:10.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.978+0000 7f020d668640 1 --2- 172.21.15.32:0/1084979672 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0208101890 0x7f0208197f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:10.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.978+0000 7f020de69640 1 --2- 172.21.15.32:0/1084979672 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f02081006a0 0x7f0208197a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:10.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.978+0000 7f020e66a640 1 -- 172.21.15.32:0/1084979672 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0208101890 msgr2=0x7f0208197f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:10.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.978+0000 7f020e66a640 1 --2- 172.21.15.32:0/1084979672 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0208101890 0x7f0208197f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:10.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.978+0000 7f020e66a640 1 -- 172.21.15.32:0/1084979672 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f02081006a0 msgr2=0x7f0208197a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:10.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.978+0000 7f020e66a640 1 --2- 172.21.15.32:0/1084979672 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f02081006a0 0x7f0208197a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:10.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.978+0000 7f020e66a640 1 -- 172.21.15.32:0/1084979672 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f020819f620 con 0x7f0208102250 2024-04-25T00:33:10.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.978+0000 7f020e66a640 1 --2- 172.21.15.32:0/1084979672 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0208102250 0x7f020819efb0 secure :-1 s=READY pgs=451 cs=0 l=1 rev1=1 crypto rx=0x7f020000eb10 tx=0x7f020000c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:10.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.978+0000 7f01feffd640 1 -- 172.21.15.32:0/1084979672 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0200004b20 con 0x7f0208102250 2024-04-25T00:33:10.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.978+0000 7f01feffd640 1 -- 172.21.15.32:0/1084979672 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0200004cc0 con 0x7f0208102250 2024-04-25T00:33:10.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.978+0000 7f02100cd640 1 -- 172.21.15.32:0/1084979672 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f020819f910 con 0x7f0208102250 2024-04-25T00:33:10.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.978+0000 7f01feffd640 1 -- 172.21.15.32:0/1084979672 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0200010760 con 0x7f0208102250 2024-04-25T00:33:10.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.979+0000 7f02100cd640 1 -- 172.21.15.32:0/1084979672 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f020819fe30 con 0x7f0208102250 2024-04-25T00:33:10.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.979+0000 7f02100cd640 1 -- 172.21.15.32:0/1084979672 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f02081a0060 con 0x7f0208102250 2024-04-25T00:33:10.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.981+0000 7f01feffd640 1 -- 172.21.15.32:0/1084979672 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 19) ==== 137850+0+0 (secure 0 0 0) 0x7f0200005000 con 0x7f0208102250 2024-04-25T00:33:10.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.981+0000 7f01feffd640 1 --2- 172.21.15.32:0/1084979672 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f01f40a2c80 0x7f01f40a5140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:10.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.981+0000 7f020de69640 1 -- 172.21.15.32:0/1084979672 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f01f40a2c80 msgr2=0x7f01f40a5140 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/585810931 2024-04-25T00:33:10.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.982+0000 7f01feffd640 1 -- 172.21.15.32:0/1084979672 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7f02000cfe90 con 0x7f0208102250 2024-04-25T00:33:10.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.982+0000 7f020de69640 1 --2- 172.21.15.32:0/1084979672 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f01f40a2c80 0x7f01f40a5140 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-25T00:33:10.985 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:10.985+0000 7f01feffd640 1 -- 172.21.15.32:0/1084979672 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f020009ca20 con 0x7f0208102250 2024-04-25T00:33:11.141 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:10 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:10.989+0000 7f651a1f7200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-25T00:33:11.172 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:11.172+0000 7f02100cd640 1 -- 172.21.15.32:0/1084979672 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command([{prefix=config set, name=mgr/snap_schedule/dump_on_update}] v 0) -- 0x7f02081a08d0 con 0x7f0208102250 2024-04-25T00:33:11.176 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:11.175+0000 7f01feffd640 1 -- 172.21.15.32:0/1084979672 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/snap_schedule/dump_on_update}]=0 v30)=0 v30) ==== 133+0+0 (secure 0 0 0) 0x7f020009c320 con 0x7f0208102250 2024-04-25T00:33:11.178 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:11.178+0000 7f02100cd640 1 -- 172.21.15.32:0/1084979672 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f01f40a2c80 msgr2=0x7f01f40a5140 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:33:11.178 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:11.178+0000 7f02100cd640 1 --2- 172.21.15.32:0/1084979672 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f01f40a2c80 0x7f01f40a5140 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:11.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:11.178+0000 7f02100cd640 1 -- 172.21.15.32:0/1084979672 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0208102250 msgr2=0x7f020819efb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:11.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:11.178+0000 7f02100cd640 1 --2- 172.21.15.32:0/1084979672 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0208102250 0x7f020819efb0 secure :-1 s=READY pgs=451 cs=0 l=1 rev1=1 crypto rx=0x7f020000eb10 tx=0x7f020000c2d0 comp rx=0 tx=0).stop 2024-04-25T00:33:11.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:11.178+0000 7f02100cd640 1 -- 172.21.15.32:0/1084979672 shutdown_connections 2024-04-25T00:33:11.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:11.178+0000 7f02100cd640 1 --2- 172.21.15.32:0/1084979672 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f01f40a2c80 0x7f01f40a5140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:11.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:11.178+0000 7f02100cd640 1 --2- 172.21.15.32:0/1084979672 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0208102250 0x7f020819efb0 unknown :-1 s=CLOSED pgs=451 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:11.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:11.178+0000 7f02100cd640 1 --2- 172.21.15.32:0/1084979672 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0208101890 0x7f0208197f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:11.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:11.178+0000 7f02100cd640 1 --2- 172.21.15.32:0/1084979672 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f02081006a0 0x7f0208197a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:11.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:11.178+0000 7f02100cd640 1 -- 172.21.15.32:0/1084979672 >> 172.21.15.32:0/1084979672 conn(0x7f02080fbc10 msgr2=0x7f02080fd630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:11.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:11.178+0000 7f02100cd640 1 -- 172.21.15.32:0/1084979672 shutdown_connections 2024-04-25T00:33:11.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:11.178+0000 7f02100cd640 1 -- 172.21.15.32:0/1084979672 wait complete. 2024-04-25T00:33:11.188 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:10 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:10.916+0000 7fc561bbf200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-25T00:33:11.188 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:11 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:11.028+0000 7fc561bbf200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-25T00:33:11.189 INFO:teuthology.run_tasks:Running task kclient... 2024-04-25T00:33:11.205 INFO:tasks.kclient:Mounting kernel clients... 2024-04-25T00:33:11.205 INFO:tasks.kclient:config is {'client.0': {}} 2024-04-25T00:33:11.205 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:33:11.205 DEBUG:teuthology.orchestra.run.smithi032:> ip netns list 2024-04-25T00:33:11.221 INFO:teuthology.orchestra.run.smithi032.stdout:ceph-ns--home-ubuntu-cephtest-mnt.admin (id: 0) 2024-04-25T00:33:11.222 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:33:11.222 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip netns delete ceph-ns--home-ubuntu-cephtest-mnt.admin 2024-04-25T00:33:11.287 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:33:11.287 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip link delete ceph-brx 2024-04-25T00:33:11.300 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:10 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:10.889+0000 7f058f8e0200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-25T00:33:11.300 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:10 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:10.992+0000 7f058f8e0200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-25T00:33:11.300 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:11 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:11.299+0000 7f058f8e0200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-25T00:33:11.388 INFO:tasks.kclient:client.0 config is {'mntopts': ['ms_mode=crc', 'wsync'], 'syntax': 'v1', 'mount_subvol_num': 1} 2024-04-25T00:33:11.388 INFO:tasks.cephfs.mount:cephfs_mntpt = None 2024-04-25T00:33:11.389 INFO:tasks.cephfs.mount:hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-04-25T00:33:11.389 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph fs ls 2024-04-25T00:33:11.460 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:11 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:11.364+0000 7fc561bbf200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-25T00:33:11.587 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:11 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:11.384+0000 7f058f8e0200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-25T00:33:11.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:11 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2578818065' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "snap_schedule"}]': finished 2024-04-25T00:33:11.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:11 smithi121 ceph-mon[32231]: mgrmap e19: x(active, since 9m), standbys: z, y 2024-04-25T00:33:11.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:11 smithi121 ceph-mon[32231]: from='client.? ' entity='client.admin' 2024-04-25T00:33:11.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:11 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1084979672' entity='client.admin' 2024-04-25T00:33:11.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:11 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2578818065' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "snap_schedule"}]': finished 2024-04-25T00:33:11.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:11 smithi130 ceph-mon[32525]: mgrmap e19: x(active, since 9m), standbys: z, y 2024-04-25T00:33:11.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:11 smithi130 ceph-mon[32525]: from='client.? ' entity='client.admin' 2024-04-25T00:33:11.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:11 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1084979672' entity='client.admin' 2024-04-25T00:33:11.642 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:11 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:11.299+0000 7f651a1f7200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-25T00:33:11.642 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:11 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:11.383+0000 7f651a1f7200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-25T00:33:11.645 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:33:11.748 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:11 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:11.459+0000 7fc561bbf200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-25T00:33:11.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:11 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2578818065' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "snap_schedule"}]': finished 2024-04-25T00:33:11.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:11 smithi032 ceph-mon[29357]: mgrmap e19: x(active, since 9m), standbys: z, y 2024-04-25T00:33:11.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:11 smithi032 ceph-mon[29357]: from='client.? ' entity='client.admin' 2024-04-25T00:33:11.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:11 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1084979672' entity='client.admin' 2024-04-25T00:33:12.837 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:12 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:12.560+0000 7f058f8e0200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-25T00:33:12.837 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:12 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:12.728+0000 7f058f8e0200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-25T00:33:12.892 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:12 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:12.554+0000 7f651a1f7200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-25T00:33:12.892 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:12 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:12.722+0000 7f651a1f7200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-25T00:33:12.904 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:12 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:12.742+0000 7fc561bbf200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-25T00:33:13.176 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:12 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:12.892+0000 7f651a1f7200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-25T00:33:13.181 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:12 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:12.897+0000 7f058f8e0200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-25T00:33:13.213 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:12 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:12.924+0000 7fc561bbf200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-25T00:33:13.214 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:13 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:13.104+0000 7fc561bbf200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-25T00:33:13.468 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:13 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:13.175+0000 7f651a1f7200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-25T00:33:13.468 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:13 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:13.266+0000 7f651a1f7200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-25T00:33:13.473 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:13 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:13.181+0000 7f058f8e0200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-25T00:33:13.473 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:13 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:13.272+0000 7f058f8e0200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-25T00:33:13.492 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:13 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:13.415+0000 7fc561bbf200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-25T00:33:13.608 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.598+0000 7f51373be700 1 -- 172.21.15.32:0/3553508128 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f513006b2a0 msgr2=0x7f513010f110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:13.608 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.598+0000 7f51373be700 1 --2- 172.21.15.32:0/3553508128 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f513006b2a0 0x7f513010f110 secure :-1 s=READY pgs=452 cs=0 l=1 rev1=1 crypto rx=0x7f512800b0a0 tx=0x7f512800b3b0 comp rx=0 tx=0).stop 2024-04-25T00:33:13.609 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.598+0000 7f51373be700 1 -- 172.21.15.32:0/3553508128 shutdown_connections 2024-04-25T00:33:13.609 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.598+0000 7f51373be700 1 --2- 172.21.15.32:0/3553508128 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f513006b2a0 0x7f513010f110 unknown :-1 s=CLOSED pgs=452 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:13.609 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.598+0000 7f51373be700 1 --2- 172.21.15.32:0/3553508128 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f51301082d0 0x7f51301086b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:13.609 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.598+0000 7f51373be700 1 --2- 172.21.15.32:0/3553508128 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f513006a900 0x7f513006ad60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:13.609 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.598+0000 7f51373be700 1 -- 172.21.15.32:0/3553508128 >> 172.21.15.32:0/3553508128 conn(0x7f51300780e0 msgr2=0x7f51300784f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:13.609 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.598+0000 7f51373be700 1 -- 172.21.15.32:0/3553508128 shutdown_connections 2024-04-25T00:33:13.609 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.598+0000 7f51373be700 1 -- 172.21.15.32:0/3553508128 wait complete. 2024-04-25T00:33:13.609 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.598+0000 7f51373be700 1 Processor -- start 2024-04-25T00:33:13.609 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.598+0000 7f51373be700 1 -- start start 2024-04-25T00:33:13.609 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.599+0000 7f51373be700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f513006a900 0x7f513019e6a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:13.610 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.599+0000 7f51373be700 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f513006b2a0 0x7f513019ebe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:13.610 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.599+0000 7f51373be700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f51301082d0 0x7f51301987c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:13.610 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.599+0000 7f51373be700 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f5130112540 con 0x7f513006a900 2024-04-25T00:33:13.610 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.599+0000 7f51373be700 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f51301123c0 con 0x7f513006b2a0 2024-04-25T00:33:13.610 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.599+0000 7f51373be700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f51301126c0 con 0x7f51301082d0 2024-04-25T00:33:13.610 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.599+0000 7f513515a700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f513006a900 0x7f513019e6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:13.610 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.599+0000 7f513515a700 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f513006a900 0x7f513019e6a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:43924/0 (socket says 172.21.15.32:43924) 2024-04-25T00:33:13.610 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.599+0000 7f513515a700 1 -- 172.21.15.32:0/3679269590 learned_addr learned my addr 172.21.15.32:0/3679269590 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:13.610 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.599+0000 7f5134959700 1 --2- 172.21.15.32:0/3679269590 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f513006b2a0 0x7f513019ebe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:13.610 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.599+0000 7f513595b700 1 --2- 172.21.15.32:0/3679269590 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f51301082d0 0x7f51301987c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:13.610 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.599+0000 7f513515a700 1 -- 172.21.15.32:0/3679269590 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f51301082d0 msgr2=0x7f51301987c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:13.610 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.599+0000 7f513515a700 1 --2- 172.21.15.32:0/3679269590 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f51301082d0 0x7f51301987c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:13.611 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.599+0000 7f513515a700 1 -- 172.21.15.32:0/3679269590 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f513006b2a0 msgr2=0x7f513019ebe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:13.611 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.599+0000 7f513515a700 1 --2- 172.21.15.32:0/3679269590 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f513006b2a0 0x7f513019ebe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:13.611 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.599+0000 7f513515a700 1 -- 172.21.15.32:0/3679269590 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f51301990a0 con 0x7f513006a900 2024-04-25T00:33:13.611 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.599+0000 7f513595b700 1 --2- 172.21.15.32:0/3679269590 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f51301082d0 0x7f51301987c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:13.611 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.599+0000 7f5134959700 1 --2- 172.21.15.32:0/3679269590 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f513006b2a0 0x7f513019ebe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:13.611 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.599+0000 7f513515a700 1 --2- 172.21.15.32:0/3679269590 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f513006a900 0x7f513019e6a0 secure :-1 s=READY pgs=453 cs=0 l=1 rev1=1 crypto rx=0x7f512c00e9c0 tx=0x7f512c00ecd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:13.611 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.599+0000 7f51267fc700 1 -- 172.21.15.32:0/3679269590 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f512c00cbd0 con 0x7f513006a900 2024-04-25T00:33:13.611 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.600+0000 7f51267fc700 1 -- 172.21.15.32:0/3679269590 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f512c00ee60 con 0x7f513006a900 2024-04-25T00:33:13.611 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.600+0000 7f51373be700 1 -- 172.21.15.32:0/3679269590 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5130199390 con 0x7f513006a900 2024-04-25T00:33:13.611 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.600+0000 7f51267fc700 1 -- 172.21.15.32:0/3679269590 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f512c005630 con 0x7f513006a900 2024-04-25T00:33:13.611 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.600+0000 7f51373be700 1 -- 172.21.15.32:0/3679269590 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f51301a5620 con 0x7f513006a900 2024-04-25T00:33:13.611 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.601+0000 7f51373be700 1 -- 172.21.15.32:0/3679269590 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f513006a1b0 con 0x7f513006a900 2024-04-25T00:33:13.611 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.602+0000 7f51267fc700 1 -- 172.21.15.32:0/3679269590 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 19) ==== 137850+0+0 (secure 0 0 0) 0x7f512c004750 con 0x7f513006a900 2024-04-25T00:33:13.611 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.603+0000 7f51267fc700 1 --2- 172.21.15.32:0/3679269590 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f511c0a2c70 0x7f511c0a5130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:13.612 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.603+0000 7f5134959700 1 -- 172.21.15.32:0/3679269590 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f511c0a2c70 msgr2=0x7f511c0a5130 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/585810931 2024-04-25T00:33:13.612 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.603+0000 7f51267fc700 1 -- 172.21.15.32:0/3679269590 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7f512c0cf820 con 0x7f513006a900 2024-04-25T00:33:13.612 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.603+0000 7f5134959700 1 --2- 172.21.15.32:0/3679269590 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f511c0a2c70 0x7f511c0a5130 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-25T00:33:13.612 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.606+0000 7f51267fc700 1 -- 172.21.15.32:0/3679269590 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f512c053740 con 0x7f513006a900 2024-04-25T00:33:13.748 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:13 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:13.514+0000 7fc561bbf200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-25T00:33:13.748 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:13 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:13.730+0000 7fc561bbf200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-25T00:33:13.804 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.804+0000 7f5134959700 1 -- 172.21.15.32:0/3679269590 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f511c0a2c70 msgr2=0x7f511c0a5130 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/585810931 2024-04-25T00:33:13.804 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.804+0000 7f5134959700 1 --2- 172.21.15.32:0/3679269590 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f511c0a2c70 0x7f511c0a5130 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-04-25T00:33:13.837 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:13 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:13.473+0000 7f058f8e0200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-25T00:33:13.837 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:13 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:13.665+0000 7f058f8e0200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-25T00:33:13.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.863+0000 7f51373be700 1 -- 172.21.15.32:0/3679269590 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs ls"} v 0) -- 0x7f5130199ca0 con 0x7f513006a900 2024-04-25T00:33:13.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.863+0000 7f51267fc700 1 -- 172.21.15.32:0/3679269590 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v24) ==== 53+0+88 (secure 0 0 0) 0x7f512c09c3b0 con 0x7f513006a900 2024-04-25T00:33:13.864 INFO:teuthology.orchestra.run.smithi032.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data cephfs_data_ec ] 2024-04-25T00:33:13.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.866+0000 7f51373be700 1 -- 172.21.15.32:0/3679269590 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f511c0a2c70 msgr2=0x7f511c0a5130 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:33:13.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.866+0000 7f51373be700 1 --2- 172.21.15.32:0/3679269590 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f511c0a2c70 0x7f511c0a5130 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:13.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.866+0000 7f51373be700 1 -- 172.21.15.32:0/3679269590 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f513006a900 msgr2=0x7f513019e6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:13.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.866+0000 7f51373be700 1 --2- 172.21.15.32:0/3679269590 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f513006a900 0x7f513019e6a0 secure :-1 s=READY pgs=453 cs=0 l=1 rev1=1 crypto rx=0x7f512c00e9c0 tx=0x7f512c00ecd0 comp rx=0 tx=0).stop 2024-04-25T00:33:13.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.866+0000 7f51373be700 1 -- 172.21.15.32:0/3679269590 shutdown_connections 2024-04-25T00:33:13.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.866+0000 7f51373be700 1 --2- 172.21.15.32:0/3679269590 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f511c0a2c70 0x7f511c0a5130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:13.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.866+0000 7f51373be700 1 --2- 172.21.15.32:0/3679269590 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f51301082d0 0x7f51301987c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:13.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.866+0000 7f51373be700 1 --2- 172.21.15.32:0/3679269590 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f513006a900 0x7f513019e6a0 unknown :-1 s=CLOSED pgs=453 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:13.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.866+0000 7f51373be700 1 --2- 172.21.15.32:0/3679269590 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f513006b2a0 0x7f513019ebe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:13.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.866+0000 7f51373be700 1 -- 172.21.15.32:0/3679269590 >> 172.21.15.32:0/3679269590 conn(0x7f51300780e0 msgr2=0x7f5130100090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:13.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.866+0000 7f51373be700 1 -- 172.21.15.32:0/3679269590 shutdown_connections 2024-04-25T00:33:13.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:13.866+0000 7f51373be700 1 -- 172.21.15.32:0/3679269590 wait complete. 2024-04-25T00:33:13.891 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:13 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:13.467+0000 7f651a1f7200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-25T00:33:13.891 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:13 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:13.659+0000 7f651a1f7200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-25T00:33:14.188 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:13 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3679269590' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2024-04-25T00:33:14.188 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:13 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:13.943+0000 7fc561bbf200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-25T00:33:14.254 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:13 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3679269590' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2024-04-25T00:33:14.259 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:13 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3679269590' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2024-04-25T00:33:14.555 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2024-04-25T00:33:14.555 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-04-25T00:33:14.555 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi032.front.sepia.ceph.com 2024-04-25T00:33:14.555 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-04-25T00:33:14.556 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-04-25T00:33:14.556 INFO:tasks.cephfs.mount:self.cephfs_name = None 2024-04-25T00:33:14.556 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-04-25T00:33:14.556 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-04-25T00:33:14.556 INFO:tasks.cephfs.mount:Setting the 'None' netns for '/home/ubuntu/cephtest/mnt.0' 2024-04-25T00:33:14.556 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:33:14.556 DEBUG:teuthology.orchestra.run.smithi032:> ip addr 2024-04-25T00:33:14.572 INFO:teuthology.orchestra.run.smithi032.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-04-25T00:33:14.572 INFO:teuthology.orchestra.run.smithi032.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-04-25T00:33:14.572 INFO:teuthology.orchestra.run.smithi032.stdout: inet 127.0.0.1/8 scope host lo 2024-04-25T00:33:14.572 INFO:teuthology.orchestra.run.smithi032.stdout: valid_lft forever preferred_lft forever 2024-04-25T00:33:14.572 INFO:teuthology.orchestra.run.smithi032.stdout: inet6 ::1/128 scope host 2024-04-25T00:33:14.573 INFO:teuthology.orchestra.run.smithi032.stdout: valid_lft forever preferred_lft forever 2024-04-25T00:33:14.573 INFO:teuthology.orchestra.run.smithi032.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-04-25T00:33:14.573 INFO:teuthology.orchestra.run.smithi032.stdout: link/ether 0c:c4:7a:6b:fd:0c brd ff:ff:ff:ff:ff:ff 2024-04-25T00:33:14.573 INFO:teuthology.orchestra.run.smithi032.stdout: altname enp7s0f0 2024-04-25T00:33:14.573 INFO:teuthology.orchestra.run.smithi032.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-04-25T00:33:14.573 INFO:teuthology.orchestra.run.smithi032.stdout: link/ether 0c:c4:7a:6b:fd:0d brd ff:ff:ff:ff:ff:ff 2024-04-25T00:33:14.573 INFO:teuthology.orchestra.run.smithi032.stdout: altname enp7s0f1 2024-04-25T00:33:14.573 INFO:teuthology.orchestra.run.smithi032.stdout:4: ens1f0: mtu 1500 qdisc mq state UP group default qlen 1000 2024-04-25T00:33:14.575 INFO:teuthology.orchestra.run.smithi032.stdout: link/ether ac:1f:6b:f8:26:02 brd ff:ff:ff:ff:ff:ff 2024-04-25T00:33:14.575 INFO:teuthology.orchestra.run.smithi032.stdout: altname enp1s0f0 2024-04-25T00:33:14.575 INFO:teuthology.orchestra.run.smithi032.stdout: inet 172.21.15.32/20 brd 172.21.15.255 scope global dynamic noprefixroute ens1f0 2024-04-25T00:33:14.575 INFO:teuthology.orchestra.run.smithi032.stdout: valid_lft 42065sec preferred_lft 42065sec 2024-04-25T00:33:14.576 INFO:teuthology.orchestra.run.smithi032.stdout: inet6 fe80::ae1f:6bff:fef8:2602/64 scope link noprefixroute 2024-04-25T00:33:14.576 INFO:teuthology.orchestra.run.smithi032.stdout: valid_lft forever preferred_lft forever 2024-04-25T00:33:14.576 INFO:teuthology.orchestra.run.smithi032.stdout:5: ens1f1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-04-25T00:33:14.576 INFO:teuthology.orchestra.run.smithi032.stdout: link/ether ac:1f:6b:f8:26:03 brd ff:ff:ff:ff:ff:ff 2024-04-25T00:33:14.576 INFO:teuthology.orchestra.run.smithi032.stdout: altname enp1s0f1 2024-04-25T00:33:14.576 INFO:tasks.cephfs.mount:Setuping the 'ceph-brx' with 192.168.255.254/16 2024-04-25T00:33:14.576 DEBUG:teuthology.orchestra.run.smithi032:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-04-25T00:33:14.576 DEBUG:teuthology.orchestra.run.smithi032:> set -e 2024-04-25T00:33:14.576 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip link add name ceph-brx type bridge 2024-04-25T00:33:14.576 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip addr flush dev ceph-brx 2024-04-25T00:33:14.576 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip link set ceph-brx up 2024-04-25T00:33:14.577 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip addr add 192.168.255.254/16 brd 192.168.255.255 dev ceph-brx 2024-04-25T00:33:14.577 DEBUG:teuthology.orchestra.run.smithi032:> ') 2024-04-25T00:33:14.587 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:14 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:14.258+0000 7f058f8e0200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-25T00:33:14.641 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:14 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:14.253+0000 7f651a1f7200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-25T00:33:14.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:14 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-04-25T00:33:14.709 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:14 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-25T00:33:14.716 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:33:14.716 DEBUG:teuthology.orchestra.run.smithi032:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-04-25T00:33:14.740 INFO:teuthology.orchestra.run.smithi032.stdout:1 2024-04-25T00:33:14.742 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:33:14.742 DEBUG:teuthology.orchestra.run.smithi032:> route 2024-04-25T00:33:14.799 INFO:teuthology.orchestra.run.smithi032.stdout:Kernel IP routing table 2024-04-25T00:33:14.799 INFO:teuthology.orchestra.run.smithi032.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-04-25T00:33:14.799 INFO:teuthology.orchestra.run.smithi032.stdout:default _gateway 0.0.0.0 UG 100 0 0 ens1f0 2024-04-25T00:33:14.799 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 ens1f0 2024-04-25T00:33:14.799 INFO:teuthology.orchestra.run.smithi032.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-04-25T00:33:14.800 DEBUG:teuthology.orchestra.run.smithi032:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-04-25T00:33:14.800 DEBUG:teuthology.orchestra.run.smithi032:> set -e 2024-04-25T00:33:14.800 DEBUG:teuthology.orchestra.run.smithi032:> sudo iptables -A FORWARD -o ens1f0 -i ceph-brx -j ACCEPT 2024-04-25T00:33:14.800 DEBUG:teuthology.orchestra.run.smithi032:> sudo iptables -A FORWARD -i ens1f0 -o ceph-brx -j ACCEPT 2024-04-25T00:33:14.800 DEBUG:teuthology.orchestra.run.smithi032:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o ens1f0 -j MASQUERADE 2024-04-25T00:33:14.800 DEBUG:teuthology.orchestra.run.smithi032:> ') 2024-04-25T00:33:14.890 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:14 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-04-25T00:33:14.898 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:14 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:14.584+0000 7fc561bbf200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-25T00:33:14.922 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:14 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-25T00:33:14.929 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:33:14.929 DEBUG:teuthology.orchestra.run.smithi032:> ip netns list 2024-04-25T00:33:14.945 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:33:14.945 DEBUG:teuthology.orchestra.run.smithi032:> ip netns list-id 2024-04-25T00:33:14.962 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:14 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:14.668+0000 7f651a1f7200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-25T00:33:14.962 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:14 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:14.762+0000 7f651a1f7200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-25T00:33:14.967 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:14 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:14.675+0000 7f058f8e0200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-25T00:33:14.967 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:14 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:14.769+0000 7f058f8e0200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-25T00:33:15.001 DEBUG:teuthology.orchestra.run.smithi032:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-04-25T00:33:15.001 DEBUG:teuthology.orchestra.run.smithi032:> set -e 2024-04-25T00:33:15.001 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip netns add ceph-ns--home-ubuntu-cephtest-mnt.0 2024-04-25T00:33:15.001 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip netns set ceph-ns--home-ubuntu-cephtest-mnt.0 0 2024-04-25T00:33:15.001 DEBUG:teuthology.orchestra.run.smithi032:> ') 2024-04-25T00:33:15.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:15 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-04-25T00:33:15.116 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:15 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-25T00:33:15.122 INFO:tasks.cephfs.mount:Setuping the netns 'ceph-ns--home-ubuntu-cephtest-mnt.0' with 192.168.0.1/16 2024-04-25T00:33:15.122 DEBUG:teuthology.orchestra.run.smithi032:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-04-25T00:33:15.122 DEBUG:teuthology.orchestra.run.smithi032:> set -e 2024-04-25T00:33:15.122 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip link add veth0 netns ceph-ns--home-ubuntu-cephtest-mnt.0 type veth peer name brx.0 2024-04-25T00:33:15.123 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip addr add 192.168.0.1/16 brd 192.168.255.255 dev veth0 2024-04-25T00:33:15.123 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip link set veth0 up 2024-04-25T00:33:15.123 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip link set lo up 2024-04-25T00:33:15.123 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip route add default via 192.168.255.254 2024-04-25T00:33:15.123 DEBUG:teuthology.orchestra.run.smithi032:> ') 2024-04-25T00:33:15.152 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:15 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:15.045+0000 7fc561bbf200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-25T00:33:15.152 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:15 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:15.151+0000 7fc561bbf200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-25T00:33:15.172 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:15 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-04-25T00:33:15.233 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:15 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-25T00:33:15.239 DEBUG:teuthology.orchestra.run.smithi032:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-04-25T00:33:15.239 DEBUG:teuthology.orchestra.run.smithi032:> set -e 2024-04-25T00:33:15.239 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip link set brx.0 up 2024-04-25T00:33:15.240 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip link set dev brx.0 master ceph-brx 2024-04-25T00:33:15.240 DEBUG:teuthology.orchestra.run.smithi032:> ') 2024-04-25T00:33:15.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:15 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-04-25T00:33:15.337 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:14 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:14.966+0000 7f058f8e0200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-25T00:33:15.351 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:15 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-25T00:33:15.358 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-04-25T00:33:15.358 DEBUG:teuthology.orchestra.run.smithi032:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-04-25T00:33:15.373 INFO:teuthology.orchestra.run.smithi032.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-04-25T00:33:15.373 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-04-25T00:33:15.373 DEBUG:teuthology.orchestra.run.smithi032:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-04-25T00:33:15.391 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:14 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:14.961+0000 7f651a1f7200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-25T00:33:15.428 WARNING:tasks.cephfs.mount:You cannot specify both: cephfs_mntpt and mount_subvol_num 2024-04-25T00:33:15.428 INFO:tasks.cephfs.mount:Mounting subvol 1 for now 2024-04-25T00:33:15.428 INFO:tasks.cephfs.kernel_mount:mounting using device: :/volumes/qa/sv_1/13b120f4-666f-4737-942b-13f01a95a021 2024-04-25T00:33:15.428 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:33:15.429 DEBUG:teuthology.orchestra.run.smithi032:> sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/mount -t ceph :/volumes/qa/sv_1/13b120f4-666f-4737-942b-13f01a95a021 /home/ubuntu/cephtest/mnt.0 -v -o norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,name=0,mds_namespace=cephfs,ms_mode=crc,wsync 2024-04-25T00:33:15.490 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:15 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:15.375+0000 7fc561bbf200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-25T00:33:15.512 INFO:teuthology.orchestra.run.smithi032.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,name=0,mds_namespace=cephfs,ms_mode=crc,wsync 2024-04-25T00:33:15.512 INFO:teuthology.orchestra.run.smithi032.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,mds_namespace=cephfs,ms_mode=crc,wsync". 2024-04-25T00:33:15.512 INFO:teuthology.orchestra.run.smithi032.stdout:invalid new device string format 2024-04-25T00:33:15.655 INFO:teuthology.orchestra.run.smithi032.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,name=0,mds_namespace=cephfs,ms_mode=crc,wsync 2024-04-25T00:33:15.655 INFO:teuthology.orchestra.run.smithi032.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,mds_namespace=cephfs,ms_mode=crc,wsync". 2024-04-25T00:33:15.655 INFO:teuthology.orchestra.run.smithi032.stdout:invalid new device string format 2024-04-25T00:33:15.655 INFO:teuthology.orchestra.run.smithi032.stdout:mount.ceph: resolved to: "172.21.15.32:3300,172.21.15.121:3300,172.21.15.130:3300" 2024-04-25T00:33:15.655 INFO:teuthology.orchestra.run.smithi032.stdout:mount.ceph: trying mount with old device syntax: 172.21.15.32:3300,172.21.15.121:3300,172.21.15.130:3300:/volumes/qa/sv_1/13b120f4-666f-4737-942b-13f01a95a021 2024-04-25T00:33:15.655 INFO:teuthology.orchestra.run.smithi032.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,mds_namespace=cephfs,ms_mode=crc,wsync,key=0,fsid=cb997156-0299-11ef-bc93-c7b262605968" will pass to kernel 2024-04-25T00:33:15.656 INFO:teuthology.orchestra.run.smithi032.stdout:mount: /home/ubuntu/cephtest/mnt.0 does not contain SELinux labels. 2024-04-25T00:33:15.656 INFO:teuthology.orchestra.run.smithi032.stdout: You just mounted a file system that supports labels which does not 2024-04-25T00:33:15.656 INFO:teuthology.orchestra.run.smithi032.stdout: contain labels, onto an SELinux box. It is likely that confined 2024-04-25T00:33:15.656 INFO:teuthology.orchestra.run.smithi032.stdout: applications will generate AVC messages and not be allowed access to 2024-04-25T00:33:15.657 INFO:teuthology.orchestra.run.smithi032.stdout: this file system. For more details see restorecon(8) and mount(8). 2024-04-25T00:33:15.658 INFO:tasks.cephfs.kernel_mount:mount command passed 2024-04-25T00:33:15.658 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:33:15.659 DEBUG:teuthology.orchestra.run.smithi032:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-04-25T00:33:15.727 DEBUG:teuthology.orchestra.run.smithi032:> (cd /home/ubuntu/cephtest/mnt.0 && exec stdin-killer --timeout=900 -- bash -c 'getfattr --only-values -n ceph.client_id .') 2024-04-25T00:33:15.810 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:15 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:15.481+0000 7f651a1f7200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-25T00:33:15.810 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:15 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:15.571+0000 7f651a1f7200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-25T00:33:15.816 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:15 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:15.483+0000 7f058f8e0200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-25T00:33:15.816 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:15 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:15.573+0000 7f058f8e0200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-25T00:33:15.818 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:15 stdin-killer INFO: expiration expected; waiting 900 seconds for command to complete 2024-04-25T00:33:15.820 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:15 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-25T00:33:15.826 INFO:teuthology.orchestra.run.smithi032.stdout:client15504 2024-04-25T00:33:15.826 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-04-25T00:33:15.826 DEBUG:teuthology.orchestra.run.smithi032:> dd if=/proc/self/mounts of=/dev/stdout 2024-04-25T00:33:15.843 DEBUG:teuthology.orchestra.run.smithi032:> (cd /home/ubuntu/cephtest/mnt.0 && exec stdin-killer --timeout=900 -- bash -c 'getfattr --only-values -n ceph.client_id .') 2024-04-25T00:33:15.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:15 stdin-killer INFO: expiration expected; waiting 900 seconds for command to complete 2024-04-25T00:33:15.935 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:15 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-25T00:33:15.941 INFO:teuthology.orchestra.run.smithi032.stdout:client15504 2024-04-25T00:33:15.942 DEBUG:teuthology.orchestra.run.smithi032:> (cd /home/ubuntu/cephtest/mnt.0 && exec stdin-killer --timeout=900 -- bash -c 'getfattr --only-values -n ceph.client_id .') 2024-04-25T00:33:15.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:15 stdin-killer INFO: expiration expected; waiting 900 seconds for command to complete 2024-04-25T00:33:15.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:15 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-25T00:33:15.999 INFO:teuthology.orchestra.run.smithi032.stdout:client15504 2024-04-25T00:33:16.000 DEBUG:teuthology.orchestra.run.smithi032:> (cd /home/ubuntu/cephtest/mnt.0 && exec stdin-killer --timeout=300 -- bash -c 'sudo dd if=/sys/kernel/debug/ceph/cb997156-0299-11ef-bc93-c7b262605968.client15504/status') 2024-04-25T00:33:16.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-04-25T00:33:16.059 INFO:teuthology.orchestra.run.smithi032.stdout:instance: client.15504 (3)192.168.0.1:0/1002705358 2024-04-25T00:33:16.059 INFO:teuthology.orchestra.run.smithi032.stdout:blocklisted: false 2024-04-25T00:33:16.059 INFO:teuthology.orchestra.run.smithi032.stderr:0+1 records in 2024-04-25T00:33:16.059 INFO:teuthology.orchestra.run.smithi032.stderr:0+1 records out 2024-04-25T00:33:16.059 INFO:teuthology.orchestra.run.smithi032.stderr:70 bytes copied, 2.548e-05 s, 2.7 MB/s 2024-04-25T00:33:16.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-25T00:33:16.067 INFO:teuthology.run_tasks:Running task exec... 2024-04-25T00:33:16.076 INFO:teuthology.task.exec:Executing custom commands... 2024-04-25T00:33:16.076 INFO:teuthology.task.exec:Running commands on role mon.a host ubuntu@smithi032.front.sepia.ceph.com 2024-04-25T00:33:16.076 DEBUG:teuthology.orchestra.run.smithi032:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs set cephfs balance_automate true' 2024-04-25T00:33:16.087 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:15 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:15.815+0000 7f058f8e0200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-25T00:33:16.087 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:15 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:15.949+0000 7f058f8e0200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-25T00:33:16.087 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:16 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:16.043+0000 7f058f8e0200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-25T00:33:16.141 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:15 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:15.809+0000 7f651a1f7200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-25T00:33:16.142 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:15 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:15.940+0000 7f651a1f7200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-25T00:33:16.142 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:16 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:16.035+0000 7f651a1f7200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-25T00:33:16.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.182+0000 7f36df012640 1 -- 172.21.15.32:0/790353158 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f36d8103360 msgr2=0x7f36d8109bf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:16.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.182+0000 7f36df012640 1 --2- 172.21.15.32:0/790353158 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f36d8103360 0x7f36d8109bf0 secure :-1 s=READY pgs=455 cs=0 l=1 rev1=1 crypto rx=0x7f36c0009900 tx=0x7f36c002a410 comp rx=0 tx=0).stop 2024-04-25T00:33:16.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.182+0000 7f36df012640 1 -- 172.21.15.32:0/790353158 shutdown_connections 2024-04-25T00:33:16.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.182+0000 7f36df012640 1 --2- 172.21.15.32:0/790353158 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f36d8103360 0x7f36d8109bf0 unknown :-1 s=CLOSED pgs=455 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.182+0000 7f36df012640 1 --2- 172.21.15.32:0/790353158 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f36d81029a0 0x7f36d8102e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.182+0000 7f36df012640 1 --2- 172.21.15.32:0/790353158 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f36d81017b0 0x7f36d8101bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.184 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.182+0000 7f36df012640 1 -- 172.21.15.32:0/790353158 >> 172.21.15.32:0/790353158 conn(0x7f36d80fcd00 msgr2=0x7f36d80ff160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:16.184 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.183+0000 7f36df012640 1 -- 172.21.15.32:0/790353158 shutdown_connections 2024-04-25T00:33:16.184 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.183+0000 7f36df012640 1 -- 172.21.15.32:0/790353158 wait complete. 2024-04-25T00:33:16.186 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.183+0000 7f36df012640 1 Processor -- start 2024-04-25T00:33:16.186 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.183+0000 7f36df012640 1 -- start start 2024-04-25T00:33:16.186 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.183+0000 7f36df012640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f36d81017b0 0x7f36d80754f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:16.186 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.183+0000 7f36df012640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f36d81029a0 0x7f36d8073b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:16.186 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.183+0000 7f36df012640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f36d8103360 0x7f36d8074070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:16.186 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.183+0000 7f36df012640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f36d810d410 con 0x7f36d81017b0 2024-04-25T00:33:16.186 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.183+0000 7f36df012640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f36d810d290 con 0x7f36d8103360 2024-04-25T00:33:16.186 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.183+0000 7f36df012640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f36d810d590 con 0x7f36d81029a0 2024-04-25T00:33:16.187 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.183+0000 7f36dcdae640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f36d81017b0 0x7f36d80754f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:16.187 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.183+0000 7f36dcdae640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f36d81017b0 0x7f36d80754f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:43942/0 (socket says 172.21.15.32:43942) 2024-04-25T00:33:16.187 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.184+0000 7f36dcdae640 1 -- 172.21.15.32:0/796802912 learned_addr learned my addr 172.21.15.32:0/796802912 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:16.187 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.184+0000 7f36d7fff640 1 --2- 172.21.15.32:0/796802912 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f36d81029a0 0x7f36d8073b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:16.187 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.184+0000 7f36dd5af640 1 --2- 172.21.15.32:0/796802912 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f36d8103360 0x7f36d8074070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:16.187 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.184+0000 7f36dcdae640 1 -- 172.21.15.32:0/796802912 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f36d81029a0 msgr2=0x7f36d8073b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:16.187 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.184+0000 7f36dcdae640 1 --2- 172.21.15.32:0/796802912 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f36d81029a0 0x7f36d8073b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.187 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.184+0000 7f36dcdae640 1 -- 172.21.15.32:0/796802912 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f36d8103360 msgr2=0x7f36d8074070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:16.187 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.184+0000 7f36dcdae640 1 --2- 172.21.15.32:0/796802912 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f36d8103360 0x7f36d8074070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.187 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.184+0000 7f36dcdae640 1 -- 172.21.15.32:0/796802912 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f36d8074900 con 0x7f36d81017b0 2024-04-25T00:33:16.187 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.184+0000 7f36d7fff640 1 --2- 172.21.15.32:0/796802912 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f36d81029a0 0x7f36d8073b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:16.187 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.184+0000 7f36dcdae640 1 --2- 172.21.15.32:0/796802912 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f36d81017b0 0x7f36d80754f0 secure :-1 s=READY pgs=456 cs=0 l=1 rev1=1 crypto rx=0x7f36d0009d20 tx=0x7f36d000c6a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:16.187 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.184+0000 7f36d5ffb640 1 -- 172.21.15.32:0/796802912 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f36d0010630 con 0x7f36d81017b0 2024-04-25T00:33:16.187 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.184+0000 7f36d5ffb640 1 -- 172.21.15.32:0/796802912 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f36d0010cb0 con 0x7f36d81017b0 2024-04-25T00:33:16.187 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.184+0000 7f36df012640 1 -- 172.21.15.32:0/796802912 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f36d81a1a80 con 0x7f36d81017b0 2024-04-25T00:33:16.188 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.184+0000 7f36d5ffb640 1 -- 172.21.15.32:0/796802912 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f36d0018de0 con 0x7f36d81017b0 2024-04-25T00:33:16.188 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.185+0000 7f36df012640 1 -- 172.21.15.32:0/796802912 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f36d81a1fd0 con 0x7f36d81017b0 2024-04-25T00:33:16.188 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.186+0000 7f36df012640 1 -- 172.21.15.32:0/796802912 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f36d804f500 con 0x7f36d81017b0 2024-04-25T00:33:16.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.187+0000 7f36d5ffb640 1 -- 172.21.15.32:0/796802912 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 19) ==== 137850+0+0 (secure 0 0 0) 0x7f36d00107d0 con 0x7f36d81017b0 2024-04-25T00:33:16.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.187+0000 7f36d5ffb640 1 --2- 172.21.15.32:0/796802912 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f36c40a2c80 0x7f36c40a5140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:16.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.188+0000 7f36d7fff640 1 -- 172.21.15.32:0/796802912 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f36c40a2c80 msgr2=0x7f36c40a5140 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/585810931 2024-04-25T00:33:16.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.188+0000 7f36d5ffb640 1 -- 172.21.15.32:0/796802912 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7f36d00cfc90 con 0x7f36d81017b0 2024-04-25T00:33:16.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.188+0000 7f36d7fff640 1 --2- 172.21.15.32:0/796802912 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f36c40a2c80 0x7f36c40a5140 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-25T00:33:16.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.192+0000 7f36d5ffb640 1 -- 172.21.15.32:0/796802912 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f36d0053740 con 0x7f36d81017b0 2024-04-25T00:33:16.248 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:15 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:15.941+0000 7fc561bbf200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-25T00:33:16.248 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:16 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:16.042+0000 7fc561bbf200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-25T00:33:16.389 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.389+0000 7f36d7fff640 1 -- 172.21.15.32:0/796802912 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f36c40a2c80 msgr2=0x7f36c40a5140 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/585810931 2024-04-25T00:33:16.389 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.389+0000 7f36d7fff640 1 --2- 172.21.15.32:0/796802912 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f36c40a2c80 0x7f36c40a5140 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-04-25T00:33:16.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.419+0000 7f36df012640 1 -- 172.21.15.32:0/796802912 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "true"} v 0) -- 0x7f36d8108a00 con 0x7f36d81017b0 2024-04-25T00:33:16.471 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.471+0000 7f36d5ffb640 1 -- 172.21.15.32:0/796802912 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "true"}]=0 v25) ==== 117+0+0 (secure 0 0 0) 0x7f36d009c820 con 0x7f36d81017b0 2024-04-25T00:33:16.475 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.474+0000 7f36df012640 1 -- 172.21.15.32:0/796802912 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f36c40a2c80 msgr2=0x7f36c40a5140 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:33:16.475 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.474+0000 7f36df012640 1 --2- 172.21.15.32:0/796802912 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f36c40a2c80 0x7f36c40a5140 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.475 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.474+0000 7f36df012640 1 -- 172.21.15.32:0/796802912 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f36d81017b0 msgr2=0x7f36d80754f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:16.475 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.474+0000 7f36df012640 1 --2- 172.21.15.32:0/796802912 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f36d81017b0 0x7f36d80754f0 secure :-1 s=READY pgs=456 cs=0 l=1 rev1=1 crypto rx=0x7f36d0009d20 tx=0x7f36d000c6a0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.475 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.474+0000 7f36df012640 1 -- 172.21.15.32:0/796802912 shutdown_connections 2024-04-25T00:33:16.475 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.474+0000 7f36df012640 1 --2- 172.21.15.32:0/796802912 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f36c40a2c80 0x7f36c40a5140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.475 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.474+0000 7f36df012640 1 --2- 172.21.15.32:0/796802912 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f36d8103360 0x7f36d8074070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.475 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.474+0000 7f36df012640 1 --2- 172.21.15.32:0/796802912 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f36d81029a0 0x7f36d8073b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.475 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.474+0000 7f36df012640 1 --2- 172.21.15.32:0/796802912 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f36d81017b0 0x7f36d80754f0 unknown :-1 s=CLOSED pgs=456 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.475 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.474+0000 7f36df012640 1 -- 172.21.15.32:0/796802912 >> 172.21.15.32:0/796802912 conn(0x7f36d80fcd00 msgr2=0x7f36d8105bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:16.475 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.474+0000 7f36df012640 1 -- 172.21.15.32:0/796802912 shutdown_connections 2024-04-25T00:33:16.476 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.474+0000 7f36df012640 1 -- 172.21.15.32:0/796802912 wait complete. 2024-04-25T00:33:16.476 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:16 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:16.151+0000 7f651a1f7200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-25T00:33:16.476 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:16 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:16.244+0000 7f651a1f7200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-25T00:33:16.476 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:16 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:16.158+0000 7f058f8e0200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-25T00:33:16.476 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:16 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:16.250+0000 7f058f8e0200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-25T00:33:16.487 INFO:teuthology.run_tasks:Running task check-counter... 2024-04-25T00:33:16.498 INFO:teuthology.run_tasks:Running task fwd_scrub... 2024-04-25T00:33:16.514 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-25T00:33:16.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.617+0000 7f3d041ab640 1 -- 172.21.15.32:0/1865391252 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3cfc101870 msgr2=0x7f3cfc101cf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:16.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.617+0000 7f3d041ab640 1 --2- 172.21.15.32:0/1865391252 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3cfc101870 0x7f3cfc101cf0 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f3cf8009860 tx=0x7f3cf802a340 comp rx=0 tx=0).stop 2024-04-25T00:33:16.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.617+0000 7f3d041ab640 1 -- 172.21.15.32:0/1865391252 shutdown_connections 2024-04-25T00:33:16.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.617+0000 7f3d041ab640 1 --2- 172.21.15.32:0/1865391252 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3cfc102230 0x7f3cfc108ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.617+0000 7f3d041ab640 1 --2- 172.21.15.32:0/1865391252 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3cfc101870 0x7f3cfc101cf0 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.617+0000 7f3d041ab640 1 --2- 172.21.15.32:0/1865391252 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3cfc100680 0x7f3cfc100aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.617+0000 7f3d041ab640 1 -- 172.21.15.32:0/1865391252 >> 172.21.15.32:0/1865391252 conn(0x7f3cfc0fbbf0 msgr2=0x7f3cfc0fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:16.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.617+0000 7f3d041ab640 1 -- 172.21.15.32:0/1865391252 shutdown_connections 2024-04-25T00:33:16.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.617+0000 7f3d041ab640 1 -- 172.21.15.32:0/1865391252 wait complete. 2024-04-25T00:33:16.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.617+0000 7f3d041ab640 1 Processor -- start 2024-04-25T00:33:16.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.617+0000 7f3d041ab640 1 -- start start 2024-04-25T00:33:16.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.618+0000 7f3d041ab640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3cfc100680 0x7f3cfc199c90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:16.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.618+0000 7f3d041ab640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3cfc101870 0x7f3cfc19a1d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:16.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.618+0000 7f3d041ab640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3cfc102230 0x7f3cfc1a1230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:16.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.618+0000 7f3d041ab640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f3cfc10c300 con 0x7f3cfc101870 2024-04-25T00:33:16.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.618+0000 7f3d041ab640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f3cfc10c180 con 0x7f3cfc100680 2024-04-25T00:33:16.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.618+0000 7f3d041ab640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f3cfc10c480 con 0x7f3cfc102230 2024-04-25T00:33:16.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.618+0000 7f3d01f47640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3cfc100680 0x7f3cfc199c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:16.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.618+0000 7f3d01746640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3cfc101870 0x7f3cfc19a1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:16.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.618+0000 7f3d01f47640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3cfc100680 0x7f3cfc199c90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:34360/0 (socket says 172.21.15.32:34360) 2024-04-25T00:33:16.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.618+0000 7f3d01f47640 1 -- 172.21.15.32:0/1488625388 learned_addr learned my addr 172.21.15.32:0/1488625388 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:16.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.618+0000 7f3d02748640 1 --2- 172.21.15.32:0/1488625388 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3cfc102230 0x7f3cfc1a1230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:16.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.618+0000 7f3d01746640 1 -- 172.21.15.32:0/1488625388 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3cfc102230 msgr2=0x7f3cfc1a1230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:16.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.618+0000 7f3d01746640 1 --2- 172.21.15.32:0/1488625388 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3cfc102230 0x7f3cfc1a1230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.618+0000 7f3d01746640 1 -- 172.21.15.32:0/1488625388 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3cfc100680 msgr2=0x7f3cfc199c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:16.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.618+0000 7f3d01746640 1 --2- 172.21.15.32:0/1488625388 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3cfc100680 0x7f3cfc199c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.618+0000 7f3d01746640 1 -- 172.21.15.32:0/1488625388 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3cfc1a18d0 con 0x7f3cfc101870 2024-04-25T00:33:16.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.618+0000 7f3d01f47640 1 --2- 172.21.15.32:0/1488625388 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3cfc100680 0x7f3cfc199c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:16.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.618+0000 7f3d01746640 1 --2- 172.21.15.32:0/1488625388 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3cfc101870 0x7f3cfc19a1d0 secure :-1 s=READY pgs=457 cs=0 l=1 rev1=1 crypto rx=0x7f3cf8035040 tx=0x7f3cf8033d90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:16.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.618+0000 7f3ceeffd640 1 -- 172.21.15.32:0/1488625388 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3cf8037040 con 0x7f3cfc101870 2024-04-25T00:33:16.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.618+0000 7f3d041ab640 1 -- 172.21.15.32:0/1488625388 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3cfc1a1b60 con 0x7f3cfc101870 2024-04-25T00:33:16.622 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.618+0000 7f3ceeffd640 1 -- 172.21.15.32:0/1488625388 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3cf8004040 con 0x7f3cfc101870 2024-04-25T00:33:16.622 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.618+0000 7f3ceeffd640 1 -- 172.21.15.32:0/1488625388 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3cf80055c0 con 0x7f3cfc101870 2024-04-25T00:33:16.622 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.619+0000 7f3d041ab640 1 -- 172.21.15.32:0/1488625388 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f3cfc1a20a0 con 0x7f3cfc101870 2024-04-25T00:33:16.623 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.619+0000 7f3d041ab640 1 -- 172.21.15.32:0/1488625388 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f3cfc1a2330 con 0x7f3cfc101870 2024-04-25T00:33:16.623 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.621+0000 7f3ceeffd640 1 -- 172.21.15.32:0/1488625388 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 19) ==== 137850+0+0 (secure 0 0 0) 0x7f3cf8042020 con 0x7f3cfc101870 2024-04-25T00:33:16.623 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.622+0000 7f3ceeffd640 1 --2- 172.21.15.32:0/1488625388 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3ce80a2be0 0x7f3ce80a50a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:16.623 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.622+0000 7f3d01f47640 1 -- 172.21.15.32:0/1488625388 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3ce80a2be0 msgr2=0x7f3ce80a50a0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/585810931 2024-04-25T00:33:16.623 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.622+0000 7f3ceeffd640 1 -- 172.21.15.32:0/1488625388 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7f3cf80eed80 con 0x7f3cfc101870 2024-04-25T00:33:16.623 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.622+0000 7f3d01f47640 1 --2- 172.21.15.32:0/1488625388 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3ce80a2be0 0x7f3ce80a50a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-25T00:33:16.627 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.627+0000 7f3ceeffd640 1 -- 172.21.15.32:0/1488625388 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f3cf8049020 con 0x7f3cfc101870 2024-04-25T00:33:16.687 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:16 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/796802912' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "true"}]: dispatch 2024-04-25T00:33:16.687 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:16 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:16.309+0000 7fc561bbf200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-25T00:33:16.687 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:16 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:16.453+0000 7fc561bbf200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-25T00:33:16.687 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:16 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:16.559+0000 7fc561bbf200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-25T00:33:16.688 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:16 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:16.686+0000 7fc561bbf200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-25T00:33:16.820 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.820+0000 7f3d041ab640 1 -- 172.21.15.32:0/1488625388 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f3cfc1a2a20 con 0x7f3cfc101870 2024-04-25T00:33:16.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.822+0000 7f3ceeffd640 1 -- 172.21.15.32:0/1488625388 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v93) ==== 74+0+24843 (secure 0 0 0) 0x7f3cf80bb8b0 con 0x7f3cfc101870 2024-04-25T00:33:16.822 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:33:16.823 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":93,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:32:45.458564+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-25T00:30:46.239556+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-25T00:30:47.257182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2024-04-25T00:30:51.193975+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.b","mappings":[{"from":8,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":1}]},{"pgid":"2.f","mappings":[{"from":8,"to":11}]},{"pgid":"2.10","mappings":[{"from":2,"to":4}]},{"pgid":"3.6","mappings":[{"from":3,"to":6}]},{"pgid":"3.17","mappings":[{"from":4,"to":7}]},{"pgid":"3.1b","mappings":[{"from":5,"to":9}]},{"pgid":"3.20","mappings":[{"from":5,"to":9}]},{"pgid":"3.23","mappings":[{"from":4,"to":8}]},{"pgid":"3.2c","mappings":[{"from":4,"to":9}]},{"pgid":"4.c","mappings":[{"from":9,"to":10}]},{"pgid":"4.12","mappings":[{"from":9,"to":10}]},{"pgid":"4.1c","mappings":[{"from":9,"to":8}]},{"pgid":"4.1d","mappings":[{"from":5,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-num-failure-domains":"0","crush-osds-per-failure-domain":"0","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:33:16.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.823+0000 7f3d01f47640 1 -- 172.21.15.32:0/1488625388 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3ce80a2be0 msgr2=0x7f3ce80a50a0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/585810931 2024-04-25T00:33:16.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.823+0000 7f3d01f47640 1 --2- 172.21.15.32:0/1488625388 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3ce80a2be0 0x7f3ce80a50a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-04-25T00:33:16.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.824+0000 7f3d041ab640 1 -- 172.21.15.32:0/1488625388 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3ce80a2be0 msgr2=0x7f3ce80a50a0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:33:16.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.824+0000 7f3d041ab640 1 --2- 172.21.15.32:0/1488625388 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3ce80a2be0 0x7f3ce80a50a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.824+0000 7f3d041ab640 1 -- 172.21.15.32:0/1488625388 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3cfc101870 msgr2=0x7f3cfc19a1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:16.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.824+0000 7f3d041ab640 1 --2- 172.21.15.32:0/1488625388 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3cfc101870 0x7f3cfc19a1d0 secure :-1 s=READY pgs=457 cs=0 l=1 rev1=1 crypto rx=0x7f3cf8035040 tx=0x7f3cf8033d90 comp rx=0 tx=0).stop 2024-04-25T00:33:16.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.824+0000 7f3d041ab640 1 -- 172.21.15.32:0/1488625388 shutdown_connections 2024-04-25T00:33:16.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.824+0000 7f3d041ab640 1 --2- 172.21.15.32:0/1488625388 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f3ce80a2be0 0x7f3ce80a50a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.824+0000 7f3d041ab640 1 --2- 172.21.15.32:0/1488625388 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3cfc102230 0x7f3cfc1a1230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.824+0000 7f3d041ab640 1 --2- 172.21.15.32:0/1488625388 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3cfc101870 0x7f3cfc19a1d0 unknown :-1 s=CLOSED pgs=457 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.824+0000 7f3d041ab640 1 --2- 172.21.15.32:0/1488625388 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3cfc100680 0x7f3cfc199c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.824+0000 7f3d041ab640 1 -- 172.21.15.32:0/1488625388 >> 172.21.15.32:0/1488625388 conn(0x7f3cfc0fbbf0 msgr2=0x7f3cfc0fd670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:16.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.824+0000 7f3d041ab640 1 -- 172.21.15.32:0/1488625388 shutdown_connections 2024-04-25T00:33:16.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.824+0000 7f3d041ab640 1 -- 172.21.15.32:0/1488625388 wait complete. 2024-04-25T00:33:16.836 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-25T00:26:07.306839+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-04-25T00:30:46.239556+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '84', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.8799999952316284, 'score_stable': 1.8799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.8799999952316284, 'raw_score_stable': 1.8799999952316284, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-04-25T00:30:47.257182+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '85', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.05000000074505806}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_data_ec', 'create_time': '2024-04-25T00:30:51.193975+0000', 'flags': 5, 'flags_names': 'hashpspool,ec_overwrites', 'type': 3, 'size': 4, 'min_size': 3, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '89', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'cephfs_data_ec', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 8192, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}] 2024-04-25T00:33:16.837 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-25T00:33:16.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:16 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/796802912' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "true"}]: dispatch 2024-04-25T00:33:16.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:16 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/796802912' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "true"}]: dispatch 2024-04-25T00:33:16.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.947+0000 7f064fa5d640 1 -- 172.21.15.32:0/3407111943 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0648101790 msgr2=0x7f0648101bb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:16.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.947+0000 7f064fa5d640 1 --2- 172.21.15.32:0/3407111943 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0648101790 0x7f0648101bb0 secure :-1 s=READY pgs=458 cs=0 l=1 rev1=1 crypto rx=0x7f0644009900 tx=0x7f064402a410 comp rx=0 tx=0).stop 2024-04-25T00:33:16.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.947+0000 7f064fa5d640 1 -- 172.21.15.32:0/3407111943 shutdown_connections 2024-04-25T00:33:16.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.947+0000 7f064fa5d640 1 --2- 172.21.15.32:0/3407111943 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0648103340 0x7f0648109bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.947+0000 7f064fa5d640 1 --2- 172.21.15.32:0/3407111943 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0648102980 0x7f0648102e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.947+0000 7f064fa5d640 1 --2- 172.21.15.32:0/3407111943 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0648101790 0x7f0648101bb0 unknown :-1 s=CLOSED pgs=458 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.947+0000 7f064fa5d640 1 -- 172.21.15.32:0/3407111943 >> 172.21.15.32:0/3407111943 conn(0x7f06480fcd20 msgr2=0x7f06480ff140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:16.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.947+0000 7f064fa5d640 1 -- 172.21.15.32:0/3407111943 shutdown_connections 2024-04-25T00:33:16.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.947+0000 7f064fa5d640 1 -- 172.21.15.32:0/3407111943 wait complete. 2024-04-25T00:33:16.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.948+0000 7f064fa5d640 1 Processor -- start 2024-04-25T00:33:16.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.948+0000 7f064fa5d640 1 -- start start 2024-04-25T00:33:16.951 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.948+0000 7f064fa5d640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0648101790 0x7f0648197aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:16.951 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.948+0000 7f064fa5d640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0648102980 0x7f0648197fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:16.951 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.948+0000 7f064fa5d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0648103340 0x7f064819f050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:16.951 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.948+0000 7f064fa5d640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f064810d3e0 con 0x7f0648103340 2024-04-25T00:33:16.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.948+0000 7f064fa5d640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f064810d260 con 0x7f0648102980 2024-04-25T00:33:16.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.948+0000 7f064fa5d640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f064810d560 con 0x7f0648101790 2024-04-25T00:33:16.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.948+0000 7f064dffa640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0648103340 0x7f064819f050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:16.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.948+0000 7f064dffa640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0648103340 0x7f064819f050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:43992/0 (socket says 172.21.15.32:43992) 2024-04-25T00:33:16.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.948+0000 7f064dffa640 1 -- 172.21.15.32:0/763163142 learned_addr learned my addr 172.21.15.32:0/763163142 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:16.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.948+0000 7f064cff8640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0648102980 0x7f0648197fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:16.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.948+0000 7f064d7f9640 1 --2- 172.21.15.32:0/763163142 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0648101790 0x7f0648197aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:16.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.949+0000 7f064dffa640 1 -- 172.21.15.32:0/763163142 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0648101790 msgr2=0x7f0648197aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:16.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.949+0000 7f064dffa640 1 --2- 172.21.15.32:0/763163142 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0648101790 0x7f0648197aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.949+0000 7f064dffa640 1 -- 172.21.15.32:0/763163142 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0648102980 msgr2=0x7f0648197fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:16.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.949+0000 7f064dffa640 1 --2- 172.21.15.32:0/763163142 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0648102980 0x7f0648197fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:16.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.949+0000 7f064dffa640 1 -- 172.21.15.32:0/763163142 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f064819f6c0 con 0x7f0648103340 2024-04-25T00:33:16.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.949+0000 7f064cff8640 1 --2- 172.21.15.32:0/763163142 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0648102980 0x7f0648197fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:16.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.949+0000 7f064dffa640 1 --2- 172.21.15.32:0/763163142 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0648103340 0x7f064819f050 secure :-1 s=READY pgs=459 cs=0 l=1 rev1=1 crypto rx=0x7f064000eb10 tx=0x7f064000c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:16.953 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.949+0000 7f063e7fc640 1 -- 172.21.15.32:0/763163142 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0640004b20 con 0x7f0648103340 2024-04-25T00:33:16.953 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.949+0000 7f063e7fc640 1 -- 172.21.15.32:0/763163142 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0640004cc0 con 0x7f0648103340 2024-04-25T00:33:16.953 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.949+0000 7f064fa5d640 1 -- 172.21.15.32:0/763163142 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f064819f9b0 con 0x7f0648103340 2024-04-25T00:33:16.953 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.949+0000 7f063e7fc640 1 -- 172.21.15.32:0/763163142 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0640010760 con 0x7f0648103340 2024-04-25T00:33:16.953 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.949+0000 7f064fa5d640 1 -- 172.21.15.32:0/763163142 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f064819feb0 con 0x7f0648103340 2024-04-25T00:33:16.953 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.950+0000 7f064fa5d640 1 -- 172.21.15.32:0/763163142 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0648108b80 con 0x7f0648103340 2024-04-25T00:33:16.954 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.952+0000 7f063e7fc640 1 -- 172.21.15.32:0/763163142 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 19) ==== 137850+0+0 (secure 0 0 0) 0x7f0640005020 con 0x7f0648103340 2024-04-25T00:33:16.954 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.953+0000 7f063e7fc640 1 --2- 172.21.15.32:0/763163142 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f06340a2be0 0x7f06340a50a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:16.954 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.953+0000 7f064d7f9640 1 -- 172.21.15.32:0/763163142 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f06340a2be0 msgr2=0x7f06340a50a0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/585810931 2024-04-25T00:33:16.954 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.953+0000 7f063e7fc640 1 -- 172.21.15.32:0/763163142 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7f06400cfa20 con 0x7f0648103340 2024-04-25T00:33:16.954 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.953+0000 7f064d7f9640 1 --2- 172.21.15.32:0/763163142 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f06340a2be0 0x7f06340a50a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-25T00:33:16.958 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:16.957+0000 7f063e7fc640 1 -- 172.21.15.32:0/763163142 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f064009c5b0 con 0x7f0648103340 2024-04-25T00:33:16.998 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:16 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:16.788+0000 7fc561bbf200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-25T00:33:17.143 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.143+0000 7f064fa5d640 1 -- 172.21.15.32:0/763163142 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7f0648062340 con 0x7f0648103340 2024-04-25T00:33:17.144 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.144+0000 7f063e7fc640 1 -- 172.21.15.32:0/763163142 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v93) ==== 93+0+10 (secure 0 0 0) 0x7f064009beb0 con 0x7f0648103340 2024-04-25T00:33:17.144 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 1 2024-04-25T00:33:17.146 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.146+0000 7f064fa5d640 1 -- 172.21.15.32:0/763163142 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f06340a2be0 msgr2=0x7f06340a50a0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:33:17.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.146+0000 7f064fa5d640 1 --2- 172.21.15.32:0/763163142 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f06340a2be0 0x7f06340a50a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.146+0000 7f064fa5d640 1 -- 172.21.15.32:0/763163142 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0648103340 msgr2=0x7f064819f050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:17.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.146+0000 7f064fa5d640 1 --2- 172.21.15.32:0/763163142 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0648103340 0x7f064819f050 secure :-1 s=READY pgs=459 cs=0 l=1 rev1=1 crypto rx=0x7f064000eb10 tx=0x7f064000c2d0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.146+0000 7f064fa5d640 1 -- 172.21.15.32:0/763163142 shutdown_connections 2024-04-25T00:33:17.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.146+0000 7f064fa5d640 1 --2- 172.21.15.32:0/763163142 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f06340a2be0 0x7f06340a50a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.146+0000 7f064fa5d640 1 --2- 172.21.15.32:0/763163142 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0648103340 0x7f064819f050 unknown :-1 s=CLOSED pgs=459 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.146+0000 7f064fa5d640 1 --2- 172.21.15.32:0/763163142 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0648102980 0x7f0648197fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.146+0000 7f064fa5d640 1 --2- 172.21.15.32:0/763163142 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0648101790 0x7f0648197aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.146+0000 7f064fa5d640 1 -- 172.21.15.32:0/763163142 >> 172.21.15.32:0/763163142 conn(0x7f06480fcd20 msgr2=0x7f0648105d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:17.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.146+0000 7f064fa5d640 1 -- 172.21.15.32:0/763163142 shutdown_connections 2024-04-25T00:33:17.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.146+0000 7f064fa5d640 1 -- 172.21.15.32:0/763163142 wait complete. 2024-04-25T00:33:17.157 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-04-25T00:33:17.264 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.263+0000 7f5b3c8c3640 1 -- 172.21.15.32:0/2368083639 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5b38101890 msgr2=0x7f5b38101d10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:17.264 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.263+0000 7f5b3c8c3640 1 --2- 172.21.15.32:0/2368083639 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5b38101890 0x7f5b38101d10 secure :-1 s=READY pgs=460 cs=0 l=1 rev1=1 crypto rx=0x7f5b20009900 tx=0x7f5b2002a410 comp rx=0 tx=0).stop 2024-04-25T00:33:17.264 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.263+0000 7f5b3c8c3640 1 -- 172.21.15.32:0/2368083639 shutdown_connections 2024-04-25T00:33:17.264 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.263+0000 7f5b3c8c3640 1 --2- 172.21.15.32:0/2368083639 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5b38102250 0x7f5b38108ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.264 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.263+0000 7f5b3c8c3640 1 --2- 172.21.15.32:0/2368083639 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5b38101890 0x7f5b38101d10 unknown :-1 s=CLOSED pgs=460 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.264 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.263+0000 7f5b3c8c3640 1 --2- 172.21.15.32:0/2368083639 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5b381006a0 0x7f5b38100ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.263+0000 7f5b3c8c3640 1 -- 172.21.15.32:0/2368083639 >> 172.21.15.32:0/2368083639 conn(0x7f5b380fbc30 msgr2=0x7f5b380fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:17.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.263+0000 7f5b3c8c3640 1 -- 172.21.15.32:0/2368083639 shutdown_connections 2024-04-25T00:33:17.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.263+0000 7f5b3c8c3640 1 -- 172.21.15.32:0/2368083639 wait complete. 2024-04-25T00:33:17.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.264+0000 7f5b3c8c3640 1 Processor -- start 2024-04-25T00:33:17.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.264+0000 7f5b3c8c3640 1 -- start start 2024-04-25T00:33:17.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.264+0000 7f5b3c8c3640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5b381006a0 0x7f5b3810b020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:17.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.264+0000 7f5b3c8c3640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5b38101890 0x7f5b38109660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:17.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.264+0000 7f5b3c8c3640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5b38102250 0x7f5b38109ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:17.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.264+0000 7f5b3c8c3640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f5b3810d440 con 0x7f5b38102250 2024-04-25T00:33:17.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.264+0000 7f5b3c8c3640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f5b3810d2c0 con 0x7f5b381006a0 2024-04-25T00:33:17.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.264+0000 7f5b3c8c3640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f5b3810d5c0 con 0x7f5b38101890 2024-04-25T00:33:17.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.264+0000 7f5b36d9d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5b38102250 0x7f5b38109ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:17.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.264+0000 7f5b36d9d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5b38102250 0x7f5b38109ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44022/0 (socket says 172.21.15.32:44022) 2024-04-25T00:33:17.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.264+0000 7f5b36d9d640 1 -- 172.21.15.32:0/1545593835 learned_addr learned my addr 172.21.15.32:0/1545593835 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:17.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.264+0000 7f5b3659c640 1 --2- 172.21.15.32:0/1545593835 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5b381006a0 0x7f5b3810b020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:17.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.265+0000 7f5b35d9b640 1 --2- 172.21.15.32:0/1545593835 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5b38101890 0x7f5b38109660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:17.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.265+0000 7f5b3659c640 1 -- 172.21.15.32:0/1545593835 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5b38101890 msgr2=0x7f5b38109660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:17.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.265+0000 7f5b3659c640 1 --2- 172.21.15.32:0/1545593835 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5b38101890 0x7f5b38109660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.265+0000 7f5b3659c640 1 -- 172.21.15.32:0/1545593835 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5b38102250 msgr2=0x7f5b38109ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:17.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.265+0000 7f5b3659c640 1 --2- 172.21.15.32:0/1545593835 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5b38102250 0x7f5b38109ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.265+0000 7f5b3659c640 1 -- 172.21.15.32:0/1545593835 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5b3810a430 con 0x7f5b381006a0 2024-04-25T00:33:17.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.265+0000 7f5b36d9d640 1 --2- 172.21.15.32:0/1545593835 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5b38102250 0x7f5b38109ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:17.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.265+0000 7f5b35d9b640 1 --2- 172.21.15.32:0/1545593835 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5b38101890 0x7f5b38109660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:17.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.265+0000 7f5b3659c640 1 --2- 172.21.15.32:0/1545593835 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5b381006a0 0x7f5b3810b020 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f5b2800e9c0 tx=0x7f5b2800ee80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:17.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.265+0000 7f5b2f7fe640 1 -- 172.21.15.32:0/1545593835 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5b2800cd80 con 0x7f5b381006a0 2024-04-25T00:33:17.268 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.265+0000 7f5b3c8c3640 1 -- 172.21.15.32:0/1545593835 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5b381a1be0 con 0x7f5b381006a0 2024-04-25T00:33:17.269 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.265+0000 7f5b2f7fe640 1 -- 172.21.15.32:0/1545593835 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5b28010910 con 0x7f5b381006a0 2024-04-25T00:33:17.269 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.265+0000 7f5b2f7fe640 1 -- 172.21.15.32:0/1545593835 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5b28018980 con 0x7f5b381006a0 2024-04-25T00:33:17.269 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.265+0000 7f5b3c8c3640 1 -- 172.21.15.32:0/1545593835 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5b381a2150 con 0x7f5b381006a0 2024-04-25T00:33:17.269 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.266+0000 7f5b3c8c3640 1 -- 172.21.15.32:0/1545593835 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5b38107a90 con 0x7f5b381006a0 2024-04-25T00:33:17.270 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.268+0000 7f5b2f7fe640 1 -- 172.21.15.32:0/1545593835 <== mon.2 v2:172.21.15.121:3300/0 4 ==== mgrmap(e 19) ==== 137850+0+0 (secure 0 0 0) 0x7f5b2801f020 con 0x7f5b381006a0 2024-04-25T00:33:17.270 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.269+0000 7f5b2f7fe640 1 --2- 172.21.15.32:0/1545593835 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f5b240a2be0 0x7f5b240a50a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:17.270 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.269+0000 7f5b35d9b640 1 -- 172.21.15.32:0/1545593835 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f5b240a2be0 msgr2=0x7f5b240a50a0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/585810931 2024-04-25T00:33:17.270 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.269+0000 7f5b2f7fe640 1 -- 172.21.15.32:0/1545593835 <== mon.2 v2:172.21.15.121:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7f5b280cf620 con 0x7f5b381006a0 2024-04-25T00:33:17.270 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.269+0000 7f5b35d9b640 1 --2- 172.21.15.32:0/1545593835 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f5b240a2be0 0x7f5b240a50a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-25T00:33:17.272 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.272+0000 7f5b2f7fe640 1 -- 172.21.15.32:0/1545593835 <== mon.2 v2:172.21.15.121:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f5b2809c150 con 0x7f5b381006a0 2024-04-25T00:33:17.337 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:17 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:17.060+0000 7f058f8e0200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-25T00:33:17.337 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:33:17 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y[33622]: 2024-04-25T00:33:17.274+0000 7f058f8e0200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-25T00:33:17.392 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:17 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:17.056+0000 7f651a1f7200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-25T00:33:17.392 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:33:17 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z[34366]: 2024-04-25T00:33:17.269+0000 7f651a1f7200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-25T00:33:17.457 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.457+0000 7f5b3c8c3640 1 -- 172.21.15.32:0/1545593835 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) -- 0x7f5b3802d420 con 0x7f5b381006a0 2024-04-25T00:33:17.458 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.458+0000 7f5b2f7fe640 1 -- 172.21.15.32:0/1545593835 <== mon.2 v2:172.21.15.121:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v93) ==== 104+0+11 (secure 0 0 0) 0x7f5b2809ba50 con 0x7f5b381006a0 2024-04-25T00:33:17.458 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 32 2024-04-25T00:33:17.460 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.460+0000 7f5b3c8c3640 1 -- 172.21.15.32:0/1545593835 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f5b240a2be0 msgr2=0x7f5b240a50a0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:33:17.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.460+0000 7f5b3c8c3640 1 --2- 172.21.15.32:0/1545593835 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f5b240a2be0 0x7f5b240a50a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.460+0000 7f5b3c8c3640 1 -- 172.21.15.32:0/1545593835 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5b381006a0 msgr2=0x7f5b3810b020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:17.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.460+0000 7f5b3c8c3640 1 --2- 172.21.15.32:0/1545593835 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5b381006a0 0x7f5b3810b020 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f5b2800e9c0 tx=0x7f5b2800ee80 comp rx=0 tx=0).stop 2024-04-25T00:33:17.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.460+0000 7f5b3c8c3640 1 -- 172.21.15.32:0/1545593835 shutdown_connections 2024-04-25T00:33:17.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.460+0000 7f5b3c8c3640 1 --2- 172.21.15.32:0/1545593835 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f5b240a2be0 0x7f5b240a50a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.460+0000 7f5b3c8c3640 1 --2- 172.21.15.32:0/1545593835 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5b38102250 0x7f5b38109ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.460+0000 7f5b3c8c3640 1 --2- 172.21.15.32:0/1545593835 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5b38101890 0x7f5b38109660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.460+0000 7f5b3c8c3640 1 --2- 172.21.15.32:0/1545593835 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5b381006a0 0x7f5b3810b020 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.460+0000 7f5b3c8c3640 1 -- 172.21.15.32:0/1545593835 >> 172.21.15.32:0/1545593835 conn(0x7f5b380fbc30 msgr2=0x7f5b38104c60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:17.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.460+0000 7f5b3c8c3640 1 -- 172.21.15.32:0/1545593835 shutdown_connections 2024-04-25T00:33:17.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.460+0000 7f5b3c8c3640 1 -- 172.21.15.32:0/1545593835 wait complete. 2024-04-25T00:33:17.471 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-04-25T00:33:17.583 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.581+0000 7f00edc9e640 1 -- 172.21.15.32:0/3147690707 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f00e8102260 msgr2=0x7f00e8104650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:17.583 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.581+0000 7f00edc9e640 1 --2- 172.21.15.32:0/3147690707 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f00e8102260 0x7f00e8104650 secure :-1 s=READY pgs=461 cs=0 l=1 rev1=1 crypto rx=0x7f00d0009900 tx=0x7f00d002a410 comp rx=0 tx=0).stop 2024-04-25T00:33:17.583 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.581+0000 7f00edc9e640 1 -- 172.21.15.32:0/3147690707 shutdown_connections 2024-04-25T00:33:17.583 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.581+0000 7f00edc9e640 1 --2- 172.21.15.32:0/3147690707 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f00e81074c0 0x7f00e81098b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.581+0000 7f00edc9e640 1 --2- 172.21.15.32:0/3147690707 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f00e8104b90 0x7f00e8106f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.581+0000 7f00edc9e640 1 --2- 172.21.15.32:0/3147690707 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f00e8102260 0x7f00e8104650 unknown :-1 s=CLOSED pgs=461 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.581+0000 7f00edc9e640 1 -- 172.21.15.32:0/3147690707 >> 172.21.15.32:0/3147690707 conn(0x7f00e80fbbf0 msgr2=0x7f00e80fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:17.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.581+0000 7f00edc9e640 1 -- 172.21.15.32:0/3147690707 shutdown_connections 2024-04-25T00:33:17.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.581+0000 7f00edc9e640 1 -- 172.21.15.32:0/3147690707 wait complete. 2024-04-25T00:33:17.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.581+0000 7f00edc9e640 1 Processor -- start 2024-04-25T00:33:17.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.581+0000 7f00edc9e640 1 -- start start 2024-04-25T00:33:17.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.582+0000 7f00edc9e640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f00e8102260 0x7f00e8197af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:17.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.582+0000 7f00edc9e640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f00e8104b90 0x7f00e8198030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:17.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.582+0000 7f00edc9e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f00e81074c0 0x7f00e819f0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:17.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.582+0000 7f00edc9e640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f00e810d3f0 con 0x7f00e81074c0 2024-04-25T00:33:17.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.582+0000 7f00edc9e640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f00e810d270 con 0x7f00e8104b90 2024-04-25T00:33:17.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.582+0000 7f00edc9e640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f00e810d570 con 0x7f00e8102260 2024-04-25T00:33:17.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.582+0000 7f00e7fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f00e81074c0 0x7f00e819f0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:17.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.582+0000 7f00e6ffd640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f00e8104b90 0x7f00e8198030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:17.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.582+0000 7f00e6ffd640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f00e8104b90 0x7f00e8198030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:34430/0 (socket says 172.21.15.32:34430) 2024-04-25T00:33:17.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.582+0000 7f00e77fe640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f00e8102260 0x7f00e8197af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:17.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.582+0000 7f00e6ffd640 1 -- 172.21.15.32:0/3219007336 learned_addr learned my addr 172.21.15.32:0/3219007336 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:17.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.582+0000 7f00e7fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f00e81074c0 0x7f00e819f0a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44046/0 (socket says 172.21.15.32:44046) 2024-04-25T00:33:17.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.582+0000 7f00e6ffd640 1 -- 172.21.15.32:0/3219007336 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f00e8102260 msgr2=0x7f00e8197af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:17.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.582+0000 7f00e6ffd640 1 --2- 172.21.15.32:0/3219007336 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f00e8102260 0x7f00e8197af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.582+0000 7f00e6ffd640 1 -- 172.21.15.32:0/3219007336 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f00e81074c0 msgr2=0x7f00e819f0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:17.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.582+0000 7f00e6ffd640 1 --2- 172.21.15.32:0/3219007336 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f00e81074c0 0x7f00e819f0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.582+0000 7f00e6ffd640 1 -- 172.21.15.32:0/3219007336 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f00e819f710 con 0x7f00e8104b90 2024-04-25T00:33:17.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.582+0000 7f00e6ffd640 1 --2- 172.21.15.32:0/3219007336 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f00e8104b90 0x7f00e8198030 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f00d800e9c0 tx=0x7f00d800ee80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:17.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.583+0000 7f00e4ff9640 1 -- 172.21.15.32:0/3219007336 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f00d800cd80 con 0x7f00e8104b90 2024-04-25T00:33:17.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.583+0000 7f00e4ff9640 1 -- 172.21.15.32:0/3219007336 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f00d8005500 con 0x7f00e8104b90 2024-04-25T00:33:17.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.583+0000 7f00e4ff9640 1 -- 172.21.15.32:0/3219007336 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f00d8010690 con 0x7f00e8104b90 2024-04-25T00:33:17.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.583+0000 7f00edc9e640 1 -- 172.21.15.32:0/3219007336 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f00e819fa00 con 0x7f00e8104b90 2024-04-25T00:33:17.586 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.583+0000 7f00edc9e640 1 -- 172.21.15.32:0/3219007336 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f00e819ff50 con 0x7f00e8104b90 2024-04-25T00:33:17.586 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.584+0000 7f00edc9e640 1 -- 172.21.15.32:0/3219007336 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f00e80fd6e0 con 0x7f00e8104b90 2024-04-25T00:33:17.587 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.585+0000 7f00e4ff9640 1 -- 172.21.15.32:0/3219007336 <== mon.2 v2:172.21.15.121:3300/0 4 ==== mgrmap(e 20) ==== 137850+0+0 (secure 0 0 0) 0x7f00d80046c0 con 0x7f00e8104b90 2024-04-25T00:33:17.587 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.585+0000 7f00e4ff9640 1 --2- 172.21.15.32:0/3219007336 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f00d40a2be0 0x7f00d40a50a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:17.587 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.585+0000 7f00e77fe640 1 -- 172.21.15.32:0/3219007336 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f00d40a2be0 msgr2=0x7f00d40a50a0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/585810931 2024-04-25T00:33:17.587 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.585+0000 7f00e4ff9640 1 -- 172.21.15.32:0/3219007336 <== mon.2 v2:172.21.15.121:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7f00d80cf7f0 con 0x7f00e8104b90 2024-04-25T00:33:17.587 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.585+0000 7f00e77fe640 1 --2- 172.21.15.32:0/3219007336 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f00d40a2be0 0x7f00d40a50a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-25T00:33:17.590 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.590+0000 7f00e4ff9640 1 -- 172.21.15.32:0/3219007336 <== mon.2 v2:172.21.15.121:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f00d809c380 con 0x7f00e8104b90 2024-04-25T00:33:17.748 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:17 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:17.682+0000 7fc561bbf200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-25T00:33:17.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:17 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/796802912' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "true"}]': finished 2024-04-25T00:33:17.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:17 smithi032 ceph-mon[29357]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:33:17.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:17 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1488625388' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:33:17.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:17 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/763163142' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:33:17.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:17 smithi032 ceph-mon[29357]: Standby manager daemon z restarted 2024-04-25T00:33:17.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:17 smithi032 ceph-mon[29357]: Standby manager daemon z started 2024-04-25T00:33:17.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:17 smithi032 ceph-mon[29357]: from='mgr.? 172.21.15.130:0/3021170551' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2024-04-25T00:33:17.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:17 smithi032 ceph-mon[29357]: from='mgr.? 172.21.15.130:0/3021170551' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T00:33:17.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:17 smithi032 ceph-mon[29357]: from='mgr.? 172.21.15.130:0/3021170551' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2024-04-25T00:33:17.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:17 smithi032 ceph-mon[29357]: from='mgr.? 172.21.15.130:0/3021170551' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T00:33:17.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:17 smithi032 ceph-mon[29357]: Standby manager daemon y restarted 2024-04-25T00:33:17.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:17 smithi032 ceph-mon[29357]: Standby manager daemon y started 2024-04-25T00:33:17.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:17 smithi032 ceph-mon[29357]: from='mgr.? 172.21.15.121:0/3752882752' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-25T00:33:17.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:17 smithi032 ceph-mon[29357]: from='mgr.? 172.21.15.121:0/3752882752' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T00:33:17.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:17 smithi032 ceph-mon[29357]: from='mgr.? 172.21.15.121:0/3752882752' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-25T00:33:17.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:17 smithi032 ceph-mon[29357]: from='mgr.? 172.21.15.121:0/3752882752' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T00:33:17.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:17 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1545593835' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:33:17.775 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.775+0000 7f00edc9e640 1 -- 172.21.15.32:0/3219007336 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) -- 0x7f00e8062340 con 0x7f00e8104b90 2024-04-25T00:33:17.776 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.776+0000 7f00e4ff9640 1 -- 172.21.15.32:0/3219007336 <== mon.2 v2:172.21.15.121:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v93) ==== 100+0+11 (secure 0 0 0) 0x7f00d809bc80 con 0x7f00e8104b90 2024-04-25T00:33:17.776 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 64 2024-04-25T00:33:17.778 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.778+0000 7f00edc9e640 1 -- 172.21.15.32:0/3219007336 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f00d40a2be0 msgr2=0x7f00d40a50a0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:33:17.778 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.778+0000 7f00edc9e640 1 --2- 172.21.15.32:0/3219007336 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f00d40a2be0 0x7f00d40a50a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.778 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.778+0000 7f00edc9e640 1 -- 172.21.15.32:0/3219007336 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f00e8104b90 msgr2=0x7f00e8198030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:17.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.778+0000 7f00edc9e640 1 --2- 172.21.15.32:0/3219007336 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f00e8104b90 0x7f00e8198030 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f00d800e9c0 tx=0x7f00d800ee80 comp rx=0 tx=0).stop 2024-04-25T00:33:17.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.778+0000 7f00edc9e640 1 -- 172.21.15.32:0/3219007336 shutdown_connections 2024-04-25T00:33:17.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.778+0000 7f00edc9e640 1 --2- 172.21.15.32:0/3219007336 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f00d40a2be0 0x7f00d40a50a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.778+0000 7f00edc9e640 1 --2- 172.21.15.32:0/3219007336 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f00e81074c0 0x7f00e819f0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.778+0000 7f00edc9e640 1 --2- 172.21.15.32:0/3219007336 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f00e8104b90 0x7f00e8198030 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.778+0000 7f00edc9e640 1 --2- 172.21.15.32:0/3219007336 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f00e8102260 0x7f00e8197af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.778+0000 7f00edc9e640 1 -- 172.21.15.32:0/3219007336 >> 172.21.15.32:0/3219007336 conn(0x7f00e80fbbf0 msgr2=0x7f00e8102b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:17.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.778+0000 7f00edc9e640 1 -- 172.21.15.32:0/3219007336 shutdown_connections 2024-04-25T00:33:17.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.778+0000 7f00edc9e640 1 -- 172.21.15.32:0/3219007336 wait complete. 2024-04-25T00:33:17.790 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data_ec pg_num 2024-04-25T00:33:17.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:17 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/796802912' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "true"}]': finished 2024-04-25T00:33:17.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:17 smithi121 ceph-mon[32231]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:33:17.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:17 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1488625388' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:33:17.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:17 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/763163142' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:33:17.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:17 smithi121 ceph-mon[32231]: Standby manager daemon z restarted 2024-04-25T00:33:17.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:17 smithi121 ceph-mon[32231]: Standby manager daemon z started 2024-04-25T00:33:17.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:17 smithi121 ceph-mon[32231]: from='mgr.? 172.21.15.130:0/3021170551' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2024-04-25T00:33:17.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:17 smithi121 ceph-mon[32231]: from='mgr.? 172.21.15.130:0/3021170551' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T00:33:17.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:17 smithi121 ceph-mon[32231]: from='mgr.? 172.21.15.130:0/3021170551' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2024-04-25T00:33:17.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:17 smithi121 ceph-mon[32231]: from='mgr.? 172.21.15.130:0/3021170551' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T00:33:17.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:17 smithi121 ceph-mon[32231]: Standby manager daemon y restarted 2024-04-25T00:33:17.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:17 smithi121 ceph-mon[32231]: Standby manager daemon y started 2024-04-25T00:33:17.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:17 smithi121 ceph-mon[32231]: from='mgr.? 172.21.15.121:0/3752882752' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-25T00:33:17.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:17 smithi121 ceph-mon[32231]: from='mgr.? 172.21.15.121:0/3752882752' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T00:33:17.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:17 smithi121 ceph-mon[32231]: from='mgr.? 172.21.15.121:0/3752882752' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-25T00:33:17.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:17 smithi121 ceph-mon[32231]: from='mgr.? 172.21.15.121:0/3752882752' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T00:33:17.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:17 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1545593835' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:33:17.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:17 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/796802912' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "true"}]': finished 2024-04-25T00:33:17.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:17 smithi130 ceph-mon[32525]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:33:17.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:17 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1488625388' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:33:17.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:17 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/763163142' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:33:17.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:17 smithi130 ceph-mon[32525]: Standby manager daemon z restarted 2024-04-25T00:33:17.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:17 smithi130 ceph-mon[32525]: Standby manager daemon z started 2024-04-25T00:33:17.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:17 smithi130 ceph-mon[32525]: from='mgr.? 172.21.15.130:0/3021170551' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2024-04-25T00:33:17.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:17 smithi130 ceph-mon[32525]: from='mgr.? 172.21.15.130:0/3021170551' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T00:33:17.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:17 smithi130 ceph-mon[32525]: from='mgr.? 172.21.15.130:0/3021170551' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2024-04-25T00:33:17.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:17 smithi130 ceph-mon[32525]: from='mgr.? 172.21.15.130:0/3021170551' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T00:33:17.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:17 smithi130 ceph-mon[32525]: Standby manager daemon y restarted 2024-04-25T00:33:17.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:17 smithi130 ceph-mon[32525]: Standby manager daemon y started 2024-04-25T00:33:17.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:17 smithi130 ceph-mon[32525]: from='mgr.? 172.21.15.121:0/3752882752' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-25T00:33:17.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:17 smithi130 ceph-mon[32525]: from='mgr.? 172.21.15.121:0/3752882752' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T00:33:17.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:17 smithi130 ceph-mon[32525]: from='mgr.? 172.21.15.121:0/3752882752' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-25T00:33:17.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:17 smithi130 ceph-mon[32525]: from='mgr.? 172.21.15.121:0/3752882752' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T00:33:17.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:17 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1545593835' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:33:17.899 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.896+0000 7f8f3d09a640 1 -- 172.21.15.32:0/1745136287 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8f38101870 msgr2=0x7f8f38101cf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:17.899 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.896+0000 7f8f3d09a640 1 --2- 172.21.15.32:0/1745136287 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8f38101870 0x7f8f38101cf0 secure :-1 s=READY pgs=462 cs=0 l=1 rev1=1 crypto rx=0x7f8f28009900 tx=0x7f8f2802a410 comp rx=0 tx=0).stop 2024-04-25T00:33:17.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.896+0000 7f8f3d09a640 1 -- 172.21.15.32:0/1745136287 shutdown_connections 2024-04-25T00:33:17.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.896+0000 7f8f3d09a640 1 --2- 172.21.15.32:0/1745136287 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8f38102230 0x7f8f38108ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.896+0000 7f8f3d09a640 1 --2- 172.21.15.32:0/1745136287 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8f38101870 0x7f8f38101cf0 unknown :-1 s=CLOSED pgs=462 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.896+0000 7f8f3d09a640 1 --2- 172.21.15.32:0/1745136287 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8f38100680 0x7f8f38100aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.896+0000 7f8f3d09a640 1 -- 172.21.15.32:0/1745136287 >> 172.21.15.32:0/1745136287 conn(0x7f8f380fbbf0 msgr2=0x7f8f380fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:17.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.896+0000 7f8f3d09a640 1 -- 172.21.15.32:0/1745136287 shutdown_connections 2024-04-25T00:33:17.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.896+0000 7f8f3d09a640 1 -- 172.21.15.32:0/1745136287 wait complete. 2024-04-25T00:33:17.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.896+0000 7f8f3d09a640 1 Processor -- start 2024-04-25T00:33:17.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.897+0000 7f8f3d09a640 1 -- start start 2024-04-25T00:33:17.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.897+0000 7f8f3d09a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8f38100680 0x7f8f38197ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:17.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.897+0000 7f8f3d09a640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8f38101870 0x7f8f38198010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:17.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.897+0000 7f8f3d09a640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8f38102230 0x7f8f3819f080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:17.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.897+0000 7f8f3d09a640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f8f3810c2e0 con 0x7f8f38100680 2024-04-25T00:33:17.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.897+0000 7f8f3d09a640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f8f3810c160 con 0x7f8f38102230 2024-04-25T00:33:17.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.897+0000 7f8f3d09a640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f8f3810c460 con 0x7f8f38101870 2024-04-25T00:33:17.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.897+0000 7f8f36d9d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8f38100680 0x7f8f38197ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:17.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.897+0000 7f8f36d9d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8f38100680 0x7f8f38197ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44064/0 (socket says 172.21.15.32:44064) 2024-04-25T00:33:17.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.897+0000 7f8f36d9d640 1 -- 172.21.15.32:0/2316703521 learned_addr learned my addr 172.21.15.32:0/2316703521 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:17.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.897+0000 7f8f3659c640 1 --2- 172.21.15.32:0/2316703521 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8f38101870 0x7f8f38198010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:17.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.897+0000 7f8f3759e640 1 --2- 172.21.15.32:0/2316703521 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8f38102230 0x7f8f3819f080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:17.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.897+0000 7f8f36d9d640 1 -- 172.21.15.32:0/2316703521 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8f38101870 msgr2=0x7f8f38198010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:17.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.897+0000 7f8f36d9d640 1 --2- 172.21.15.32:0/2316703521 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8f38101870 0x7f8f38198010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.897+0000 7f8f36d9d640 1 -- 172.21.15.32:0/2316703521 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8f38102230 msgr2=0x7f8f3819f080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:17.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.897+0000 7f8f36d9d640 1 --2- 172.21.15.32:0/2316703521 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8f38102230 0x7f8f3819f080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:17.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.897+0000 7f8f36d9d640 1 -- 172.21.15.32:0/2316703521 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8f3819f6f0 con 0x7f8f38100680 2024-04-25T00:33:17.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.897+0000 7f8f3759e640 1 --2- 172.21.15.32:0/2316703521 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8f38102230 0x7f8f3819f080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:17.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.897+0000 7f8f3659c640 1 --2- 172.21.15.32:0/2316703521 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8f38101870 0x7f8f38198010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:17.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.898+0000 7f8f36d9d640 1 --2- 172.21.15.32:0/2316703521 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8f38100680 0x7f8f38197ad0 secure :-1 s=READY pgs=463 cs=0 l=1 rev1=1 crypto rx=0x7f8f2c00e9c0 tx=0x7f8f2c00ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:17.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.898+0000 7f8f1ffff640 1 -- 172.21.15.32:0/2316703521 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8f2c00cd80 con 0x7f8f38100680 2024-04-25T00:33:17.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.898+0000 7f8f3d09a640 1 -- 172.21.15.32:0/2316703521 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8f3819f9e0 con 0x7f8f38100680 2024-04-25T00:33:17.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.898+0000 7f8f1ffff640 1 -- 172.21.15.32:0/2316703521 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8f2c0035f0 con 0x7f8f38100680 2024-04-25T00:33:17.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.898+0000 7f8f1ffff640 1 -- 172.21.15.32:0/2316703521 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8f2c010690 con 0x7f8f38100680 2024-04-25T00:33:17.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.898+0000 7f8f3d09a640 1 -- 172.21.15.32:0/2316703521 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8f3819ff00 con 0x7f8f38100680 2024-04-25T00:33:17.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.899+0000 7f8f3d09a640 1 -- 172.21.15.32:0/2316703521 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8f381a0190 con 0x7f8f38100680 2024-04-25T00:33:17.903 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.900+0000 7f8f1ffff640 1 -- 172.21.15.32:0/2316703521 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 20) ==== 137850+0+0 (secure 0 0 0) 0x7f8f2c0037a0 con 0x7f8f38100680 2024-04-25T00:33:17.903 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.901+0000 7f8f1ffff640 1 --2- 172.21.15.32:0/2316703521 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8f200a2be0 0x7f8f200a50a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:17.903 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.901+0000 7f8f3659c640 1 -- 172.21.15.32:0/2316703521 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8f200a2be0 msgr2=0x7f8f200a50a0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.32:6800/585810931 2024-04-25T00:33:17.903 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.901+0000 7f8f1ffff640 1 -- 172.21.15.32:0/2316703521 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 9208+0+0 (secure 0 0 0) 0x7f8f2c0cf9e0 con 0x7f8f38100680 2024-04-25T00:33:17.903 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.901+0000 7f8f3659c640 1 --2- 172.21.15.32:0/2316703521 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8f200a2be0 0x7f8f200a50a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-25T00:33:17.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.905+0000 7f8f1ffff640 1 -- 172.21.15.32:0/2316703521 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f8f2c09c5a0 con 0x7f8f38100680 2024-04-25T00:33:17.930 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.930+0000 7f8f1ffff640 1 -- 172.21.15.32:0/2316703521 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mgrmap(e 21) ==== 137518+0+0 (secure 0 0 0) 0x7f8f2c098510 con 0x7f8f38100680 2024-04-25T00:33:17.931 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.930+0000 7f8f1ffff640 1 -- 172.21.15.32:0/2316703521 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8f200a2be0 msgr2=0x7f8f200a50a0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:33:17.931 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:17.930+0000 7f8f1ffff640 1 --2- 172.21.15.32:0/2316703521 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8f200a2be0 0x7f8f200a50a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.093 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:17 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:17.917+0000 7fc561bbf200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-25T00:33:18.149 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.148+0000 7f8f3d09a640 1 -- 172.21.15.32:0/2316703521 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"} v 0) -- 0x7f8f381a0360 con 0x7f8f38100680 2024-04-25T00:33:18.149 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.149+0000 7f8f1ffff640 1 -- 172.21.15.32:0/2316703521 <== mon.0 v2:172.21.15.32:3300/0 8 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]=0 v94) ==== 103+0+11 (secure 0 0 0) 0x7f8f2c0c5d00 con 0x7f8f38100680 2024-04-25T00:33:18.149 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 32 2024-04-25T00:33:18.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.151+0000 7f8f3d09a640 1 -- 172.21.15.32:0/2316703521 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8f38100680 msgr2=0x7f8f38197ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:18.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.151+0000 7f8f3d09a640 1 --2- 172.21.15.32:0/2316703521 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8f38100680 0x7f8f38197ad0 secure :-1 s=READY pgs=463 cs=0 l=1 rev1=1 crypto rx=0x7f8f2c00e9c0 tx=0x7f8f2c00ee80 comp rx=0 tx=0).stop 2024-04-25T00:33:18.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.151+0000 7f8f3d09a640 1 -- 172.21.15.32:0/2316703521 shutdown_connections 2024-04-25T00:33:18.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.151+0000 7f8f3d09a640 1 --2- 172.21.15.32:0/2316703521 >> [v2:172.21.15.32:6800/585810931,v1:172.21.15.32:6801/585810931] conn(0x7f8f200a2be0 0x7f8f200a50a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.151+0000 7f8f3d09a640 1 --2- 172.21.15.32:0/2316703521 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8f38102230 0x7f8f3819f080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.151+0000 7f8f3d09a640 1 --2- 172.21.15.32:0/2316703521 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8f38101870 0x7f8f38198010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.151+0000 7f8f3d09a640 1 --2- 172.21.15.32:0/2316703521 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8f38100680 0x7f8f38197ad0 unknown :-1 s=CLOSED pgs=463 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.151+0000 7f8f3d09a640 1 -- 172.21.15.32:0/2316703521 >> 172.21.15.32:0/2316703521 conn(0x7f8f380fbbf0 msgr2=0x7f8f380fd710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:18.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.151+0000 7f8f3d09a640 1 -- 172.21.15.32:0/2316703521 shutdown_connections 2024-04-25T00:33:18.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.152+0000 7f8f3d09a640 1 -- 172.21.15.32:0/2316703521 wait complete. 2024-04-25T00:33:18.167 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-25T00:33:18.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.277+0000 7f1a64502640 1 -- 172.21.15.32:0/2296946471 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1a5c102200 msgr2=0x7f1a5c1045f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:18.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.277+0000 7f1a64502640 1 --2- 172.21.15.32:0/2296946471 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1a5c102200 0x7f1a5c1045f0 secure :-1 s=READY pgs=472 cs=0 l=1 rev1=1 crypto rx=0x7f1a54009ca0 tx=0x7f1a5402a8c0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.277+0000 7f1a64502640 1 -- 172.21.15.32:0/2296946471 shutdown_connections 2024-04-25T00:33:18.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.277+0000 7f1a64502640 1 --2- 172.21.15.32:0/2296946471 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1a5c107460 0x7f1a5c1098b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.277+0000 7f1a64502640 1 --2- 172.21.15.32:0/2296946471 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1a5c104b30 0x7f1a5c106f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.277+0000 7f1a64502640 1 --2- 172.21.15.32:0/2296946471 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1a5c102200 0x7f1a5c1045f0 unknown :-1 s=CLOSED pgs=472 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.277+0000 7f1a64502640 1 -- 172.21.15.32:0/2296946471 >> 172.21.15.32:0/2296946471 conn(0x7f1a5c0fbbf0 msgr2=0x7f1a5c0fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:18.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.277+0000 7f1a64502640 1 -- 172.21.15.32:0/2296946471 shutdown_connections 2024-04-25T00:33:18.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.277+0000 7f1a64502640 1 -- 172.21.15.32:0/2296946471 wait complete. 2024-04-25T00:33:18.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.277+0000 7f1a64502640 1 Processor -- start 2024-04-25T00:33:18.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.277+0000 7f1a64502640 1 -- start start 2024-04-25T00:33:18.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.278+0000 7f1a64502640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1a5c102200 0x7f1a5c197b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:18.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.278+0000 7f1a64502640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1a5c104b30 0x7f1a5c198050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:18.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.278+0000 7f1a64502640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1a5c107460 0x7f1a5c19f0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:18.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.278+0000 7f1a64502640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f1a5c10d4a0 con 0x7f1a5c102200 2024-04-25T00:33:18.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.278+0000 7f1a64502640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f1a5c10d320 con 0x7f1a5c107460 2024-04-25T00:33:18.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.278+0000 7f1a64502640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f1a5c10d620 con 0x7f1a5c104b30 2024-04-25T00:33:18.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.278+0000 7f1a6229e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1a5c102200 0x7f1a5c197b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:18.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.278+0000 7f1a6229e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1a5c102200 0x7f1a5c197b10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44216/0 (socket says 172.21.15.32:44216) 2024-04-25T00:33:18.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.278+0000 7f1a6229e640 1 -- 172.21.15.32:0/233926691 learned_addr learned my addr 172.21.15.32:0/233926691 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:18.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.278+0000 7f1a62a9f640 1 --2- 172.21.15.32:0/233926691 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1a5c107460 0x7f1a5c19f0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:18.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.278+0000 7f1a61a9d640 1 --2- 172.21.15.32:0/233926691 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1a5c104b30 0x7f1a5c198050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:18.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.278+0000 7f1a6229e640 1 -- 172.21.15.32:0/233926691 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1a5c104b30 msgr2=0x7f1a5c198050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:18.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.278+0000 7f1a6229e640 1 --2- 172.21.15.32:0/233926691 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1a5c104b30 0x7f1a5c198050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.278+0000 7f1a6229e640 1 -- 172.21.15.32:0/233926691 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1a5c107460 msgr2=0x7f1a5c19f0c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:18.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.278+0000 7f1a6229e640 1 --2- 172.21.15.32:0/233926691 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1a5c107460 0x7f1a5c19f0c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.282 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.278+0000 7f1a6229e640 1 -- 172.21.15.32:0/233926691 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1a5c19f730 con 0x7f1a5c102200 2024-04-25T00:33:18.282 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.278+0000 7f1a61a9d640 1 --2- 172.21.15.32:0/233926691 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1a5c104b30 0x7f1a5c198050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:18.282 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.278+0000 7f1a6229e640 1 --2- 172.21.15.32:0/233926691 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1a5c102200 0x7f1a5c197b10 secure :-1 s=READY pgs=473 cs=0 l=1 rev1=1 crypto rx=0x7f1a54031a40 tx=0x7f1a54031ec0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:18.282 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.278+0000 7f1a5b7fe640 1 -- 172.21.15.32:0/233926691 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a5402faa0 con 0x7f1a5c102200 2024-04-25T00:33:18.282 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.278+0000 7f1a5b7fe640 1 -- 172.21.15.32:0/233926691 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1a5402fc40 con 0x7f1a5c102200 2024-04-25T00:33:18.282 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.279+0000 7f1a5b7fe640 1 -- 172.21.15.32:0/233926691 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a5403a790 con 0x7f1a5c102200 2024-04-25T00:33:18.282 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.279+0000 7f1a64502640 1 -- 172.21.15.32:0/233926691 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1a5c19f9c0 con 0x7f1a5c102200 2024-04-25T00:33:18.282 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.279+0000 7f1a64502640 1 -- 172.21.15.32:0/233926691 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1a5c19fed0 con 0x7f1a5c102200 2024-04-25T00:33:18.283 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.280+0000 7f1a64502640 1 -- 172.21.15.32:0/233926691 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1a5c0fd6e0 con 0x7f1a5c102200 2024-04-25T00:33:18.283 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.281+0000 7f1a5b7fe640 1 -- 172.21.15.32:0/233926691 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 21) ==== 137518+0+0 (secure 0 0 0) 0x7f1a54030b30 con 0x7f1a5c102200 2024-04-25T00:33:18.283 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.281+0000 7f1a5b7fe640 1 -- 172.21.15.32:0/233926691 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f1a540f06b0 con 0x7f1a5c102200 2024-04-25T00:33:18.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.288+0000 7f1a5b7fe640 1 -- 172.21.15.32:0/233926691 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f1a540bd540 con 0x7f1a5c102200 2024-04-25T00:33:18.484 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:18.178+0000 7fc4c3d49700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-25T00:33:18.484 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:18.178+0000 7fc4c3d49700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-25T00:33:18.485 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:18.179+0000 7fc4c3d49700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-25T00:33:18.485 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:18.179+0000 7fc4c3d49700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-25T00:33:18.485 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:18.179+0000 7fc4c3d49700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-25T00:33:18.485 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:18.179+0000 7fc4c1544700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-25T00:33:18.485 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:18.179+0000 7fc4c1544700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-25T00:33:18.485 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:18.179+0000 7fc4c1544700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-25T00:33:18.485 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:18.179+0000 7fc4c1544700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-25T00:33:18.485 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:33:18.179+0000 7fc4c1544700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-25T00:33:18.486 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: mgrmap e20: x(active, since 9m), standbys: z, y 2024-04-25T00:33:18.486 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3219007336' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:33:18.486 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: Active manager daemon x restarted 2024-04-25T00:33:18.486 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: Activating manager daemon x 2024-04-25T00:33:18.486 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: osdmap e94: 12 total, 12 up, 12 in 2024-04-25T00:33:18.486 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: mgrmap e21: x(active, starting, since 0.0120804s), standbys: z, y 2024-04-25T00:33:18.486 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-25T00:33:18.486 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:33:18.486 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:33:18.486 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2024-04-25T00:33:18.486 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2024-04-25T00:33:18.486 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2024-04-25T00:33:18.487 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-04-25T00:33:18.487 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2024-04-25T00:33:18.487 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2024-04-25T00:33:18.487 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-04-25T00:33:18.487 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-04-25T00:33:18.487 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-04-25T00:33:18.487 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-04-25T00:33:18.487 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-04-25T00:33:18.487 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-04-25T00:33:18.487 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-25T00:33:18.487 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2024-04-25T00:33:18.487 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-25T00:33:18.487 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T00:33:18.487 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T00:33:18.488 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T00:33:18.488 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T00:33:18.488 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T00:33:18.488 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T00:33:18.488 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T00:33:18.488 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T00:33:18.488 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-25T00:33:18.507 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.507+0000 7f1a64502640 1 -- 172.21.15.32:0/233926691 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f1a5c062340 con 0x7f1a5c102200 2024-04-25T00:33:18.509 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.509+0000 7f1a5b7fe640 1 -- 172.21.15.32:0/233926691 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v94) ==== 74+0+25219 (secure 0 0 0) 0x7f1a540bce40 con 0x7f1a5c102200 2024-04-25T00:33:18.509 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:33:18.510 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":94,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:33:17.919932+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-25T00:30:46.239556+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-25T00:30:47.257182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2024-04-25T00:30:51.193975+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.b","mappings":[{"from":8,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":1}]},{"pgid":"2.f","mappings":[{"from":8,"to":11}]},{"pgid":"2.10","mappings":[{"from":2,"to":4}]},{"pgid":"3.6","mappings":[{"from":3,"to":6}]},{"pgid":"3.17","mappings":[{"from":4,"to":7}]},{"pgid":"3.1b","mappings":[{"from":5,"to":9}]},{"pgid":"3.20","mappings":[{"from":5,"to":9}]},{"pgid":"3.23","mappings":[{"from":4,"to":8}]},{"pgid":"3.2c","mappings":[{"from":4,"to":9}]},{"pgid":"4.c","mappings":[{"from":9,"to":10}]},{"pgid":"4.12","mappings":[{"from":9,"to":10}]},{"pgid":"4.1c","mappings":[{"from":9,"to":8}]},{"pgid":"4.1d","mappings":[{"from":5,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:6801/585810931":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/1096527590":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/1767429535":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/3915353181":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6800/585810931":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/1694990206":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-num-failure-domains":"0","crush-osds-per-failure-domain":"0","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:33:18.511 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.511+0000 7f1a64502640 1 -- 172.21.15.32:0/233926691 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1a5c102200 msgr2=0x7f1a5c197b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:18.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.511+0000 7f1a64502640 1 --2- 172.21.15.32:0/233926691 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1a5c102200 0x7f1a5c197b10 secure :-1 s=READY pgs=473 cs=0 l=1 rev1=1 crypto rx=0x7f1a54031a40 tx=0x7f1a54031ec0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.511+0000 7f1a64502640 1 -- 172.21.15.32:0/233926691 shutdown_connections 2024-04-25T00:33:18.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.511+0000 7f1a64502640 1 --2- 172.21.15.32:0/233926691 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1a5c107460 0x7f1a5c19f0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.511+0000 7f1a64502640 1 --2- 172.21.15.32:0/233926691 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1a5c104b30 0x7f1a5c198050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.511+0000 7f1a64502640 1 --2- 172.21.15.32:0/233926691 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1a5c102200 0x7f1a5c197b10 unknown :-1 s=CLOSED pgs=473 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.511+0000 7f1a64502640 1 -- 172.21.15.32:0/233926691 >> 172.21.15.32:0/233926691 conn(0x7f1a5c0fbbf0 msgr2=0x7f1a5c102c00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:18.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.511+0000 7f1a64502640 1 -- 172.21.15.32:0/233926691 shutdown_connections 2024-04-25T00:33:18.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.511+0000 7f1a64502640 1 -- 172.21.15.32:0/233926691 wait complete. 2024-04-25T00:33:18.523 INFO:tasks.fwd_scrub.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-25T00:26:07.306839+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-04-25T00:30:46.239556+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '84', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.8799999952316284, 'score_stable': 1.8799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.8799999952316284, 'raw_score_stable': 1.8799999952316284, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-04-25T00:30:47.257182+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '85', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.05000000074505806}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_data_ec', 'create_time': '2024-04-25T00:30:51.193975+0000', 'flags': 5, 'flags_names': 'hashpspool,ec_overwrites', 'type': 3, 'size': 4, 'min_size': 3, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '89', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'cephfs_data_ec', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 8192, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}] 2024-04-25T00:33:18.524 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-25T00:33:18.635 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.633+0000 7f315574f640 1 -- 172.21.15.32:0/1141259770 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3150107460 msgr2=0x7f31501098b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:18.635 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.633+0000 7f315574f640 1 --2- 172.21.15.32:0/1141259770 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3150107460 0x7f31501098b0 secure :-1 s=READY pgs=474 cs=0 l=1 rev1=1 crypto rx=0x7f3148009ca0 tx=0x7f314802a8c0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.635 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.633+0000 7f315574f640 1 -- 172.21.15.32:0/1141259770 shutdown_connections 2024-04-25T00:33:18.636 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.633+0000 7f315574f640 1 --2- 172.21.15.32:0/1141259770 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3150107460 0x7f31501098b0 unknown :-1 s=CLOSED pgs=474 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.636 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.633+0000 7f315574f640 1 --2- 172.21.15.32:0/1141259770 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3150104b30 0x7f3150106f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.636 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.633+0000 7f315574f640 1 --2- 172.21.15.32:0/1141259770 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3150102200 0x7f31501045f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.636 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.633+0000 7f315574f640 1 -- 172.21.15.32:0/1141259770 >> 172.21.15.32:0/1141259770 conn(0x7f31500fbbf0 msgr2=0x7f31500fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:18.636 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.633+0000 7f315574f640 1 -- 172.21.15.32:0/1141259770 shutdown_connections 2024-04-25T00:33:18.636 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.633+0000 7f315574f640 1 -- 172.21.15.32:0/1141259770 wait complete. 2024-04-25T00:33:18.636 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.633+0000 7f315574f640 1 Processor -- start 2024-04-25T00:33:18.636 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.633+0000 7f315574f640 1 -- start start 2024-04-25T00:33:18.636 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.634+0000 7f315574f640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3150102200 0x7f3150197aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:18.636 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.634+0000 7f315574f640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3150104b30 0x7f3150197fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:18.636 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.634+0000 7f315574f640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3150107460 0x7f315019f050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:18.636 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.634+0000 7f315574f640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f315010d400 con 0x7f3150107460 2024-04-25T00:33:18.637 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.634+0000 7f315574f640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f315010d280 con 0x7f3150102200 2024-04-25T00:33:18.637 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.634+0000 7f315574f640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f315010d580 con 0x7f3150104b30 2024-04-25T00:33:18.637 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.634+0000 7f314f7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3150107460 0x7f315019f050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:18.637 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.634+0000 7f314f7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3150107460 0x7f315019f050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44226/0 (socket says 172.21.15.32:44226) 2024-04-25T00:33:18.637 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.634+0000 7f314f7fe640 1 -- 172.21.15.32:0/3523172435 learned_addr learned my addr 172.21.15.32:0/3523172435 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:18.637 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.634+0000 7f314effd640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3150102200 0x7f3150197aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:18.637 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.634+0000 7f314e7fc640 1 --2- 172.21.15.32:0/3523172435 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3150104b30 0x7f3150197fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:18.637 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.634+0000 7f314f7fe640 1 -- 172.21.15.32:0/3523172435 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3150104b30 msgr2=0x7f3150197fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:18.637 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.634+0000 7f314f7fe640 1 --2- 172.21.15.32:0/3523172435 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3150104b30 0x7f3150197fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.637 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.634+0000 7f314f7fe640 1 -- 172.21.15.32:0/3523172435 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3150102200 msgr2=0x7f3150197aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:18.637 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.634+0000 7f314f7fe640 1 --2- 172.21.15.32:0/3523172435 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3150102200 0x7f3150197aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.637 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.634+0000 7f314f7fe640 1 -- 172.21.15.32:0/3523172435 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f315019f6c0 con 0x7f3150107460 2024-04-25T00:33:18.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.634+0000 7f314effd640 1 --2- 172.21.15.32:0/3523172435 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3150102200 0x7f3150197aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:18.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.634+0000 7f314f7fe640 1 --2- 172.21.15.32:0/3523172435 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3150107460 0x7f315019f050 secure :-1 s=READY pgs=475 cs=0 l=1 rev1=1 crypto rx=0x7f3148005e30 tx=0x7f3148031ee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:18.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.635+0000 7f3137fff640 1 -- 172.21.15.32:0/3523172435 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3148003c30 con 0x7f3150107460 2024-04-25T00:33:18.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.635+0000 7f315574f640 1 -- 172.21.15.32:0/3523172435 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f315019f950 con 0x7f3150107460 2024-04-25T00:33:18.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.635+0000 7f3137fff640 1 -- 172.21.15.32:0/3523172435 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3148003dd0 con 0x7f3150107460 2024-04-25T00:33:18.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.635+0000 7f3137fff640 1 -- 172.21.15.32:0/3523172435 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f314802fde0 con 0x7f3150107460 2024-04-25T00:33:18.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.635+0000 7f315574f640 1 -- 172.21.15.32:0/3523172435 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f315019fea0 con 0x7f3150107460 2024-04-25T00:33:18.639 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.636+0000 7f315574f640 1 -- 172.21.15.32:0/3523172435 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f31500fd6e0 con 0x7f3150107460 2024-04-25T00:33:18.639 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.637+0000 7f3137fff640 1 -- 172.21.15.32:0/3523172435 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 21) ==== 137518+0+0 (secure 0 0 0) 0x7f3148004050 con 0x7f3150107460 2024-04-25T00:33:18.639 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.638+0000 7f3137fff640 1 -- 172.21.15.32:0/3523172435 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f31480f02a0 con 0x7f3150107460 2024-04-25T00:33:18.643 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.642+0000 7f3137fff640 1 -- 172.21.15.32:0/3523172435 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f31480bd130 con 0x7f3150107460 2024-04-25T00:33:18.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-25T00:33:18.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-25T00:33:18.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-25T00:33:18.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-25T00:33:18.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-25T00:33:18.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-25T00:33:18.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: Manager daemon x is now available 2024-04-25T00:33:18.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:33:18.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-25T00:33:18.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-25T00:33:18.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:18 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2316703521' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:33:18.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.830+0000 7f315574f640 1 -- 172.21.15.32:0/3523172435 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7f315002d350 con 0x7f3150107460 2024-04-25T00:33:18.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.830+0000 7f3137fff640 1 -- 172.21.15.32:0/3523172435 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v94) ==== 93+0+10 (secure 0 0 0) 0x7f31480bca30 con 0x7f3150107460 2024-04-25T00:33:18.831 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 1 2024-04-25T00:33:18.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.832+0000 7f315574f640 1 -- 172.21.15.32:0/3523172435 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3150107460 msgr2=0x7f315019f050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:18.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.832+0000 7f315574f640 1 --2- 172.21.15.32:0/3523172435 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3150107460 0x7f315019f050 secure :-1 s=READY pgs=475 cs=0 l=1 rev1=1 crypto rx=0x7f3148005e30 tx=0x7f3148031ee0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.832+0000 7f315574f640 1 -- 172.21.15.32:0/3523172435 shutdown_connections 2024-04-25T00:33:18.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.832+0000 7f315574f640 1 --2- 172.21.15.32:0/3523172435 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f3150107460 0x7f315019f050 unknown :-1 s=CLOSED pgs=475 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.832+0000 7f315574f640 1 --2- 172.21.15.32:0/3523172435 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3150104b30 0x7f3150197fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.832+0000 7f315574f640 1 --2- 172.21.15.32:0/3523172435 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3150102200 0x7f3150197aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.832+0000 7f315574f640 1 -- 172.21.15.32:0/3523172435 >> 172.21.15.32:0/3523172435 conn(0x7f31500fbbf0 msgr2=0x7f3150102aa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:18.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.832+0000 7f315574f640 1 -- 172.21.15.32:0/3523172435 shutdown_connections 2024-04-25T00:33:18.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.832+0000 7f315574f640 1 -- 172.21.15.32:0/3523172435 wait complete. 2024-04-25T00:33:18.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: mgrmap e20: x(active, since 9m), standbys: z, y 2024-04-25T00:33:18.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3219007336' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:33:18.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: Active manager daemon x restarted 2024-04-25T00:33:18.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: Activating manager daemon x 2024-04-25T00:33:18.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: osdmap e94: 12 total, 12 up, 12 in 2024-04-25T00:33:18.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: mgrmap e21: x(active, starting, since 0.0120804s), standbys: z, y 2024-04-25T00:33:18.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-25T00:33:18.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:33:18.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:33:18.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2024-04-25T00:33:18.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2024-04-25T00:33:18.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2024-04-25T00:33:18.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-04-25T00:33:18.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2024-04-25T00:33:18.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2024-04-25T00:33:18.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-04-25T00:33:18.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-04-25T00:33:18.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-04-25T00:33:18.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-04-25T00:33:18.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-04-25T00:33:18.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-04-25T00:33:18.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-25T00:33:18.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2024-04-25T00:33:18.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-25T00:33:18.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T00:33:18.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T00:33:18.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T00:33:18.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T00:33:18.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T00:33:18.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T00:33:18.840 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T00:33:18.840 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T00:33:18.840 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-25T00:33:18.840 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-25T00:33:18.840 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-25T00:33:18.840 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-25T00:33:18.840 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-25T00:33:18.840 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-25T00:33:18.840 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-25T00:33:18.840 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: Manager daemon x is now available 2024-04-25T00:33:18.840 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:33:18.840 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-25T00:33:18.840 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-25T00:33:18.840 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:18 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2316703521' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:33:18.844 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-04-25T00:33:18.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: mgrmap e20: x(active, since 9m), standbys: z, y 2024-04-25T00:33:18.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3219007336' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:33:18.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: Active manager daemon x restarted 2024-04-25T00:33:18.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: Activating manager daemon x 2024-04-25T00:33:18.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: osdmap e94: 12 total, 12 up, 12 in 2024-04-25T00:33:18.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: mgrmap e21: x(active, starting, since 0.0120804s), standbys: z, y 2024-04-25T00:33:18.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-25T00:33:18.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-25T00:33:18.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-25T00:33:18.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2024-04-25T00:33:18.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2024-04-25T00:33:18.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2024-04-25T00:33:18.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-04-25T00:33:18.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2024-04-25T00:33:18.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2024-04-25T00:33:18.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-04-25T00:33:18.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-04-25T00:33:18.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-04-25T00:33:18.895 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-04-25T00:33:18.895 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-04-25T00:33:18.895 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-04-25T00:33:18.895 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-25T00:33:18.895 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2024-04-25T00:33:18.895 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-25T00:33:18.895 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T00:33:18.895 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T00:33:18.896 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T00:33:18.896 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T00:33:18.896 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T00:33:18.896 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T00:33:18.896 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T00:33:18.896 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T00:33:18.896 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-25T00:33:18.897 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-25T00:33:18.897 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-25T00:33:18.897 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-25T00:33:18.897 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-25T00:33:18.897 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-25T00:33:18.897 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-25T00:33:18.898 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: Manager daemon x is now available 2024-04-25T00:33:18.898 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:33:18.898 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-25T00:33:18.899 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-25T00:33:18.899 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:18 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2316703521' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:33:18.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.967+0000 7f912fe7d640 1 -- 172.21.15.32:0/3204960788 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9128102990 msgr2=0x7f9128102e10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:18.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.967+0000 7f912fe7d640 1 --2- 172.21.15.32:0/3204960788 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9128102990 0x7f9128102e10 secure :-1 s=READY pgs=476 cs=0 l=1 rev1=1 crypto rx=0x7f9118009900 tx=0x7f911802a410 comp rx=0 tx=0).stop 2024-04-25T00:33:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.967+0000 7f912fe7d640 1 -- 172.21.15.32:0/3204960788 shutdown_connections 2024-04-25T00:33:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.967+0000 7f912fe7d640 1 --2- 172.21.15.32:0/3204960788 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9128103350 0x7f9128109be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.967+0000 7f912fe7d640 1 --2- 172.21.15.32:0/3204960788 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9128102990 0x7f9128102e10 unknown :-1 s=CLOSED pgs=476 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.967+0000 7f912fe7d640 1 --2- 172.21.15.32:0/3204960788 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f91281017a0 0x7f9128101bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.967+0000 7f912fe7d640 1 -- 172.21.15.32:0/3204960788 >> 172.21.15.32:0/3204960788 conn(0x7f91280fcd10 msgr2=0x7f91280ff150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.967+0000 7f912fe7d640 1 -- 172.21.15.32:0/3204960788 shutdown_connections 2024-04-25T00:33:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.967+0000 7f912fe7d640 1 -- 172.21.15.32:0/3204960788 wait complete. 2024-04-25T00:33:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.967+0000 7f912fe7d640 1 Processor -- start 2024-04-25T00:33:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.967+0000 7f912fe7d640 1 -- start start 2024-04-25T00:33:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.968+0000 7f912fe7d640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f91281017a0 0x7f9128197a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.968+0000 7f912fe7d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9128102990 0x7f9128197fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:18.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.968+0000 7f912fe7d640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9128103350 0x7f912819f030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:18.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.968+0000 7f912fe7d640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f912810d400 con 0x7f9128102990 2024-04-25T00:33:18.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.968+0000 7f912fe7d640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f912810d280 con 0x7f9128103350 2024-04-25T00:33:18.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.968+0000 7f912fe7d640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f912810d580 con 0x7f91281017a0 2024-04-25T00:33:18.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.968+0000 7f912d418640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9128102990 0x7f9128197fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:18.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.968+0000 7f912dc19640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f91281017a0 0x7f9128197a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:18.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.968+0000 7f912dc19640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f91281017a0 0x7f9128197a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:37086/0 (socket says 172.21.15.32:37086) 2024-04-25T00:33:18.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.968+0000 7f912dc19640 1 -- 172.21.15.32:0/608026417 learned_addr learned my addr 172.21.15.32:0/608026417 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:18.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.968+0000 7f912e41a640 1 --2- 172.21.15.32:0/608026417 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9128103350 0x7f912819f030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:18.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.968+0000 7f912dc19640 1 -- 172.21.15.32:0/608026417 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9128103350 msgr2=0x7f912819f030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:18.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.968+0000 7f912dc19640 1 --2- 172.21.15.32:0/608026417 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9128103350 0x7f912819f030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.968+0000 7f912dc19640 1 -- 172.21.15.32:0/608026417 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9128102990 msgr2=0x7f9128197fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:18.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.968+0000 7f912dc19640 1 --2- 172.21.15.32:0/608026417 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9128102990 0x7f9128197fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:18.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.968+0000 7f912dc19640 1 -- 172.21.15.32:0/608026417 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f912819f6a0 con 0x7f91281017a0 2024-04-25T00:33:18.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.968+0000 7f912dc19640 1 --2- 172.21.15.32:0/608026417 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f91281017a0 0x7f9128197a80 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f912400e9c0 tx=0x7f912400ee80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:18.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.969+0000 7f911effd640 1 -- 172.21.15.32:0/608026417 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f912400cd80 con 0x7f91281017a0 2024-04-25T00:33:18.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.969+0000 7f912fe7d640 1 -- 172.21.15.32:0/608026417 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f912819f990 con 0x7f91281017a0 2024-04-25T00:33:18.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.969+0000 7f911effd640 1 -- 172.21.15.32:0/608026417 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f91240035f0 con 0x7f91281017a0 2024-04-25T00:33:18.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.969+0000 7f911effd640 1 -- 172.21.15.32:0/608026417 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9124010690 con 0x7f91281017a0 2024-04-25T00:33:18.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.969+0000 7f912fe7d640 1 -- 172.21.15.32:0/608026417 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f912819ff00 con 0x7f91281017a0 2024-04-25T00:33:18.973 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.970+0000 7f912fe7d640 1 -- 172.21.15.32:0/608026417 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f91280689b0 con 0x7f91281017a0 2024-04-25T00:33:18.973 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.972+0000 7f911effd640 1 -- 172.21.15.32:0/608026417 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 22) ==== 137645+0+0 (secure 0 0 0) 0x7f91240037a0 con 0x7f91281017a0 2024-04-25T00:33:18.973 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.973+0000 7f911effd640 1 --2- 172.21.15.32:0/608026417 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f91140a2a10 0x7f91140a4ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:18.973 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.973+0000 7f912d418640 1 --2- 172.21.15.32:0/608026417 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f91140a2a10 0x7f91140a4ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:18.973 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.973+0000 7f911effd640 1 -- 172.21.15.32:0/608026417 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f91240cf5c0 con 0x7f91281017a0 2024-04-25T00:33:18.974 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.973+0000 7f912d418640 1 --2- 172.21.15.32:0/608026417 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f91140a2a10 0x7f91140a4ed0 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7f91180314e0 tx=0x7f9118000bc0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:18.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:18.979+0000 7f911effd640 1 -- 172.21.15.32:0/608026417 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f912409c390 con 0x7f91281017a0 2024-04-25T00:33:19.169 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.169+0000 7f912fe7d640 1 -- 172.21.15.32:0/608026417 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) -- 0x7f9128108b90 con 0x7f91281017a0 2024-04-25T00:33:19.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.170+0000 7f911effd640 1 -- 172.21.15.32:0/608026417 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v94) ==== 104+0+11 (secure 0 0 0) 0x7f912409bc90 con 0x7f91281017a0 2024-04-25T00:33:19.170 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 32 2024-04-25T00:33:19.172 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.172+0000 7f912fe7d640 1 -- 172.21.15.32:0/608026417 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f91140a2a10 msgr2=0x7f91140a4ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:19.172 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.172+0000 7f912fe7d640 1 --2- 172.21.15.32:0/608026417 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f91140a2a10 0x7f91140a4ed0 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7f91180314e0 tx=0x7f9118000bc0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.172 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.172+0000 7f912fe7d640 1 -- 172.21.15.32:0/608026417 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f91281017a0 msgr2=0x7f9128197a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:19.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.172+0000 7f912fe7d640 1 --2- 172.21.15.32:0/608026417 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f91281017a0 0x7f9128197a80 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f912400e9c0 tx=0x7f912400ee80 comp rx=0 tx=0).stop 2024-04-25T00:33:19.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.172+0000 7f912fe7d640 1 -- 172.21.15.32:0/608026417 shutdown_connections 2024-04-25T00:33:19.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.172+0000 7f912fe7d640 1 --2- 172.21.15.32:0/608026417 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f91140a2a10 0x7f91140a4ed0 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.172+0000 7f912fe7d640 1 --2- 172.21.15.32:0/608026417 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9128103350 0x7f912819f030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.172+0000 7f912fe7d640 1 --2- 172.21.15.32:0/608026417 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9128102990 0x7f9128197fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.172+0000 7f912fe7d640 1 --2- 172.21.15.32:0/608026417 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f91281017a0 0x7f9128197a80 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.172+0000 7f912fe7d640 1 -- 172.21.15.32:0/608026417 >> 172.21.15.32:0/608026417 conn(0x7f91280fcd10 msgr2=0x7f9128105d60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:19.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.172+0000 7f912fe7d640 1 -- 172.21.15.32:0/608026417 shutdown_connections 2024-04-25T00:33:19.173 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.172+0000 7f912fe7d640 1 -- 172.21.15.32:0/608026417 wait complete. 2024-04-25T00:33:19.183 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-04-25T00:33:19.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.289+0000 7f94ad4d4640 1 -- 172.21.15.32:0/1813645014 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f94a8104c10 msgr2=0x7f94a8107000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:19.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.289+0000 7f94ad4d4640 1 --2- 172.21.15.32:0/1813645014 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f94a8104c10 0x7f94a8107000 secure :-1 s=READY pgs=477 cs=0 l=1 rev1=1 crypto rx=0x7f9490009900 tx=0x7f949002a410 comp rx=0 tx=0).stop 2024-04-25T00:33:19.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.289+0000 7f94ad4d4640 1 -- 172.21.15.32:0/1813645014 shutdown_connections 2024-04-25T00:33:19.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.289+0000 7f94ad4d4640 1 --2- 172.21.15.32:0/1813645014 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f94a8107540 0x7f94a8109930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.289+0000 7f94ad4d4640 1 --2- 172.21.15.32:0/1813645014 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f94a8104c10 0x7f94a8107000 unknown :-1 s=CLOSED pgs=477 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.289+0000 7f94ad4d4640 1 --2- 172.21.15.32:0/1813645014 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f94a81022e0 0x7f94a81046d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.289+0000 7f94ad4d4640 1 -- 172.21.15.32:0/1813645014 >> 172.21.15.32:0/1813645014 conn(0x7f94a80fbc30 msgr2=0x7f94a80fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:19.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.289+0000 7f94ad4d4640 1 -- 172.21.15.32:0/1813645014 shutdown_connections 2024-04-25T00:33:19.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.289+0000 7f94ad4d4640 1 -- 172.21.15.32:0/1813645014 wait complete. 2024-04-25T00:33:19.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.290+0000 7f94ad4d4640 1 Processor -- start 2024-04-25T00:33:19.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.290+0000 7f94ad4d4640 1 -- start start 2024-04-25T00:33:19.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.290+0000 7f94ad4d4640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f94a81022e0 0x7f94a8197b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:19.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.290+0000 7f94ad4d4640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f94a8104c10 0x7f94a8198090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:19.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.290+0000 7f94ad4d4640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f94a8107540 0x7f94a819f100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:19.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.290+0000 7f94ad4d4640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f94a810d420 con 0x7f94a81022e0 2024-04-25T00:33:19.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.290+0000 7f94ad4d4640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f94a810d2a0 con 0x7f94a8104c10 2024-04-25T00:33:19.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.290+0000 7f94ad4d4640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f94a810d5a0 con 0x7f94a8107540 2024-04-25T00:33:19.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.290+0000 7f94a6ffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f94a81022e0 0x7f94a8197b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:19.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.290+0000 7f94a6ffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f94a81022e0 0x7f94a8197b50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44274/0 (socket says 172.21.15.32:44274) 2024-04-25T00:33:19.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.290+0000 7f94a6ffd640 1 -- 172.21.15.32:0/1898524971 learned_addr learned my addr 172.21.15.32:0/1898524971 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:19.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.290+0000 7f949ffff640 1 --2- 172.21.15.32:0/1898524971 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f94a8104c10 0x7f94a8198090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:19.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.290+0000 7f94a77fe640 1 --2- 172.21.15.32:0/1898524971 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f94a8107540 0x7f94a819f100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:19.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.290+0000 7f94a6ffd640 1 -- 172.21.15.32:0/1898524971 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f94a8107540 msgr2=0x7f94a819f100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:19.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.290+0000 7f94a6ffd640 1 --2- 172.21.15.32:0/1898524971 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f94a8107540 0x7f94a819f100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.290+0000 7f94a6ffd640 1 -- 172.21.15.32:0/1898524971 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f94a8104c10 msgr2=0x7f94a8198090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:19.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.290+0000 7f94a6ffd640 1 --2- 172.21.15.32:0/1898524971 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f94a8104c10 0x7f94a8198090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.290+0000 7f94a6ffd640 1 -- 172.21.15.32:0/1898524971 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f94a819f770 con 0x7f94a81022e0 2024-04-25T00:33:19.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.291+0000 7f94a6ffd640 1 --2- 172.21.15.32:0/1898524971 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f94a81022e0 0x7f94a8197b50 secure :-1 s=READY pgs=478 cs=0 l=1 rev1=1 crypto rx=0x7f949800e9f0 tx=0x7f949800edf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:19.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.291+0000 7f94a4ff9640 1 -- 172.21.15.32:0/1898524971 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f949800cd80 con 0x7f94a81022e0 2024-04-25T00:33:19.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.291+0000 7f94ad4d4640 1 -- 172.21.15.32:0/1898524971 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f94a819fa60 con 0x7f94a81022e0 2024-04-25T00:33:19.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.291+0000 7f94a4ff9640 1 -- 172.21.15.32:0/1898524971 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9498005500 con 0x7f94a81022e0 2024-04-25T00:33:19.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.291+0000 7f94a4ff9640 1 -- 172.21.15.32:0/1898524971 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9498010690 con 0x7f94a81022e0 2024-04-25T00:33:19.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.291+0000 7f94ad4d4640 1 -- 172.21.15.32:0/1898524971 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f94a819ff30 con 0x7f94a81022e0 2024-04-25T00:33:19.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.292+0000 7f94ad4d4640 1 -- 172.21.15.32:0/1898524971 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f94a80fd720 con 0x7f94a81022e0 2024-04-25T00:33:19.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.293+0000 7f94a4ff9640 1 -- 172.21.15.32:0/1898524971 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 22) ==== 137645+0+0 (secure 0 0 0) 0x7f94980046c0 con 0x7f94a81022e0 2024-04-25T00:33:19.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.293+0000 7f94a4ff9640 1 --2- 172.21.15.32:0/1898524971 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f94940a2990 0x7f94940a4e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:19.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.294+0000 7f949ffff640 1 --2- 172.21.15.32:0/1898524971 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f94940a2990 0x7f94940a4e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:19.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.294+0000 7f94a4ff9640 1 -- 172.21.15.32:0/1898524971 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f94980cf530 con 0x7f94a81022e0 2024-04-25T00:33:19.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.294+0000 7f949ffff640 1 --2- 172.21.15.32:0/1898524971 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f94940a2990 0x7f94940a4e50 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7f9490005ab0 tx=0x7f9490000bc0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:19.300 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.300+0000 7f94a4ff9640 1 -- 172.21.15.32:0/1898524971 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f949809c270 con 0x7f94a81022e0 2024-04-25T00:33:19.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.493+0000 7f94ad4d4640 1 -- 172.21.15.32:0/1898524971 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) -- 0x7f94a802d170 con 0x7f94a81022e0 2024-04-25T00:33:19.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.493+0000 7f94a4ff9640 1 -- 172.21.15.32:0/1898524971 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v94) ==== 100+0+11 (secure 0 0 0) 0x7f949809bb70 con 0x7f94a81022e0 2024-04-25T00:33:19.494 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 64 2024-04-25T00:33:19.496 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.495+0000 7f94ad4d4640 1 -- 172.21.15.32:0/1898524971 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f94940a2990 msgr2=0x7f94940a4e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:19.496 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.495+0000 7f94ad4d4640 1 --2- 172.21.15.32:0/1898524971 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f94940a2990 0x7f94940a4e50 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7f9490005ab0 tx=0x7f9490000bc0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.496 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.495+0000 7f94ad4d4640 1 -- 172.21.15.32:0/1898524971 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f94a81022e0 msgr2=0x7f94a8197b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:19.496 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.495+0000 7f94ad4d4640 1 --2- 172.21.15.32:0/1898524971 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f94a81022e0 0x7f94a8197b50 secure :-1 s=READY pgs=478 cs=0 l=1 rev1=1 crypto rx=0x7f949800e9f0 tx=0x7f949800edf0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.496 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.495+0000 7f94ad4d4640 1 -- 172.21.15.32:0/1898524971 shutdown_connections 2024-04-25T00:33:19.496 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.495+0000 7f94ad4d4640 1 --2- 172.21.15.32:0/1898524971 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f94940a2990 0x7f94940a4e50 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.496 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.495+0000 7f94ad4d4640 1 --2- 172.21.15.32:0/1898524971 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f94a8107540 0x7f94a819f100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.496 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.495+0000 7f94ad4d4640 1 --2- 172.21.15.32:0/1898524971 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f94a8104c10 0x7f94a8198090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.496 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.495+0000 7f94ad4d4640 1 --2- 172.21.15.32:0/1898524971 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f94a81022e0 0x7f94a8197b50 unknown :-1 s=CLOSED pgs=478 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.496 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.495+0000 7f94ad4d4640 1 -- 172.21.15.32:0/1898524971 >> 172.21.15.32:0/1898524971 conn(0x7f94a80fbc30 msgr2=0x7f94a8102b80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:19.497 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.495+0000 7f94ad4d4640 1 -- 172.21.15.32:0/1898524971 shutdown_connections 2024-04-25T00:33:19.497 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.495+0000 7f94ad4d4640 1 -- 172.21.15.32:0/1898524971 wait complete. 2024-04-25T00:33:19.507 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data_ec pg_num 2024-04-25T00:33:19.617 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.614+0000 7f70de6b3640 1 -- 172.21.15.32:0/2392012508 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f70d8102990 msgr2=0x7f70d8102e10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:19.617 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.614+0000 7f70de6b3640 1 --2- 172.21.15.32:0/2392012508 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f70d8102990 0x7f70d8102e10 secure :-1 s=READY pgs=479 cs=0 l=1 rev1=1 crypto rx=0x7f70cc009900 tx=0x7f70cc02a410 comp rx=0 tx=0).stop 2024-04-25T00:33:19.617 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.614+0000 7f70de6b3640 1 -- 172.21.15.32:0/2392012508 shutdown_connections 2024-04-25T00:33:19.617 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.614+0000 7f70de6b3640 1 --2- 172.21.15.32:0/2392012508 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f70d8103350 0x7f70d8109be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.617 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.614+0000 7f70de6b3640 1 --2- 172.21.15.32:0/2392012508 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f70d8102990 0x7f70d8102e10 unknown :-1 s=CLOSED pgs=479 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.617 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.614+0000 7f70de6b3640 1 --2- 172.21.15.32:0/2392012508 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f70d81017a0 0x7f70d8101bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.617 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.614+0000 7f70de6b3640 1 -- 172.21.15.32:0/2392012508 >> 172.21.15.32:0/2392012508 conn(0x7f70d80fcd10 msgr2=0x7f70d80ff150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:19.617 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.614+0000 7f70de6b3640 1 -- 172.21.15.32:0/2392012508 shutdown_connections 2024-04-25T00:33:19.617 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.614+0000 7f70de6b3640 1 -- 172.21.15.32:0/2392012508 wait complete. 2024-04-25T00:33:19.617 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.615+0000 7f70de6b3640 1 Processor -- start 2024-04-25T00:33:19.618 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.615+0000 7f70de6b3640 1 -- start start 2024-04-25T00:33:19.618 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.615+0000 7f70de6b3640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f70d81017a0 0x7f70d8199d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:19.618 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.615+0000 7f70de6b3640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f70d8102990 0x7f70d819a290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:19.618 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.615+0000 7f70de6b3640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f70d8103350 0x7f70d81a1270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:19.618 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.615+0000 7f70de6b3640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f70d810d420 con 0x7f70d81017a0 2024-04-25T00:33:19.618 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.615+0000 7f70de6b3640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f70d810d2a0 con 0x7f70d8103350 2024-04-25T00:33:19.618 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.615+0000 7f70de6b3640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f70d810d5a0 con 0x7f70d8102990 2024-04-25T00:33:19.618 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.615+0000 7f70d7fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f70d81017a0 0x7f70d8199d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:19.618 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.615+0000 7f70d7fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f70d81017a0 0x7f70d8199d50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44296/0 (socket says 172.21.15.32:44296) 2024-04-25T00:33:19.618 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.615+0000 7f70d7fff640 1 -- 172.21.15.32:0/2675413030 learned_addr learned my addr 172.21.15.32:0/2675413030 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:19.618 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.615+0000 7f70dcc50640 1 --2- 172.21.15.32:0/2675413030 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f70d8103350 0x7f70d81a1270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:19.618 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.615+0000 7f70d77fe640 1 --2- 172.21.15.32:0/2675413030 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f70d8102990 0x7f70d819a290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:19.618 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.615+0000 7f70d7fff640 1 -- 172.21.15.32:0/2675413030 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f70d8102990 msgr2=0x7f70d819a290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:19.618 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.615+0000 7f70d7fff640 1 --2- 172.21.15.32:0/2675413030 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f70d8102990 0x7f70d819a290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.615+0000 7f70d7fff640 1 -- 172.21.15.32:0/2675413030 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f70d8103350 msgr2=0x7f70d81a1270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:19.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.615+0000 7f70d7fff640 1 --2- 172.21.15.32:0/2675413030 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f70d8103350 0x7f70d81a1270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.615+0000 7f70d7fff640 1 -- 172.21.15.32:0/2675413030 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f70d81a1910 con 0x7f70d81017a0 2024-04-25T00:33:19.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.616+0000 7f70d7fff640 1 --2- 172.21.15.32:0/2675413030 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f70d81017a0 0x7f70d8199d50 secure :-1 s=READY pgs=480 cs=0 l=1 rev1=1 crypto rx=0x7f70c800d950 tx=0x7f70c800de10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:19.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.616+0000 7f70d57fa640 1 -- 172.21.15.32:0/2675413030 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f70c8017070 con 0x7f70d81017a0 2024-04-25T00:33:19.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.616+0000 7f70d57fa640 1 -- 172.21.15.32:0/2675413030 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f70c8009c30 con 0x7f70d81017a0 2024-04-25T00:33:19.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.616+0000 7f70de6b3640 1 -- 172.21.15.32:0/2675413030 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f70d81a1c00 con 0x7f70d81017a0 2024-04-25T00:33:19.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.616+0000 7f70d57fa640 1 -- 172.21.15.32:0/2675413030 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f70c800f600 con 0x7f70d81017a0 2024-04-25T00:33:19.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.616+0000 7f70de6b3640 1 -- 172.21.15.32:0/2675413030 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f70d81a2100 con 0x7f70d81017a0 2024-04-25T00:33:19.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.617+0000 7f70de6b3640 1 -- 172.21.15.32:0/2675413030 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f70d81a22d0 con 0x7f70d81017a0 2024-04-25T00:33:19.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.618+0000 7f70d57fa640 1 -- 172.21.15.32:0/2675413030 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 22) ==== 137645+0+0 (secure 0 0 0) 0x7f70c801a070 con 0x7f70d81017a0 2024-04-25T00:33:19.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.618+0000 7f70d57fa640 1 --2- 172.21.15.32:0/2675413030 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f70c00a2a10 0x7f70c00a4ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:19.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.619+0000 7f70d77fe640 1 --2- 172.21.15.32:0/2675413030 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f70c00a2a10 0x7f70c00a4ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:19.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.619+0000 7f70d57fa640 1 -- 172.21.15.32:0/2675413030 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f70c80cec80 con 0x7f70d81017a0 2024-04-25T00:33:19.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.619+0000 7f70d77fe640 1 --2- 172.21.15.32:0/2675413030 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f70c00a2a10 0x7f70c00a4ed0 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f70cc000c00 tx=0x7f70cc0315d0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:19.623 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.623+0000 7f70d57fa640 1 -- 172.21.15.32:0/2675413030 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f70c8009de0 con 0x7f70d81017a0 2024-04-25T00:33:19.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:19 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/233926691' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:33:19.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:19 smithi130 ceph-mon[32525]: [25/Apr/2024:00:33:18] ENGINE Bus STARTING 2024-04-25T00:33:19.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:19 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3523172435' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:33:19.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:19 smithi130 ceph-mon[32525]: mgrmap e22: x(active, since 1.0171s), standbys: z, y 2024-04-25T00:33:19.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:19 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/608026417' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:33:19.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:19 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/233926691' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:33:19.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:19 smithi032 ceph-mon[29357]: [25/Apr/2024:00:33:18] ENGINE Bus STARTING 2024-04-25T00:33:19.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:19 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3523172435' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:33:19.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:19 smithi032 ceph-mon[29357]: mgrmap e22: x(active, since 1.0171s), standbys: z, y 2024-04-25T00:33:19.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:19 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/608026417' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:33:19.812 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.811+0000 7f70de6b3640 1 -- 172.21.15.32:0/2675413030 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"} v 0) -- 0x7f70d81a24a0 con 0x7f70d81017a0 2024-04-25T00:33:19.812 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.812+0000 7f70d57fa640 1 -- 172.21.15.32:0/2675413030 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]=0 v94) ==== 103+0+11 (secure 0 0 0) 0x7f70c809b9c0 con 0x7f70d81017a0 2024-04-25T00:33:19.812 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 32 2024-04-25T00:33:19.815 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.814+0000 7f70de6b3640 1 -- 172.21.15.32:0/2675413030 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f70c00a2a10 msgr2=0x7f70c00a4ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:19.815 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.814+0000 7f70de6b3640 1 --2- 172.21.15.32:0/2675413030 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f70c00a2a10 0x7f70c00a4ed0 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f70cc000c00 tx=0x7f70cc0315d0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.815 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.814+0000 7f70de6b3640 1 -- 172.21.15.32:0/2675413030 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f70d81017a0 msgr2=0x7f70d8199d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:19.815 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.814+0000 7f70de6b3640 1 --2- 172.21.15.32:0/2675413030 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f70d81017a0 0x7f70d8199d50 secure :-1 s=READY pgs=480 cs=0 l=1 rev1=1 crypto rx=0x7f70c800d950 tx=0x7f70c800de10 comp rx=0 tx=0).stop 2024-04-25T00:33:19.815 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.814+0000 7f70de6b3640 1 -- 172.21.15.32:0/2675413030 shutdown_connections 2024-04-25T00:33:19.815 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.814+0000 7f70de6b3640 1 --2- 172.21.15.32:0/2675413030 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f70c00a2a10 0x7f70c00a4ed0 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.815 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.814+0000 7f70de6b3640 1 --2- 172.21.15.32:0/2675413030 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f70d8103350 0x7f70d81a1270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.815 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.814+0000 7f70de6b3640 1 --2- 172.21.15.32:0/2675413030 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f70d8102990 0x7f70d819a290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.815 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.814+0000 7f70de6b3640 1 --2- 172.21.15.32:0/2675413030 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f70d81017a0 0x7f70d8199d50 unknown :-1 s=CLOSED pgs=480 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.815 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.814+0000 7f70de6b3640 1 -- 172.21.15.32:0/2675413030 >> 172.21.15.32:0/2675413030 conn(0x7f70d80fcd10 msgr2=0x7f70d8073970 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:19.815 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.814+0000 7f70de6b3640 1 -- 172.21.15.32:0/2675413030 shutdown_connections 2024-04-25T00:33:19.816 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.814+0000 7f70de6b3640 1 -- 172.21.15.32:0/2675413030 wait complete. 2024-04-25T00:33:19.827 INFO:tasks.fwd_scrub:Wait for all MDSs to reach steady state... 2024-04-25T00:33:19.827 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:33:19.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:19 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/233926691' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:33:19.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:19 smithi121 ceph-mon[32231]: [25/Apr/2024:00:33:18] ENGINE Bus STARTING 2024-04-25T00:33:19.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:19 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3523172435' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:33:19.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:19 smithi121 ceph-mon[32231]: mgrmap e22: x(active, since 1.0171s), standbys: z, y 2024-04-25T00:33:19.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:19 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/608026417' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:33:19.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.935+0000 7fbc0db63640 1 -- 172.21.15.32:0/3389623305 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbc08101870 msgr2=0x7fbc08101cf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:19.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.935+0000 7fbc0db63640 1 --2- 172.21.15.32:0/3389623305 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbc08101870 0x7fbc08101cf0 secure :-1 s=READY pgs=481 cs=0 l=1 rev1=1 crypto rx=0x7fbbfc009900 tx=0x7fbbfc02a410 comp rx=0 tx=0).stop 2024-04-25T00:33:19.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.935+0000 7fbc0db63640 1 -- 172.21.15.32:0/3389623305 shutdown_connections 2024-04-25T00:33:19.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.935+0000 7fbc0db63640 1 --2- 172.21.15.32:0/3389623305 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbc08102230 0x7fbc08108ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.935+0000 7fbc0db63640 1 --2- 172.21.15.32:0/3389623305 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbc08101870 0x7fbc08101cf0 unknown :-1 s=CLOSED pgs=481 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.935+0000 7fbc0db63640 1 --2- 172.21.15.32:0/3389623305 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbc08100680 0x7fbc08100aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.935+0000 7fbc0db63640 1 -- 172.21.15.32:0/3389623305 >> 172.21.15.32:0/3389623305 conn(0x7fbc080fbbf0 msgr2=0x7fbc080fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:19.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.935+0000 7fbc0db63640 1 -- 172.21.15.32:0/3389623305 shutdown_connections 2024-04-25T00:33:19.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.935+0000 7fbc0db63640 1 -- 172.21.15.32:0/3389623305 wait complete. 2024-04-25T00:33:19.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.935+0000 7fbc0db63640 1 Processor -- start 2024-04-25T00:33:19.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.936+0000 7fbc0db63640 1 -- start start 2024-04-25T00:33:19.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.936+0000 7fbc0db63640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbc08100680 0x7fbc08199d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:19.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.936+0000 7fbc0db63640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbc08101870 0x7fbc0819a290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:19.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.936+0000 7fbc0db63640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbc08102230 0x7fbc081a12c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:19.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.936+0000 7fbc0db63640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fbc0810c2e0 con 0x7fbc08100680 2024-04-25T00:33:19.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.936+0000 7fbc0db63640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fbc0810c160 con 0x7fbc08101870 2024-04-25T00:33:19.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.936+0000 7fbc0db63640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fbc0810c460 con 0x7fbc08102230 2024-04-25T00:33:19.940 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.936+0000 7fbc077fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbc08100680 0x7fbc08199d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:19.940 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.936+0000 7fbc06ffd640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbc08101870 0x7fbc0819a290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:19.940 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.936+0000 7fbc077fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbc08100680 0x7fbc08199d50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44322/0 (socket says 172.21.15.32:44322) 2024-04-25T00:33:19.940 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.936+0000 7fbc077fe640 1 -- 172.21.15.32:0/4162392750 learned_addr learned my addr 172.21.15.32:0/4162392750 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:19.940 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.936+0000 7fbc07fff640 1 --2- 172.21.15.32:0/4162392750 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbc08102230 0x7fbc081a12c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:19.940 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.936+0000 7fbc06ffd640 1 -- 172.21.15.32:0/4162392750 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbc08102230 msgr2=0x7fbc081a12c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:19.940 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.936+0000 7fbc06ffd640 1 --2- 172.21.15.32:0/4162392750 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbc08102230 0x7fbc081a12c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.940 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.936+0000 7fbc06ffd640 1 -- 172.21.15.32:0/4162392750 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbc08100680 msgr2=0x7fbc08199d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:19.940 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.936+0000 7fbc06ffd640 1 --2- 172.21.15.32:0/4162392750 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbc08100680 0x7fbc08199d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:19.940 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.936+0000 7fbc06ffd640 1 -- 172.21.15.32:0/4162392750 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbc081a1960 con 0x7fbc08101870 2024-04-25T00:33:19.940 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.937+0000 7fbc06ffd640 1 --2- 172.21.15.32:0/4162392750 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbc08101870 0x7fbc0819a290 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7fbbfc005ab0 tx=0x7fbbfc004270 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:19.940 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.937+0000 7fbc04ff9640 1 -- 172.21.15.32:0/4162392750 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbbfc03f070 con 0x7fbc08101870 2024-04-25T00:33:19.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.937+0000 7fbc0db63640 1 -- 172.21.15.32:0/4162392750 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbc081a1bf0 con 0x7fbc08101870 2024-04-25T00:33:19.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.937+0000 7fbc04ff9640 1 -- 172.21.15.32:0/4162392750 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbbfc003cd0 con 0x7fbc08101870 2024-04-25T00:33:19.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.937+0000 7fbc04ff9640 1 -- 172.21.15.32:0/4162392750 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbbfc035070 con 0x7fbc08101870 2024-04-25T00:33:19.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.937+0000 7fbc0db63640 1 -- 172.21.15.32:0/4162392750 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fbc081a20e0 con 0x7fbc08101870 2024-04-25T00:33:19.942 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.938+0000 7fbc0db63640 1 -- 172.21.15.32:0/4162392750 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fbc081a22b0 con 0x7fbc08101870 2024-04-25T00:33:19.942 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.939+0000 7fbc04ff9640 1 -- 172.21.15.32:0/4162392750 <== mon.2 v2:172.21.15.121:3300/0 4 ==== mgrmap(e 22) ==== 137645+0+0 (secure 0 0 0) 0x7fbbfc03b460 con 0x7fbc08101870 2024-04-25T00:33:19.942 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.940+0000 7fbc04ff9640 1 --2- 172.21.15.32:0/4162392750 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fbbf00a2990 0x7fbbf00a4e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:19.942 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.940+0000 7fbc077fe640 1 --2- 172.21.15.32:0/4162392750 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fbbf00a2990 0x7fbbf00a4e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:19.942 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.940+0000 7fbc04ff9640 1 -- 172.21.15.32:0/4162392750 <== mon.2 v2:172.21.15.121:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fbbfc0ef9c0 con 0x7fbc08101870 2024-04-25T00:33:19.942 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.941+0000 7fbc077fe640 1 --2- 172.21.15.32:0/4162392750 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fbbf00a2990 0x7fbbf00a4e50 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7fbc0819b220 tx=0x7fbbf8009500 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:19.944 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:19.944+0000 7fbc04ff9640 1 -- 172.21.15.32:0/4162392750 <== mon.2 v2:172.21.15.121:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fbbfc0bc700 con 0x7fbc08101870 2024-04-25T00:33:20.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.227+0000 7fbc0db63640 1 -- 172.21.15.32:0/4162392750 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fbc081a2480 con 0x7fbc08101870 2024-04-25T00:33:20.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.228+0000 7fbc04ff9640 1 -- 172.21.15.32:0/4162392750 <== mon.2 v2:172.21.15.121:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 25 v25) ==== 94+0+11876 (secure 0 0 0) 0x7fbbfc0dc0c0 con 0x7fbc08101870 2024-04-25T00:33:20.228 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:33:20.229 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":25,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14643,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6839/187492674","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6838","nonce":187492674},{"type":"v1","addr":"172.21.15.32:6839","nonce":187492674}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14655,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6841/1948785545","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6840","nonce":1948785545},{"type":"v1","addr":"172.21.15.32:6841","nonce":1948785545}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":25,"flags":306,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":true},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-25T00:30:47.866875+0000","modified":"2024-04-25T00:33:16.420908+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24506,"mds_1":24481,"mds_2":24476},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24443":{"gid":24443,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24445":{"gid":24445,"name":"f","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24457":{"gid":24457,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24476":{"gid":24476,"name":"h","rank":2,"incarnation":22,"state":"up:active","state_seq":12,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24481":{"gid":24481,"name":"i","rank":1,"incarnation":20,"state":"up:active","state_seq":11,"addr":"172.21.15.130:6839/4267125809","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":4267125809},{"type":"v1","addr":"172.21.15.130:6839","nonce":4267125809}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24506":{"gid":24506,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.121:6839/258444853","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6838","nonce":258444853},{"type":"v1","addr":"172.21.15.121:6839","nonce":258444853}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24506,"qdb_cluster":[24506,24476,24481]},"id":1}]} 2024-04-25T00:33:20.229 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 25 2024-04-25T00:33:20.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.230+0000 7fbc0db63640 1 -- 172.21.15.32:0/4162392750 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fbbf00a2990 msgr2=0x7fbbf00a4e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:20.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.230+0000 7fbc0db63640 1 --2- 172.21.15.32:0/4162392750 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fbbf00a2990 0x7fbbf00a4e50 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7fbc0819b220 tx=0x7fbbf8009500 comp rx=0 tx=0).stop 2024-04-25T00:33:20.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.230+0000 7fbc0db63640 1 -- 172.21.15.32:0/4162392750 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbc08101870 msgr2=0x7fbc0819a290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:20.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.230+0000 7fbc0db63640 1 --2- 172.21.15.32:0/4162392750 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbc08101870 0x7fbc0819a290 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7fbbfc005ab0 tx=0x7fbbfc004270 comp rx=0 tx=0).stop 2024-04-25T00:33:20.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.230+0000 7fbc0db63640 1 -- 172.21.15.32:0/4162392750 shutdown_connections 2024-04-25T00:33:20.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.230+0000 7fbc0db63640 1 --2- 172.21.15.32:0/4162392750 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fbbf00a2990 0x7fbbf00a4e50 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:20.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.230+0000 7fbc0db63640 1 --2- 172.21.15.32:0/4162392750 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbc08102230 0x7fbc081a12c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:20.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.230+0000 7fbc0db63640 1 --2- 172.21.15.32:0/4162392750 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbc08101870 0x7fbc0819a290 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:20.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.230+0000 7fbc0db63640 1 --2- 172.21.15.32:0/4162392750 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbc08100680 0x7fbc08199d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:20.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.230+0000 7fbc0db63640 1 -- 172.21.15.32:0/4162392750 >> 172.21.15.32:0/4162392750 conn(0x7fbc080fbbf0 msgr2=0x7fbc080fd6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:20.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.230+0000 7fbc0db63640 1 -- 172.21.15.32:0/4162392750 shutdown_connections 2024-04-25T00:33:20.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.230+0000 7fbc0db63640 1 -- 172.21.15.32:0/4162392750 wait complete. 2024-04-25T00:33:20.242 INFO:tasks.fwd_scrub:Ready to start scrub thrashing 2024-04-25T00:33:20.242 INFO:tasks.fwd_scrub.ceph_manager:waiting for clean 2024-04-25T00:33:20.242 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-04-25T00:33:20.351 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.349+0000 7fbe97012640 1 -- 172.21.15.32:0/3074028866 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbe90100680 msgr2=0x7fbe90100aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:20.351 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.349+0000 7fbe97012640 1 --2- 172.21.15.32:0/3074028866 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbe90100680 0x7fbe90100aa0 secure :-1 s=READY pgs=484 cs=0 l=1 rev1=1 crypto rx=0x7fbe84009900 tx=0x7fbe8402a410 comp rx=0 tx=0).stop 2024-04-25T00:33:20.351 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.349+0000 7fbe97012640 1 -- 172.21.15.32:0/3074028866 shutdown_connections 2024-04-25T00:33:20.352 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.349+0000 7fbe97012640 1 --2- 172.21.15.32:0/3074028866 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbe90102230 0x7fbe90108ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:20.352 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.349+0000 7fbe97012640 1 --2- 172.21.15.32:0/3074028866 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbe90101870 0x7fbe90101cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:20.352 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.349+0000 7fbe97012640 1 --2- 172.21.15.32:0/3074028866 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbe90100680 0x7fbe90100aa0 unknown :-1 s=CLOSED pgs=484 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:20.352 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.349+0000 7fbe97012640 1 -- 172.21.15.32:0/3074028866 >> 172.21.15.32:0/3074028866 conn(0x7fbe900fbbf0 msgr2=0x7fbe900fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:20.352 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.349+0000 7fbe97012640 1 -- 172.21.15.32:0/3074028866 shutdown_connections 2024-04-25T00:33:20.352 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.349+0000 7fbe97012640 1 -- 172.21.15.32:0/3074028866 wait complete. 2024-04-25T00:33:20.352 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.349+0000 7fbe97012640 1 Processor -- start 2024-04-25T00:33:20.352 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.350+0000 7fbe97012640 1 -- start start 2024-04-25T00:33:20.352 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.350+0000 7fbe97012640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbe90100680 0x7fbe90197aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:20.352 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.350+0000 7fbe97012640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbe90101870 0x7fbe90197fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:20.352 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.350+0000 7fbe97012640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbe90102230 0x7fbe9019f050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:20.352 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.350+0000 7fbe97012640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fbe9010c300 con 0x7fbe90102230 2024-04-25T00:33:20.352 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.350+0000 7fbe97012640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fbe9010c180 con 0x7fbe90100680 2024-04-25T00:33:20.352 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.350+0000 7fbe97012640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fbe9010c480 con 0x7fbe90101870 2024-04-25T00:33:20.353 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.350+0000 7fbe94dae640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbe90100680 0x7fbe90197aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:20.353 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.350+0000 7fbe955af640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbe90102230 0x7fbe9019f050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:20.353 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.350+0000 7fbe94dae640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbe90100680 0x7fbe90197aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:34730/0 (socket says 172.21.15.32:34730) 2024-04-25T00:33:20.353 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.350+0000 7fbe8ffff640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbe90101870 0x7fbe90197fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:20.353 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.350+0000 7fbe94dae640 1 -- 172.21.15.32:0/2618967935 learned_addr learned my addr 172.21.15.32:0/2618967935 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:20.353 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.350+0000 7fbe94dae640 1 -- 172.21.15.32:0/2618967935 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbe90101870 msgr2=0x7fbe90197fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:20.353 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.350+0000 7fbe94dae640 1 --2- 172.21.15.32:0/2618967935 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbe90101870 0x7fbe90197fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:20.353 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.350+0000 7fbe94dae640 1 -- 172.21.15.32:0/2618967935 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbe90102230 msgr2=0x7fbe9019f050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:20.353 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.350+0000 7fbe94dae640 1 --2- 172.21.15.32:0/2618967935 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbe90102230 0x7fbe9019f050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:20.353 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.350+0000 7fbe94dae640 1 -- 172.21.15.32:0/2618967935 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbe9019f6c0 con 0x7fbe90100680 2024-04-25T00:33:20.353 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.350+0000 7fbe955af640 1 --2- 172.21.15.32:0/2618967935 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbe90102230 0x7fbe9019f050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:20.353 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.350+0000 7fbe94dae640 1 --2- 172.21.15.32:0/2618967935 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbe90100680 0x7fbe90197aa0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7fbe84031610 tx=0x7fbe84004270 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:20.353 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.351+0000 7fbe8dffb640 1 -- 172.21.15.32:0/2618967935 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbe8403f030 con 0x7fbe90100680 2024-04-25T00:33:20.353 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.351+0000 7fbe97012640 1 -- 172.21.15.32:0/2618967935 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbe9019f950 con 0x7fbe90100680 2024-04-25T00:33:20.353 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.351+0000 7fbe8dffb640 1 -- 172.21.15.32:0/2618967935 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbe84003ad0 con 0x7fbe90100680 2024-04-25T00:33:20.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.351+0000 7fbe8dffb640 1 -- 172.21.15.32:0/2618967935 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbe84035070 con 0x7fbe90100680 2024-04-25T00:33:20.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.351+0000 7fbe97012640 1 -- 172.21.15.32:0/2618967935 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fbe9019fe90 con 0x7fbe90100680 2024-04-25T00:33:20.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.352+0000 7fbe97012640 1 -- 172.21.15.32:0/2618967935 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fbe901a0060 con 0x7fbe90100680 2024-04-25T00:33:20.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.353+0000 7fbe8dffb640 1 -- 172.21.15.32:0/2618967935 <== mon.2 v2:172.21.15.121:3300/0 4 ==== mgrmap(e 22) ==== 137645+0+0 (secure 0 0 0) 0x7fbe84042030 con 0x7fbe90100680 2024-04-25T00:33:20.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.354+0000 7fbe8dffb640 1 --2- 172.21.15.32:0/2618967935 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fbe800a2990 0x7fbe800a4e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:20.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.354+0000 7fbe8ffff640 1 --2- 172.21.15.32:0/2618967935 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fbe800a2990 0x7fbe800a4e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:20.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.354+0000 7fbe8dffb640 1 -- 172.21.15.32:0/2618967935 <== mon.2 v2:172.21.15.121:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fbe840ef960 con 0x7fbe90100680 2024-04-25T00:33:20.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.354+0000 7fbe8ffff640 1 --2- 172.21.15.32:0/2618967935 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fbe800a2990 0x7fbe800a4e50 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7fbe90198fb0 tx=0x7fbe7c005f90 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:20.357 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.357+0000 7fbe8dffb640 1 -- 172.21.15.32:0/2618967935 <== mon.2 v2:172.21.15.121:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fbe840bc050 con 0x7fbe90100680 2024-04-25T00:33:20.500 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.499+0000 7fbe8dffb640 1 -- 172.21.15.32:0/2618967935 <== mon.2 v2:172.21.15.121:3300/0 7 ==== mgrmap(e 23) ==== 137953+0+0 (secure 0 0 0) 0x7fbe84003c80 con 0x7fbe90100680 2024-04-25T00:33:20.501 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:20 smithi032 ceph-mon[29357]: [25/Apr/2024:00:33:18] ENGINE Serving on http://172.21.15.32:8765 2024-04-25T00:33:20.501 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:20 smithi032 ceph-mon[29357]: [25/Apr/2024:00:33:18] ENGINE Serving on https://172.21.15.32:7150 2024-04-25T00:33:20.501 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:20 smithi032 ceph-mon[29357]: [25/Apr/2024:00:33:18] ENGINE Bus STARTED 2024-04-25T00:33:20.501 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:20 smithi032 ceph-mon[29357]: pgmap v3: 129 pgs: 129 active+clean; 1.1 MiB data, 373 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:33:20.501 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:20 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1898524971' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:33:20.501 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:20 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2675413030' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:33:20.501 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:20 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:20.502 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:20 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:20.502 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:20 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:20.502 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:20 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:20.502 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:20 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/4162392750' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:33:20.502 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:20 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:20.502 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:20 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:20.550 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.550+0000 7fbe97012640 1 -- 172.21.15.32:0/2618967935 --> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7fbe900fdda0 con 0x7fbe800a2990 2024-04-25T00:33:20.561 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.560+0000 7fbe8dffb640 1 -- 172.21.15.32:0/2618967935 <== mgr.15483 v2:172.21.15.32:6800/3307874911 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+339393 (secure 0 0 0) 0x7fbe900fdda0 con 0x7fbe800a2990 2024-04-25T00:33:20.565 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:33:20.568 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-04-25T00:33:20.570 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.564+0000 7fbe97012640 1 -- 172.21.15.32:0/2618967935 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fbe800a2990 msgr2=0x7fbe800a4e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:20.570 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.564+0000 7fbe97012640 1 --2- 172.21.15.32:0/2618967935 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fbe800a2990 0x7fbe800a4e50 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7fbe90198fb0 tx=0x7fbe7c005f90 comp rx=0 tx=0).stop 2024-04-25T00:33:20.570 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.564+0000 7fbe97012640 1 -- 172.21.15.32:0/2618967935 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbe90100680 msgr2=0x7fbe90197aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:20.570 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.564+0000 7fbe97012640 1 --2- 172.21.15.32:0/2618967935 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbe90100680 0x7fbe90197aa0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7fbe84031610 tx=0x7fbe84004270 comp rx=0 tx=0).stop 2024-04-25T00:33:20.570 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.564+0000 7fbe97012640 1 -- 172.21.15.32:0/2618967935 shutdown_connections 2024-04-25T00:33:20.571 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.564+0000 7fbe97012640 1 --2- 172.21.15.32:0/2618967935 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fbe800a2990 0x7fbe800a4e50 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:20.571 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.564+0000 7fbe97012640 1 --2- 172.21.15.32:0/2618967935 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbe90102230 0x7fbe9019f050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:20.571 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.564+0000 7fbe97012640 1 --2- 172.21.15.32:0/2618967935 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbe90101870 0x7fbe90197fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:20.571 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.564+0000 7fbe97012640 1 --2- 172.21.15.32:0/2618967935 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbe90100680 0x7fbe90197aa0 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:20.571 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.564+0000 7fbe97012640 1 -- 172.21.15.32:0/2618967935 >> 172.21.15.32:0/2618967935 conn(0x7fbe900fbbf0 msgr2=0x7fbe900fd690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:20.571 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.564+0000 7fbe97012640 1 -- 172.21.15.32:0/2618967935 shutdown_connections 2024-04-25T00:33:20.571 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.564+0000 7fbe97012640 1 -- 172.21.15.32:0/2618967935 wait complete. 2024-04-25T00:33:20.576 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":4,"stamp":"2024-04-25T00:33:19.938647+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1188469,"num_objects":66,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":66,"num_whiteouts":0,"num_read":1000,"num_read_kb":659,"num_write":472,"num_write_kb":4381,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":1180987,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":868,"ondisk_log_size":868,"up":419,"acting":419,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":419,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":381708,"kb_used_data":15880,"kb_used_omap":18,"kb_used_meta":365677,"kb_avail":1124560116,"statfs":{"total":1151940427776,"available":1151549558784,"internally_reserved":0,"allocated":16261120,"data_stored":9548569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19142,"internal_metadata":374453562},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.990637"},"pg_stats":[{"pgid":"3.3f","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305395+0000","last_change":"2024-04-25T00:30:48.538719+0000","last_active":"2024-04-25T00:33:18.305395+0000","last_peered":"2024-04-25T00:33:18.305395+0000","last_clean":"2024-04-25T00:33:18.305395+0000","last_became_active":"2024-04-25T00:30:48.538356+0000","last_became_peered":"2024-04-25T00:30:48.538356+0000","last_unstale":"2024-04-25T00:33:18.305395+0000","last_undegraded":"2024-04-25T00:33:18.305395+0000","last_fullsized":"2024-04-25T00:33:18.305395+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:39:08.974838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"91'4","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305901+0000","last_change":"2024-04-25T00:30:48.532262+0000","last_active":"2024-04-25T00:33:18.305901+0000","last_peered":"2024-04-25T00:33:18.305901+0000","last_clean":"2024-04-25T00:33:18.305901+0000","last_became_active":"2024-04-25T00:30:48.531926+0000","last_became_peered":"2024-04-25T00:30:48.531926+0000","last_unstale":"2024-04-25T00:33:18.305901+0000","last_undegraded":"2024-04-25T00:33:18.305901+0000","last_fullsized":"2024-04-25T00:33:18.305901+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:12:06.527649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"91'7","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305705+0000","last_change":"2024-04-25T00:30:48.536383+0000","last_active":"2024-04-25T00:33:18.305705+0000","last_peered":"2024-04-25T00:33:18.305705+0000","last_clean":"2024-04-25T00:33:18.305705+0000","last_became_active":"2024-04-25T00:30:48.536048+0000","last_became_peered":"2024-04-25T00:30:48.536048+0000","last_unstale":"2024-04-25T00:33:18.305705+0000","last_undegraded":"2024-04-25T00:33:18.305705+0000","last_fullsized":"2024-04-25T00:33:18.305705+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:32:26.728441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926366+0000","last_change":"2024-04-25T00:30:48.532195+0000","last_active":"2024-04-25T00:33:17.926366+0000","last_peered":"2024-04-25T00:33:17.926366+0000","last_clean":"2024-04-25T00:33:17.926366+0000","last_became_active":"2024-04-25T00:30:48.531851+0000","last_became_peered":"2024-04-25T00:30:48.531851+0000","last_unstale":"2024-04-25T00:33:17.926366+0000","last_undegraded":"2024-04-25T00:33:17.926366+0000","last_fullsized":"2024-04-25T00:33:17.926366+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:15:34.704958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"91'4","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.030694+0000","last_change":"2024-04-25T00:30:48.534024+0000","last_active":"2024-04-25T00:33:18.030694+0000","last_peered":"2024-04-25T00:33:18.030694+0000","last_clean":"2024-04-25T00:33:18.030694+0000","last_became_active":"2024-04-25T00:30:48.533681+0000","last_became_peered":"2024-04-25T00:30:48.533681+0000","last_unstale":"2024-04-25T00:33:18.030694+0000","last_undegraded":"2024-04-25T00:33:18.030694+0000","last_fullsized":"2024-04-25T00:33:18.030694+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:07:35.816816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"91'6","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.040282+0000","last_change":"2024-04-25T00:30:48.541263+0000","last_active":"2024-04-25T00:33:18.040282+0000","last_peered":"2024-04-25T00:33:18.040282+0000","last_clean":"2024-04-25T00:33:18.040282+0000","last_became_active":"2024-04-25T00:30:48.540889+0000","last_became_peered":"2024-04-25T00:30:48.540889+0000","last_unstale":"2024-04-25T00:33:18.040282+0000","last_undegraded":"2024-04-25T00:33:18.040282+0000","last_fullsized":"2024-04-25T00:33:18.040282+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:31:42.067004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,4],"acting":[10,9,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"91'5","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055606+0000","last_change":"2024-04-25T00:30:48.533194+0000","last_active":"2024-04-25T00:33:18.055606+0000","last_peered":"2024-04-25T00:33:18.055606+0000","last_clean":"2024-04-25T00:33:18.055606+0000","last_became_active":"2024-04-25T00:30:48.532424+0000","last_became_peered":"2024-04-25T00:30:48.532424+0000","last_unstale":"2024-04-25T00:33:18.055606+0000","last_undegraded":"2024-04-25T00:33:18.055606+0000","last_fullsized":"2024-04-25T00:33:18.055606+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:06:50.187796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"91'9","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.924700+0000","last_change":"2024-04-25T00:30:48.542829+0000","last_active":"2024-04-25T00:33:17.924700+0000","last_peered":"2024-04-25T00:33:17.924700+0000","last_clean":"2024-04-25T00:33:17.924700+0000","last_became_active":"2024-04-25T00:30:48.542204+0000","last_became_peered":"2024-04-25T00:30:48.542204+0000","last_unstale":"2024-04-25T00:33:17.924700+0000","last_undegraded":"2024-04-25T00:33:17.924700+0000","last_fullsized":"2024-04-25T00:33:17.924700+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:56:47.869075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"91'11","reported_seq":43,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.188149+0000","last_change":"2024-04-25T00:30:48.537924+0000","last_active":"2024-04-25T00:33:18.188149+0000","last_peered":"2024-04-25T00:33:18.188149+0000","last_clean":"2024-04-25T00:33:18.188149+0000","last_became_active":"2024-04-25T00:30:48.537547+0000","last_became_peered":"2024-04-25T00:30:48.537547+0000","last_unstale":"2024-04-25T00:33:18.188149+0000","last_undegraded":"2024-04-25T00:33:18.188149+0000","last_fullsized":"2024-04-25T00:33:18.188149+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:26:45.590923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"91'16","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.040641+0000","last_change":"2024-04-25T00:30:48.539717+0000","last_active":"2024-04-25T00:33:18.040641+0000","last_peered":"2024-04-25T00:33:18.040641+0000","last_clean":"2024-04-25T00:33:18.040641+0000","last_became_active":"2024-04-25T00:30:48.539242+0000","last_became_peered":"2024-04-25T00:30:48.539242+0000","last_unstale":"2024-04-25T00:33:18.040641+0000","last_undegraded":"2024-04-25T00:33:18.040641+0000","last_fullsized":"2024-04-25T00:33:18.040641+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":16,"log_dups_size":0,"ondisk_log_size":16,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:58:15.353959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"91'6","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055839+0000","last_change":"2024-04-25T00:30:48.542767+0000","last_active":"2024-04-25T00:33:18.055839+0000","last_peered":"2024-04-25T00:33:18.055839+0000","last_clean":"2024-04-25T00:33:18.055839+0000","last_became_active":"2024-04-25T00:30:48.542252+0000","last_became_peered":"2024-04-25T00:30:48.542252+0000","last_unstale":"2024-04-25T00:33:18.055839+0000","last_undegraded":"2024-04-25T00:33:18.055839+0000","last_fullsized":"2024-04-25T00:33:18.055839+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:07:49.414660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"91'7","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926388+0000","last_change":"2024-04-25T00:30:48.532248+0000","last_active":"2024-04-25T00:33:17.926388+0000","last_peered":"2024-04-25T00:33:17.926388+0000","last_clean":"2024-04-25T00:33:17.926388+0000","last_became_active":"2024-04-25T00:30:48.531876+0000","last_became_peered":"2024-04-25T00:30:48.531876+0000","last_unstale":"2024-04-25T00:33:17.926388+0000","last_undegraded":"2024-04-25T00:33:17.926388+0000","last_fullsized":"2024-04-25T00:33:17.926388+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:00:35.711693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"91'10","reported_seq":42,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.167693+0000","last_change":"2024-04-25T00:30:48.526871+0000","last_active":"2024-04-25T00:33:18.167693+0000","last_peered":"2024-04-25T00:33:18.167693+0000","last_clean":"2024-04-25T00:33:18.167693+0000","last_became_active":"2024-04-25T00:30:48.526477+0000","last_became_peered":"2024-04-25T00:30:48.526477+0000","last_unstale":"2024-04-25T00:33:18.167693+0000","last_undegraded":"2024-04-25T00:33:18.167693+0000","last_fullsized":"2024-04-25T00:33:18.167693+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:17:59.447932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"91'7","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305099+0000","last_change":"2024-04-25T00:30:48.530297+0000","last_active":"2024-04-25T00:33:18.305099+0000","last_peered":"2024-04-25T00:33:18.305099+0000","last_clean":"2024-04-25T00:33:18.305099+0000","last_became_active":"2024-04-25T00:30:48.529091+0000","last_became_peered":"2024-04-25T00:30:48.529091+0000","last_unstale":"2024-04-25T00:33:18.305099+0000","last_undegraded":"2024-04-25T00:33:18.305099+0000","last_fullsized":"2024-04-25T00:33:18.305099+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:54:52.206881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"91'7","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305119+0000","last_change":"2024-04-25T00:30:48.535629+0000","last_active":"2024-04-25T00:33:18.305119+0000","last_peered":"2024-04-25T00:33:18.305119+0000","last_clean":"2024-04-25T00:33:18.305119+0000","last_became_active":"2024-04-25T00:30:48.535190+0000","last_became_peered":"2024-04-25T00:30:48.535190+0000","last_unstale":"2024-04-25T00:33:18.305119+0000","last_undegraded":"2024-04-25T00:33:18.305119+0000","last_fullsized":"2024-04-25T00:33:18.305119+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:11:27.127964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"91'9","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.187509+0000","last_change":"2024-04-25T00:30:48.532988+0000","last_active":"2024-04-25T00:33:18.187509+0000","last_peered":"2024-04-25T00:33:18.187509+0000","last_clean":"2024-04-25T00:33:18.187509+0000","last_became_active":"2024-04-25T00:30:48.532670+0000","last_became_peered":"2024-04-25T00:30:48.532670+0000","last_unstale":"2024-04-25T00:33:18.187509+0000","last_undegraded":"2024-04-25T00:33:18.187509+0000","last_fullsized":"2024-04-25T00:33:18.187509+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:03:35.336175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305254+0000","last_change":"2024-04-25T00:30:48.539499+0000","last_active":"2024-04-25T00:33:18.305254+0000","last_peered":"2024-04-25T00:33:18.305254+0000","last_clean":"2024-04-25T00:33:18.305254+0000","last_became_active":"2024-04-25T00:30:48.538516+0000","last_became_peered":"2024-04-25T00:30:48.538516+0000","last_unstale":"2024-04-25T00:33:18.305254+0000","last_undegraded":"2024-04-25T00:33:18.305254+0000","last_fullsized":"2024-04-25T00:33:18.305254+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:03:16.781091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"91'10","reported_seq":42,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.028510+0000","last_change":"2024-04-25T00:30:48.534204+0000","last_active":"2024-04-25T00:33:18.028510+0000","last_peered":"2024-04-25T00:33:18.028510+0000","last_clean":"2024-04-25T00:33:18.028510+0000","last_became_active":"2024-04-25T00:30:48.533818+0000","last_became_peered":"2024-04-25T00:30:48.533818+0000","last_unstale":"2024-04-25T00:33:18.028510+0000","last_undegraded":"2024-04-25T00:33:18.028510+0000","last_fullsized":"2024-04-25T00:33:18.028510+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:08:42.964056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"91'7","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.927103+0000","last_change":"2024-04-25T00:30:48.542756+0000","last_active":"2024-04-25T00:33:17.927103+0000","last_peered":"2024-04-25T00:33:17.927103+0000","last_clean":"2024-04-25T00:33:17.927103+0000","last_became_active":"2024-04-25T00:30:48.541824+0000","last_became_peered":"2024-04-25T00:30:48.541824+0000","last_unstale":"2024-04-25T00:33:17.927103+0000","last_undegraded":"2024-04-25T00:33:17.927103+0000","last_fullsized":"2024-04-25T00:33:17.927103+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:31:40.996474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926670+0000","last_change":"2024-04-25T00:30:52.788477+0000","last_active":"2024-04-25T00:33:17.926670+0000","last_peered":"2024-04-25T00:33:17.926670+0000","last_clean":"2024-04-25T00:33:17.926670+0000","last_became_active":"2024-04-25T00:30:52.788073+0000","last_became_peered":"2024-04-25T00:30:52.788073+0000","last_unstale":"2024-04-25T00:33:17.926670+0000","last_undegraded":"2024-04-25T00:33:17.926670+0000","last_fullsized":"2024-04-25T00:33:17.926670+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:29:34.781119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,9,0],"acting":[2,3,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"89'9","reported_seq":44,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.028297+0000","last_change":"2024-04-25T00:30:47.524580+0000","last_active":"2024-04-25T00:33:18.028297+0000","last_peered":"2024-04-25T00:33:18.028297+0000","last_clean":"2024-04-25T00:33:18.028297+0000","last_became_active":"2024-04-25T00:30:47.524170+0000","last_became_peered":"2024-04-25T00:30:47.524170+0000","last_unstale":"2024-04-25T00:33:18.028297+0000","last_undegraded":"2024-04-25T00:33:18.028297+0000","last_fullsized":"2024-04-25T00:33:18.028297+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:53:08.734842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"91'5","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.925270+0000","last_change":"2024-04-25T00:30:48.542077+0000","last_active":"2024-04-25T00:33:17.925270+0000","last_peered":"2024-04-25T00:33:17.925270+0000","last_clean":"2024-04-25T00:33:17.925270+0000","last_became_active":"2024-04-25T00:30:48.539043+0000","last_became_peered":"2024-04-25T00:30:48.539043+0000","last_unstale":"2024-04-25T00:33:17.925270+0000","last_undegraded":"2024-04-25T00:33:17.925270+0000","last_fullsized":"2024-04-25T00:33:17.925270+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:48:52.254800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926432+0000","last_change":"2024-04-25T00:30:52.786091+0000","last_active":"2024-04-25T00:33:17.926432+0000","last_peered":"2024-04-25T00:33:17.926432+0000","last_clean":"2024-04-25T00:33:17.926432+0000","last_became_active":"2024-04-25T00:30:52.785511+0000","last_became_peered":"2024-04-25T00:30:52.785511+0000","last_unstale":"2024-04-25T00:33:17.926432+0000","last_undegraded":"2024-04-25T00:33:17.926432+0000","last_fullsized":"2024-04-25T00:33:17.926432+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:19:17.992033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,9,3],"acting":[4,6,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1e","version":"89'2","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.925488+0000","last_change":"2024-04-25T00:30:47.529293+0000","last_active":"2024-04-25T00:33:17.925488+0000","last_peered":"2024-04-25T00:33:17.925488+0000","last_clean":"2024-04-25T00:33:17.925488+0000","last_became_active":"2024-04-25T00:30:47.527975+0000","last_became_peered":"2024-04-25T00:30:47.527975+0000","last_unstale":"2024-04-25T00:33:17.925488+0000","last_undegraded":"2024-04-25T00:33:17.925488+0000","last_fullsized":"2024-04-25T00:33:17.925488+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:17:40.057962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"91'6","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305797+0000","last_change":"2024-04-25T00:30:48.530993+0000","last_active":"2024-04-25T00:33:18.305797+0000","last_peered":"2024-04-25T00:33:18.305797+0000","last_clean":"2024-04-25T00:33:18.305797+0000","last_became_active":"2024-04-25T00:30:48.530550+0000","last_became_peered":"2024-04-25T00:30:48.530550+0000","last_unstale":"2024-04-25T00:33:18.305797+0000","last_undegraded":"2024-04-25T00:33:18.305797+0000","last_fullsized":"2024-04-25T00:33:18.305797+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:11:35.308456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":24,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926274+0000","last_change":"2024-04-25T00:30:52.787774+0000","last_active":"2024-04-25T00:33:17.926274+0000","last_peered":"2024-04-25T00:33:17.926274+0000","last_clean":"2024-04-25T00:33:17.926274+0000","last_became_active":"2024-04-25T00:30:52.786988+0000","last_became_peered":"2024-04-25T00:30:52.786988+0000","last_unstale":"2024-04-25T00:33:17.926274+0000","last_undegraded":"2024-04-25T00:33:17.926274+0000","last_fullsized":"2024-04-25T00:33:17.926274+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:58:28.081767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,9,5],"acting":[4,3,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1d","version":"89'4","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.030776+0000","last_change":"2024-04-25T00:30:47.523477+0000","last_active":"2024-04-25T00:33:18.030776+0000","last_peered":"2024-04-25T00:33:18.030776+0000","last_clean":"2024-04-25T00:33:18.030776+0000","last_became_active":"2024-04-25T00:30:47.522961+0000","last_became_peered":"2024-04-25T00:30:47.522961+0000","last_unstale":"2024-04-25T00:33:18.030776+0000","last_undegraded":"2024-04-25T00:33:18.030776+0000","last_fullsized":"2024-04-25T00:33:18.030776+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:39:39.793477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"91'9","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.187779+0000","last_change":"2024-04-25T00:30:48.527746+0000","last_active":"2024-04-25T00:33:18.187779+0000","last_peered":"2024-04-25T00:33:18.187779+0000","last_clean":"2024-04-25T00:33:18.187779+0000","last_became_active":"2024-04-25T00:30:48.527374+0000","last_became_peered":"2024-04-25T00:30:48.527374+0000","last_unstale":"2024-04-25T00:33:18.187779+0000","last_undegraded":"2024-04-25T00:33:18.187779+0000","last_fullsized":"2024-04-25T00:33:18.187779+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:28:36.822220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926088+0000","last_change":"2024-04-25T00:30:52.782368+0000","last_active":"2024-04-25T00:33:17.926088+0000","last_peered":"2024-04-25T00:33:17.926088+0000","last_clean":"2024-04-25T00:33:17.926088+0000","last_became_active":"2024-04-25T00:30:52.781875+0000","last_became_peered":"2024-04-25T00:30:52.781875+0000","last_unstale":"2024-04-25T00:33:17.926088+0000","last_undegraded":"2024-04-25T00:33:17.926088+0000","last_fullsized":"2024-04-25T00:33:17.926088+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:07:55.509131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5,3],"acting":[4,6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1c","version":"84'2","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.031019+0000","last_change":"2024-04-25T00:30:47.523707+0000","last_active":"2024-04-25T00:33:18.031019+0000","last_peered":"2024-04-25T00:33:18.031019+0000","last_clean":"2024-04-25T00:33:18.031019+0000","last_became_active":"2024-04-25T00:30:47.523216+0000","last_became_peered":"2024-04-25T00:30:47.523216+0000","last_unstale":"2024-04-25T00:33:18.031019+0000","last_undegraded":"2024-04-25T00:33:18.031019+0000","last_fullsized":"2024-04-25T00:33:18.031019+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:21:21.006368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"91'15","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.037238+0000","last_change":"2024-04-25T00:30:48.529235+0000","last_active":"2024-04-25T00:33:18.037238+0000","last_peered":"2024-04-25T00:33:18.037238+0000","last_clean":"2024-04-25T00:33:18.037238+0000","last_became_active":"2024-04-25T00:30:48.528887+0000","last_became_peered":"2024-04-25T00:30:48.528887+0000","last_unstale":"2024-04-25T00:33:18.037238+0000","last_undegraded":"2024-04-25T00:33:18.037238+0000","last_fullsized":"2024-04-25T00:33:18.037238+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:18:05.290578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":31,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926986+0000","last_change":"2024-04-25T00:31:44.714433+0000","last_active":"2024-04-25T00:33:17.926986+0000","last_peered":"2024-04-25T00:33:17.926986+0000","last_clean":"2024-04-25T00:33:17.926986+0000","last_became_active":"2024-04-25T00:31:44.713810+0000","last_became_peered":"2024-04-25T00:31:44.713810+0000","last_unstale":"2024-04-25T00:33:17.926986+0000","last_undegraded":"2024-04-25T00:33:17.926986+0000","last_fullsized":"2024-04-25T00:33:17.926986+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:00:49.815134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,7,11],"acting":[2,8,7,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1b","version":"94'6","reported_seq":166,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.185075+0000","last_change":"2024-04-25T00:30:47.518472+0000","last_active":"2024-04-25T00:33:18.185075+0000","last_peered":"2024-04-25T00:33:18.185075+0000","last_clean":"2024-04-25T00:33:18.185075+0000","last_became_active":"2024-04-25T00:30:47.518140+0000","last_became_peered":"2024-04-25T00:30:47.518140+0000","last_unstale":"2024-04-25T00:33:18.185075+0000","last_undegraded":"2024-04-25T00:33:18.185075+0000","last_fullsized":"2024-04-25T00:33:18.185075+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:19:07.780949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1013,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":2,"num_write":3,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"91'7","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.030957+0000","last_change":"2024-04-25T00:30:48.533757+0000","last_active":"2024-04-25T00:33:18.030957+0000","last_peered":"2024-04-25T00:33:18.030957+0000","last_clean":"2024-04-25T00:33:18.030957+0000","last_became_active":"2024-04-25T00:30:48.533421+0000","last_became_peered":"2024-04-25T00:30:48.533421+0000","last_unstale":"2024-04-25T00:33:18.030957+0000","last_undegraded":"2024-04-25T00:33:18.030957+0000","last_fullsized":"2024-04-25T00:33:18.030957+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:11:38.190783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":31,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926818+0000","last_change":"2024-04-25T00:31:45.031412+0000","last_active":"2024-04-25T00:33:17.926818+0000","last_peered":"2024-04-25T00:33:17.926818+0000","last_clean":"2024-04-25T00:33:17.926818+0000","last_became_active":"2024-04-25T00:31:45.030822+0000","last_became_peered":"2024-04-25T00:31:45.030822+0000","last_unstale":"2024-04-25T00:33:17.926818+0000","last_undegraded":"2024-04-25T00:33:17.926818+0000","last_fullsized":"2024-04-25T00:33:17.926818+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:34:26.758966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,8,5],"acting":[2,6,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1a","version":"89'3","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.040284+0000","last_change":"2024-04-25T00:30:47.527306+0000","last_active":"2024-04-25T00:33:18.040284+0000","last_peered":"2024-04-25T00:33:18.040284+0000","last_clean":"2024-04-25T00:33:18.040284+0000","last_became_active":"2024-04-25T00:30:47.526935+0000","last_became_peered":"2024-04-25T00:30:47.526935+0000","last_unstale":"2024-04-25T00:33:18.040284+0000","last_undegraded":"2024-04-25T00:33:18.040284+0000","last_fullsized":"2024-04-25T00:33:18.040284+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:41:37.951143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"91'8","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.027475+0000","last_change":"2024-04-25T00:31:45.170610+0000","last_active":"2024-04-25T00:33:18.027475+0000","last_peered":"2024-04-25T00:33:18.027475+0000","last_clean":"2024-04-25T00:33:18.027475+0000","last_became_active":"2024-04-25T00:31:45.170183+0000","last_became_peered":"2024-04-25T00:31:45.170183+0000","last_unstale":"2024-04-25T00:33:18.027475+0000","last_undegraded":"2024-04-25T00:33:18.027475+0000","last_fullsized":"2024-04-25T00:33:18.027475+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:39:39.781997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,4,3],"acting":[9,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305164+0000","last_change":"2024-04-25T00:30:52.784986+0000","last_active":"2024-04-25T00:33:18.305164+0000","last_peered":"2024-04-25T00:33:18.305164+0000","last_clean":"2024-04-25T00:33:18.305164+0000","last_became_active":"2024-04-25T00:30:52.784593+0000","last_became_peered":"2024-04-25T00:30:52.784593+0000","last_unstale":"2024-04-25T00:33:18.305164+0000","last_undegraded":"2024-04-25T00:33:18.305164+0000","last_fullsized":"2024-04-25T00:33:18.305164+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:04:05.833730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,9,0],"acting":[11,4,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.19","version":"89'2","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.166738+0000","last_change":"2024-04-25T00:30:47.523663+0000","last_active":"2024-04-25T00:33:18.166738+0000","last_peered":"2024-04-25T00:33:18.166738+0000","last_clean":"2024-04-25T00:33:18.166738+0000","last_became_active":"2024-04-25T00:30:47.523324+0000","last_became_peered":"2024-04-25T00:30:47.523324+0000","last_unstale":"2024-04-25T00:33:18.166738+0000","last_undegraded":"2024-04-25T00:33:18.166738+0000","last_fullsized":"2024-04-25T00:33:18.166738+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:51:23.593579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.924628+0000","last_change":"2024-04-25T00:30:48.529738+0000","last_active":"2024-04-25T00:33:17.924628+0000","last_peered":"2024-04-25T00:33:17.924628+0000","last_clean":"2024-04-25T00:33:17.924628+0000","last_became_active":"2024-04-25T00:30:48.529350+0000","last_became_peered":"2024-04-25T00:30:48.529350+0000","last_unstale":"2024-04-25T00:33:17.924628+0000","last_undegraded":"2024-04-25T00:33:17.924628+0000","last_fullsized":"2024-04-25T00:33:17.924628+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:51:29.238293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305792+0000","last_change":"2024-04-25T00:30:52.788480+0000","last_active":"2024-04-25T00:33:18.305792+0000","last_peered":"2024-04-25T00:33:18.305792+0000","last_clean":"2024-04-25T00:33:18.305792+0000","last_became_active":"2024-04-25T00:30:52.788065+0000","last_became_peered":"2024-04-25T00:30:52.788065+0000","last_unstale":"2024-04-25T00:33:18.305792+0000","last_undegraded":"2024-04-25T00:33:18.305792+0000","last_fullsized":"2024-04-25T00:33:18.305792+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:16:50.179177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6,1],"acting":[0,7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.18","version":"89'1","reported_seq":35,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.036951+0000","last_change":"2024-04-25T00:30:47.524706+0000","last_active":"2024-04-25T00:33:18.036951+0000","last_peered":"2024-04-25T00:33:18.036951+0000","last_clean":"2024-04-25T00:33:18.036951+0000","last_became_active":"2024-04-25T00:30:47.524355+0000","last_became_peered":"2024-04-25T00:30:47.524355+0000","last_unstale":"2024-04-25T00:33:18.036951+0000","last_undegraded":"2024-04-25T00:33:18.036951+0000","last_fullsized":"2024-04-25T00:33:18.036951+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:22:38.093362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"91'11","reported_seq":43,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.027727+0000","last_change":"2024-04-25T00:30:48.533266+0000","last_active":"2024-04-25T00:33:18.027727+0000","last_peered":"2024-04-25T00:33:18.027727+0000","last_clean":"2024-04-25T00:33:18.027727+0000","last_became_active":"2024-04-25T00:30:48.532932+0000","last_became_peered":"2024-04-25T00:30:48.532932+0000","last_unstale":"2024-04-25T00:33:18.027727+0000","last_undegraded":"2024-04-25T00:33:18.027727+0000","last_fullsized":"2024-04-25T00:33:18.027727+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:00:54.147725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.924818+0000","last_change":"2024-04-25T00:30:52.780317+0000","last_active":"2024-04-25T00:33:17.924818+0000","last_peered":"2024-04-25T00:33:17.924818+0000","last_clean":"2024-04-25T00:33:17.924818+0000","last_became_active":"2024-04-25T00:30:52.778539+0000","last_became_peered":"2024-04-25T00:30:52.778539+0000","last_unstale":"2024-04-25T00:33:17.924818+0000","last_undegraded":"2024-04-25T00:33:17.924818+0000","last_fullsized":"2024-04-25T00:33:17.924818+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:34:49.527307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1,5],"acting":[3,7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.17","version":"94'4","reported_seq":162,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.185182+0000","last_change":"2024-04-25T00:30:47.525647+0000","last_active":"2024-04-25T00:33:18.185182+0000","last_peered":"2024-04-25T00:33:18.185182+0000","last_clean":"2024-04-25T00:33:18.185182+0000","last_became_active":"2024-04-25T00:30:47.525280+0000","last_became_peered":"2024-04-25T00:30:47.525280+0000","last_unstale":"2024-04-25T00:33:18.185182+0000","last_undegraded":"2024-04-25T00:33:18.185182+0000","last_fullsized":"2024-04-25T00:33:18.185182+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:56:15.761173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":124,"num_read_kb":124,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"91'10","reported_seq":42,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.188103+0000","last_change":"2024-04-25T00:30:48.527913+0000","last_active":"2024-04-25T00:33:18.188103+0000","last_peered":"2024-04-25T00:33:18.188103+0000","last_clean":"2024-04-25T00:33:18.188103+0000","last_became_active":"2024-04-25T00:30:48.527538+0000","last_became_peered":"2024-04-25T00:30:48.527538+0000","last_unstale":"2024-04-25T00:33:18.188103+0000","last_undegraded":"2024-04-25T00:33:18.188103+0000","last_fullsized":"2024-04-25T00:33:18.188103+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:48:38.646867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.027433+0000","last_change":"2024-04-25T00:30:52.788267+0000","last_active":"2024-04-25T00:33:18.027433+0000","last_peered":"2024-04-25T00:33:18.027433+0000","last_clean":"2024-04-25T00:33:18.027433+0000","last_became_active":"2024-04-25T00:30:52.785188+0000","last_became_peered":"2024-04-25T00:30:52.785188+0000","last_unstale":"2024-04-25T00:33:18.027433+0000","last_undegraded":"2024-04-25T00:33:18.027433+0000","last_fullsized":"2024-04-25T00:33:18.027433+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:38:25.194673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,0,2,4],"acting":[9,0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.16","version":"89'2","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055922+0000","last_change":"2024-04-25T00:30:47.522213+0000","last_active":"2024-04-25T00:33:18.055922+0000","last_peered":"2024-04-25T00:33:18.055922+0000","last_clean":"2024-04-25T00:33:18.055922+0000","last_became_active":"2024-04-25T00:30:47.521519+0000","last_became_peered":"2024-04-25T00:30:47.521519+0000","last_unstale":"2024-04-25T00:33:18.055922+0000","last_undegraded":"2024-04-25T00:33:18.055922+0000","last_fullsized":"2024-04-25T00:33:18.055922+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:16:10.001222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"91'9","reported_seq":47,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.306004+0000","last_change":"2024-04-25T00:31:45.029866+0000","last_active":"2024-04-25T00:33:18.306004+0000","last_peered":"2024-04-25T00:33:18.306004+0000","last_clean":"2024-04-25T00:33:18.306004+0000","last_became_active":"2024-04-25T00:31:45.029587+0000","last_became_peered":"2024-04-25T00:31:45.029587+0000","last_unstale":"2024-04-25T00:33:18.306004+0000","last_undegraded":"2024-04-25T00:33:18.306004+0000","last_fullsized":"2024-04-25T00:33:18.306004+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:09:09.434492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926486+0000","last_change":"2024-04-25T00:30:52.786818+0000","last_active":"2024-04-25T00:33:17.926486+0000","last_peered":"2024-04-25T00:33:17.926486+0000","last_clean":"2024-04-25T00:33:17.926486+0000","last_became_active":"2024-04-25T00:30:52.786293+0000","last_became_peered":"2024-04-25T00:30:52.786293+0000","last_unstale":"2024-04-25T00:33:17.926486+0000","last_undegraded":"2024-04-25T00:33:17.926486+0000","last_fullsized":"2024-04-25T00:33:17.926486+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:27:02.751296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2,9],"acting":[4,1,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.15","version":"89'4","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.037256+0000","last_change":"2024-04-25T00:30:47.520882+0000","last_active":"2024-04-25T00:33:18.037256+0000","last_peered":"2024-04-25T00:33:18.037256+0000","last_clean":"2024-04-25T00:33:18.037256+0000","last_became_active":"2024-04-25T00:30:47.520545+0000","last_became_peered":"2024-04-25T00:30:47.520545+0000","last_unstale":"2024-04-25T00:33:18.037256+0000","last_undegraded":"2024-04-25T00:33:18.037256+0000","last_fullsized":"2024-04-25T00:33:18.037256+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:46:04.715580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"91'11","reported_seq":43,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.187850+0000","last_change":"2024-04-25T00:30:48.536814+0000","last_active":"2024-04-25T00:33:18.187850+0000","last_peered":"2024-04-25T00:33:18.187850+0000","last_clean":"2024-04-25T00:33:18.187850+0000","last_became_active":"2024-04-25T00:30:48.536486+0000","last_became_peered":"2024-04-25T00:30:48.536486+0000","last_unstale":"2024-04-25T00:33:18.187850+0000","last_undegraded":"2024-04-25T00:33:18.187850+0000","last_fullsized":"2024-04-25T00:33:18.187850+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:58:23.861959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":18,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.040101+0000","last_change":"2024-04-25T00:31:45.031208+0000","last_active":"2024-04-25T00:33:18.040101+0000","last_peered":"2024-04-25T00:33:18.040101+0000","last_clean":"2024-04-25T00:33:18.040101+0000","last_became_active":"2024-04-25T00:31:45.030831+0000","last_became_peered":"2024-04-25T00:31:45.030831+0000","last_unstale":"2024-04-25T00:33:18.040101+0000","last_undegraded":"2024-04-25T00:33:18.040101+0000","last_fullsized":"2024-04-25T00:33:18.040101+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:42:01.869146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,11,5],"acting":[10,8,11,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.14","version":"94'39","reported_seq":331,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.187222+0000","last_change":"2024-04-25T00:30:47.526899+0000","last_active":"2024-04-25T00:33:18.187222+0000","last_peered":"2024-04-25T00:33:18.187222+0000","last_clean":"2024-04-25T00:33:18.187222+0000","last_became_active":"2024-04-25T00:30:47.526528+0000","last_became_peered":"2024-04-25T00:30:47.526528+0000","last_unstale":"2024-04-25T00:33:18.187222+0000","last_undegraded":"2024-04-25T00:33:18.187222+0000","last_fullsized":"2024-04-25T00:33:18.187222+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":39,"log_dups_size":0,"ondisk_log_size":39,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:13:34.004584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67428,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":258,"num_read_kb":195,"num_write":40,"num_write_kb":86,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"91'6","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.030904+0000","last_change":"2024-04-25T00:30:48.538225+0000","last_active":"2024-04-25T00:33:18.030904+0000","last_peered":"2024-04-25T00:33:18.030904+0000","last_clean":"2024-04-25T00:33:18.030904+0000","last_became_active":"2024-04-25T00:30:48.537807+0000","last_became_peered":"2024-04-25T00:30:48.537807+0000","last_unstale":"2024-04-25T00:33:18.030904+0000","last_undegraded":"2024-04-25T00:33:18.030904+0000","last_fullsized":"2024-04-25T00:33:18.030904+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:00:46.275584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.037669+0000","last_change":"2024-04-25T00:30:52.782266+0000","last_active":"2024-04-25T00:33:18.037669+0000","last_peered":"2024-04-25T00:33:18.037669+0000","last_clean":"2024-04-25T00:33:18.037669+0000","last_became_active":"2024-04-25T00:30:52.781133+0000","last_became_peered":"2024-04-25T00:30:52.781133+0000","last_unstale":"2024-04-25T00:33:18.037669+0000","last_undegraded":"2024-04-25T00:33:18.037669+0000","last_fullsized":"2024-04-25T00:33:18.037669+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:19:03.818560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,8,3,6],"acting":[5,8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.13","version":"89'6","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926797+0000","last_change":"2024-04-25T00:30:47.520909+0000","last_active":"2024-04-25T00:33:17.926797+0000","last_peered":"2024-04-25T00:33:17.926797+0000","last_clean":"2024-04-25T00:33:17.926797+0000","last_became_active":"2024-04-25T00:30:47.520287+0000","last_became_peered":"2024-04-25T00:30:47.520287+0000","last_unstale":"2024-04-25T00:33:17.926797+0000","last_undegraded":"2024-04-25T00:33:17.926797+0000","last_fullsized":"2024-04-25T00:33:17.926797+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:49:43.795115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"91'9","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.167123+0000","last_change":"2024-04-25T00:30:48.532270+0000","last_active":"2024-04-25T00:33:18.167123+0000","last_peered":"2024-04-25T00:33:18.167123+0000","last_clean":"2024-04-25T00:33:18.167123+0000","last_became_active":"2024-04-25T00:30:48.531882+0000","last_became_peered":"2024-04-25T00:30:48.531882+0000","last_unstale":"2024-04-25T00:33:18.167123+0000","last_undegraded":"2024-04-25T00:33:18.167123+0000","last_fullsized":"2024-04-25T00:33:18.167123+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:58:17.865635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.304924+0000","last_change":"2024-04-25T00:30:52.777708+0000","last_active":"2024-04-25T00:33:18.304924+0000","last_peered":"2024-04-25T00:33:18.304924+0000","last_clean":"2024-04-25T00:33:18.304924+0000","last_became_active":"2024-04-25T00:30:52.777254+0000","last_became_peered":"2024-04-25T00:30:52.777254+0000","last_unstale":"2024-04-25T00:33:18.304924+0000","last_undegraded":"2024-04-25T00:33:18.304924+0000","last_fullsized":"2024-04-25T00:33:18.304924+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:29:47.427190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,9,7,6],"acting":[11,9,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.12","version":"89'3","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.037312+0000","last_change":"2024-04-25T00:30:47.524667+0000","last_active":"2024-04-25T00:33:18.037312+0000","last_peered":"2024-04-25T00:33:18.037312+0000","last_clean":"2024-04-25T00:33:18.037312+0000","last_became_active":"2024-04-25T00:30:47.524361+0000","last_became_peered":"2024-04-25T00:30:47.524361+0000","last_unstale":"2024-04-25T00:33:18.037312+0000","last_undegraded":"2024-04-25T00:33:18.037312+0000","last_fullsized":"2024-04-25T00:33:18.037312+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:16:08.805720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"91'15","reported_seq":47,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055606+0000","last_change":"2024-04-25T00:30:48.538658+0000","last_active":"2024-04-25T00:33:18.055606+0000","last_peered":"2024-04-25T00:33:18.055606+0000","last_clean":"2024-04-25T00:33:18.055606+0000","last_became_active":"2024-04-25T00:30:48.538316+0000","last_became_peered":"2024-04-25T00:30:48.538316+0000","last_unstale":"2024-04-25T00:33:18.055606+0000","last_undegraded":"2024-04-25T00:33:18.055606+0000","last_fullsized":"2024-04-25T00:33:18.055606+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:24:47.303559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":24,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.028189+0000","last_change":"2024-04-25T00:30:52.789512+0000","last_active":"2024-04-25T00:33:18.028189+0000","last_peered":"2024-04-25T00:33:18.028189+0000","last_clean":"2024-04-25T00:33:18.028189+0000","last_became_active":"2024-04-25T00:30:52.788929+0000","last_became_peered":"2024-04-25T00:30:52.788929+0000","last_unstale":"2024-04-25T00:33:18.028189+0000","last_undegraded":"2024-04-25T00:33:18.028189+0000","last_fullsized":"2024-04-25T00:33:18.028189+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:21:05.195296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,0,3],"acting":[9,7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.11","version":"94'8","reported_seq":171,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.185195+0000","last_change":"2024-04-25T00:30:47.513627+0000","last_active":"2024-04-25T00:33:18.185195+0000","last_peered":"2024-04-25T00:33:18.185195+0000","last_clean":"2024-04-25T00:33:18.185195+0000","last_became_active":"2024-04-25T00:30:47.513215+0000","last_became_peered":"2024-04-25T00:30:47.513215+0000","last_unstale":"2024-04-25T00:33:18.185195+0000","last_undegraded":"2024-04-25T00:33:18.185195+0000","last_fullsized":"2024-04-25T00:33:18.185195+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:32:16.111590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2210,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":130,"num_read_kb":6,"num_write":7,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"91'6","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055536+0000","last_change":"2024-04-25T00:30:48.533056+0000","last_active":"2024-04-25T00:33:18.055536+0000","last_peered":"2024-04-25T00:33:18.055536+0000","last_clean":"2024-04-25T00:33:18.055536+0000","last_became_active":"2024-04-25T00:30:48.532698+0000","last_became_peered":"2024-04-25T00:30:48.532698+0000","last_unstale":"2024-04-25T00:33:18.055536+0000","last_undegraded":"2024-04-25T00:33:18.055536+0000","last_fullsized":"2024-04-25T00:33:18.055536+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:36:24.987784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.167620+0000","last_change":"2024-04-25T00:30:52.780669+0000","last_active":"2024-04-25T00:33:18.167620+0000","last_peered":"2024-04-25T00:33:18.167620+0000","last_clean":"2024-04-25T00:33:18.167620+0000","last_became_active":"2024-04-25T00:30:52.780222+0000","last_became_peered":"2024-04-25T00:30:52.780222+0000","last_unstale":"2024-04-25T00:33:18.167620+0000","last_undegraded":"2024-04-25T00:33:18.167620+0000","last_fullsized":"2024-04-25T00:33:18.167620+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:50:27.415077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,9,11],"acting":[6,3,9,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"89'15","reported_seq":74,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.030496+0000","last_change":"2024-04-25T00:32:45.486412+0000","last_active":"2024-04-25T00:33:18.030496+0000","last_peered":"2024-04-25T00:33:18.030496+0000","last_clean":"2024-04-25T00:33:18.030496+0000","last_became_active":"2024-04-25T00:32:45.474176+0000","last_became_peered":"2024-04-25T00:32:45.474176+0000","last_unstale":"2024-04-25T00:33:18.030496+0000","last_undegraded":"2024-04-25T00:33:18.030496+0000","last_fullsized":"2024-04-25T00:33:18.030496+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:59:50.867374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":112,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,4],"acting":[8,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"91'6","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.037133+0000","last_change":"2024-04-25T00:30:48.534363+0000","last_active":"2024-04-25T00:33:18.037133+0000","last_peered":"2024-04-25T00:33:18.037133+0000","last_clean":"2024-04-25T00:33:18.037133+0000","last_became_active":"2024-04-25T00:30:48.533855+0000","last_became_peered":"2024-04-25T00:30:48.533855+0000","last_unstale":"2024-04-25T00:33:18.037133+0000","last_undegraded":"2024-04-25T00:33:18.037133+0000","last_fullsized":"2024-04-25T00:33:18.037133+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:28:29.374501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.037011+0000","last_change":"2024-04-25T00:30:52.790323+0000","last_active":"2024-04-25T00:33:18.037011+0000","last_peered":"2024-04-25T00:33:18.037011+0000","last_clean":"2024-04-25T00:33:18.037011+0000","last_became_active":"2024-04-25T00:30:52.789905+0000","last_became_peered":"2024-04-25T00:30:52.789905+0000","last_unstale":"2024-04-25T00:33:18.037011+0000","last_undegraded":"2024-04-25T00:33:18.037011+0000","last_fullsized":"2024-04-25T00:33:18.037011+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:19:03.687318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3,1],"acting":[5,0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"93'3","reported_seq":20,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305099+0000","last_change":"2024-04-25T00:32:45.490186+0000","last_active":"2024-04-25T00:33:18.305099+0000","last_peered":"2024-04-25T00:33:18.305099+0000","last_clean":"2024-04-25T00:33:18.305099+0000","last_became_active":"2024-04-25T00:32:45.474069+0000","last_became_peered":"2024-04-25T00:32:45.474069+0000","last_unstale":"2024-04-25T00:33:18.305099+0000","last_undegraded":"2024-04-25T00:33:18.305099+0000","last_fullsized":"2024-04-25T00:33:18.305099+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:58:34.104225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.e","version":"91'5","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305056+0000","last_change":"2024-04-25T00:30:48.539679+0000","last_active":"2024-04-25T00:33:18.305056+0000","last_peered":"2024-04-25T00:33:18.305056+0000","last_clean":"2024-04-25T00:33:18.305056+0000","last_became_active":"2024-04-25T00:30:48.538065+0000","last_became_peered":"2024-04-25T00:30:48.538065+0000","last_unstale":"2024-04-25T00:33:18.305056+0000","last_undegraded":"2024-04-25T00:33:18.305056+0000","last_fullsized":"2024-04-25T00:33:18.305056+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:05:16.435010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.030891+0000","last_change":"2024-04-25T00:30:52.804824+0000","last_active":"2024-04-25T00:33:18.030891+0000","last_peered":"2024-04-25T00:33:18.030891+0000","last_clean":"2024-04-25T00:33:18.030891+0000","last_became_active":"2024-04-25T00:30:52.804322+0000","last_became_peered":"2024-04-25T00:30:52.804322+0000","last_unstale":"2024-04-25T00:33:18.030891+0000","last_undegraded":"2024-04-25T00:33:18.030891+0000","last_fullsized":"2024-04-25T00:33:18.030891+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:38:08.385170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,10,11,9],"acting":[8,10,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":34,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.027849+0000","last_change":"2024-04-25T00:30:47.516916+0000","last_active":"2024-04-25T00:33:18.027849+0000","last_peered":"2024-04-25T00:33:18.027849+0000","last_clean":"2024-04-25T00:33:18.027849+0000","last_became_active":"2024-04-25T00:30:47.516520+0000","last_became_peered":"2024-04-25T00:30:47.516520+0000","last_unstale":"2024-04-25T00:33:18.027849+0000","last_undegraded":"2024-04-25T00:33:18.027849+0000","last_fullsized":"2024-04-25T00:33:18.027849+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:29:19.099764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"91'4","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.040501+0000","last_change":"2024-04-25T00:30:48.537584+0000","last_active":"2024-04-25T00:33:18.040501+0000","last_peered":"2024-04-25T00:33:18.040501+0000","last_clean":"2024-04-25T00:33:18.040501+0000","last_became_active":"2024-04-25T00:30:48.537193+0000","last_became_peered":"2024-04-25T00:30:48.537193+0000","last_unstale":"2024-04-25T00:33:18.040501+0000","last_undegraded":"2024-04-25T00:33:18.040501+0000","last_fullsized":"2024-04-25T00:33:18.040501+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:15:22.075323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.166777+0000","last_change":"2024-04-25T00:30:52.784802+0000","last_active":"2024-04-25T00:33:18.166777+0000","last_peered":"2024-04-25T00:33:18.166777+0000","last_clean":"2024-04-25T00:33:18.166777+0000","last_became_active":"2024-04-25T00:30:52.784359+0000","last_became_peered":"2024-04-25T00:30:52.784359+0000","last_unstale":"2024-04-25T00:33:18.166777+0000","last_undegraded":"2024-04-25T00:33:18.166777+0000","last_fullsized":"2024-04-25T00:33:18.166777+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:19:57.723542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1,8],"acting":[6,0,1,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"89'1","reported_seq":35,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055523+0000","last_change":"2024-04-25T00:30:47.525494+0000","last_active":"2024-04-25T00:33:18.055523+0000","last_peered":"2024-04-25T00:33:18.055523+0000","last_clean":"2024-04-25T00:33:18.055523+0000","last_became_active":"2024-04-25T00:30:47.525158+0000","last_became_peered":"2024-04-25T00:30:47.525158+0000","last_unstale":"2024-04-25T00:33:18.055523+0000","last_undegraded":"2024-04-25T00:33:18.055523+0000","last_fullsized":"2024-04-25T00:33:18.055523+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:51:18.543352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"91'7","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.040217+0000","last_change":"2024-04-25T00:30:48.540926+0000","last_active":"2024-04-25T00:33:18.040217+0000","last_peered":"2024-04-25T00:33:18.040217+0000","last_clean":"2024-04-25T00:33:18.040217+0000","last_became_active":"2024-04-25T00:30:48.540204+0000","last_became_peered":"2024-04-25T00:30:48.540204+0000","last_unstale":"2024-04-25T00:33:18.040217+0000","last_undegraded":"2024-04-25T00:33:18.040217+0000","last_fullsized":"2024-04-25T00:33:18.040217+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:02:45.530266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.167067+0000","last_change":"2024-04-25T00:30:52.805780+0000","last_active":"2024-04-25T00:33:18.167067+0000","last_peered":"2024-04-25T00:33:18.167067+0000","last_clean":"2024-04-25T00:33:18.167067+0000","last_became_active":"2024-04-25T00:30:52.805222+0000","last_became_peered":"2024-04-25T00:30:52.805222+0000","last_unstale":"2024-04-25T00:33:18.167067+0000","last_undegraded":"2024-04-25T00:33:18.167067+0000","last_fullsized":"2024-04-25T00:33:18.167067+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:32:17.034630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,10,0],"acting":[6,5,10,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.0","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.037047+0000","last_change":"2024-04-25T00:30:48.534335+0000","last_active":"2024-04-25T00:33:18.037047+0000","last_peered":"2024-04-25T00:33:18.037047+0000","last_clean":"2024-04-25T00:33:18.037047+0000","last_became_active":"2024-04-25T00:30:48.532670+0000","last_became_peered":"2024-04-25T00:30:48.532670+0000","last_unstale":"2024-04-25T00:33:18.037047+0000","last_undegraded":"2024-04-25T00:33:18.037047+0000","last_fullsized":"2024-04-25T00:33:18.037047+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:26:23.145918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"93'6","reported_seq":164,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.166569+0000","last_change":"2024-04-25T00:30:47.527695+0000","last_active":"2024-04-25T00:33:18.166569+0000","last_peered":"2024-04-25T00:33:18.166569+0000","last_clean":"2024-04-25T00:33:18.166569+0000","last_became_active":"2024-04-25T00:30:47.527239+0000","last_became_peered":"2024-04-25T00:30:47.527239+0000","last_unstale":"2024-04-25T00:33:18.166569+0000","last_undegraded":"2024-04-25T00:33:18.166569+0000","last_fullsized":"2024-04-25T00:33:18.166569+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:54:30.911284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":124,"num_read_kb":124,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"89'1","reported_seq":26,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055558+0000","last_change":"2024-04-25T00:30:52.781091+0000","last_active":"2024-04-25T00:33:18.055558+0000","last_peered":"2024-04-25T00:33:18.055558+0000","last_clean":"2024-04-25T00:33:18.055558+0000","last_became_active":"2024-04-25T00:30:52.780538+0000","last_became_peered":"2024-04-25T00:30:52.780538+0000","last_unstale":"2024-04-25T00:33:18.055558+0000","last_undegraded":"2024-04-25T00:33:18.055558+0000","last_fullsized":"2024-04-25T00:33:18.055558+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:49:18.141802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,11,9],"acting":[7,5,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.924882+0000","last_change":"2024-04-25T00:30:48.538730+0000","last_active":"2024-04-25T00:33:17.924882+0000","last_peered":"2024-04-25T00:33:17.924882+0000","last_clean":"2024-04-25T00:33:17.924882+0000","last_became_active":"2024-04-25T00:30:48.538336+0000","last_became_peered":"2024-04-25T00:30:48.538336+0000","last_unstale":"2024-04-25T00:33:17.924882+0000","last_undegraded":"2024-04-25T00:33:17.924882+0000","last_fullsized":"2024-04-25T00:33:17.924882+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:44:22.370448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"89'3","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926327+0000","last_change":"2024-04-25T00:30:47.520450+0000","last_active":"2024-04-25T00:33:17.926327+0000","last_peered":"2024-04-25T00:33:17.926327+0000","last_clean":"2024-04-25T00:33:17.926327+0000","last_became_active":"2024-04-25T00:30:47.520073+0000","last_became_peered":"2024-04-25T00:30:47.520073+0000","last_unstale":"2024-04-25T00:33:17.926327+0000","last_undegraded":"2024-04-25T00:33:17.926327+0000","last_fullsized":"2024-04-25T00:33:17.926327+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:49:53.570421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":4,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"22'137","reported_seq":367,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055359+0000","last_change":"2024-04-25T00:29:34.720552+0000","last_active":"2024-04-25T00:33:18.055359+0000","last_peered":"2024-04-25T00:33:18.055359+0000","last_clean":"2024-04-25T00:33:18.055359+0000","last_became_active":"2024-04-25T00:29:34.720041+0000","last_became_peered":"2024-04-25T00:29:34.720041+0000","last_unstale":"2024-04-25T00:33:18.055359+0000","last_undegraded":"2024-04-25T00:33:18.055359+0000","last_fullsized":"2024-04-25T00:33:18.055359+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:26:07.691582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:26:07.691582+0000","last_clean_scrub_stamp":"2024-04-25T00:26:07.691582+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:15:27.995162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.037382+0000","last_change":"2024-04-25T00:30:52.805489+0000","last_active":"2024-04-25T00:33:18.037382+0000","last_peered":"2024-04-25T00:33:18.037382+0000","last_clean":"2024-04-25T00:33:18.037382+0000","last_became_active":"2024-04-25T00:30:52.804758+0000","last_became_peered":"2024-04-25T00:30:52.804758+0000","last_unstale":"2024-04-25T00:33:18.037382+0000","last_undegraded":"2024-04-25T00:33:18.037382+0000","last_fullsized":"2024-04-25T00:33:18.037382+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:17:31.325548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,6,10],"acting":[5,9,6,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1","version":"91'5","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.166495+0000","last_change":"2024-04-25T00:30:48.537699+0000","last_active":"2024-04-25T00:33:18.166495+0000","last_peered":"2024-04-25T00:33:18.166495+0000","last_clean":"2024-04-25T00:33:18.166495+0000","last_became_active":"2024-04-25T00:30:48.537291+0000","last_became_peered":"2024-04-25T00:30:48.537291+0000","last_unstale":"2024-04-25T00:33:18.166495+0000","last_undegraded":"2024-04-25T00:33:18.166495+0000","last_fullsized":"2024-04-25T00:33:18.166495+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:25:36.816499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"89'3","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055470+0000","last_change":"2024-04-25T00:30:47.524694+0000","last_active":"2024-04-25T00:33:18.055470+0000","last_peered":"2024-04-25T00:33:18.055470+0000","last_clean":"2024-04-25T00:33:18.055470+0000","last_became_active":"2024-04-25T00:30:47.524359+0000","last_became_peered":"2024-04-25T00:30:47.524359+0000","last_unstale":"2024-04-25T00:33:18.055470+0000","last_undegraded":"2024-04-25T00:33:18.055470+0000","last_fullsized":"2024-04-25T00:33:18.055470+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:55:16.677234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.304812+0000","last_change":"2024-04-25T00:30:52.780364+0000","last_active":"2024-04-25T00:33:18.304812+0000","last_peered":"2024-04-25T00:33:18.304812+0000","last_clean":"2024-04-25T00:33:18.304812+0000","last_became_active":"2024-04-25T00:30:52.779904+0000","last_became_peered":"2024-04-25T00:30:52.779904+0000","last_unstale":"2024-04-25T00:33:18.304812+0000","last_undegraded":"2024-04-25T00:33:18.304812+0000","last_fullsized":"2024-04-25T00:33:18.304812+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:11:26.738526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,7,6],"acting":[11,5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.5","version":"89'1","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.030818+0000","last_change":"2024-04-25T00:30:47.525205+0000","last_active":"2024-04-25T00:33:18.030818+0000","last_peered":"2024-04-25T00:33:18.030818+0000","last_clean":"2024-04-25T00:33:18.030818+0000","last_became_active":"2024-04-25T00:30:47.517729+0000","last_became_peered":"2024-04-25T00:30:47.517729+0000","last_unstale":"2024-04-25T00:33:18.030818+0000","last_undegraded":"2024-04-25T00:33:18.030818+0000","last_fullsized":"2024-04-25T00:33:18.030818+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:21:14.810119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"91'9","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.166967+0000","last_change":"2024-04-25T00:30:48.533822+0000","last_active":"2024-04-25T00:33:18.166967+0000","last_peered":"2024-04-25T00:33:18.166967+0000","last_clean":"2024-04-25T00:33:18.166967+0000","last_became_active":"2024-04-25T00:30:48.533479+0000","last_became_peered":"2024-04-25T00:30:48.533479+0000","last_unstale":"2024-04-25T00:33:18.166967+0000","last_undegraded":"2024-04-25T00:33:18.166967+0000","last_fullsized":"2024-04-25T00:33:18.166967+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:49:01.028124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.924543+0000","last_change":"2024-04-25T00:30:52.804770+0000","last_active":"2024-04-25T00:33:17.924543+0000","last_peered":"2024-04-25T00:33:17.924543+0000","last_clean":"2024-04-25T00:33:17.924543+0000","last_became_active":"2024-04-25T00:30:52.804242+0000","last_became_peered":"2024-04-25T00:30:52.804242+0000","last_unstale":"2024-04-25T00:33:17.924543+0000","last_undegraded":"2024-04-25T00:33:17.924543+0000","last_fullsized":"2024-04-25T00:33:17.924543+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:34:11.761478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,10,0,7],"acting":[3,10,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"94'44","reported_seq":101,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.144934+0000","last_change":"2024-04-25T00:30:47.521364+0000","last_active":"2024-04-25T00:33:18.144934+0000","last_peered":"2024-04-25T00:33:18.144934+0000","last_clean":"2024-04-25T00:33:18.144934+0000","last_became_active":"2024-04-25T00:30:47.521032+0000","last_became_peered":"2024-04-25T00:30:47.521032+0000","last_unstale":"2024-04-25T00:33:18.144934+0000","last_undegraded":"2024-04-25T00:33:18.144934+0000","last_fullsized":"2024-04-25T00:33:18.144934+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":44,"log_dups_size":0,"ondisk_log_size":44,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:33:26.156568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393216,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":82,"num_read_kb":70,"num_write":78,"num_write_kb":836,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,8,3],"acting":[9,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"91'5","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.925363+0000","last_change":"2024-04-25T00:30:48.532942+0000","last_active":"2024-04-25T00:33:17.925363+0000","last_peered":"2024-04-25T00:33:17.925363+0000","last_clean":"2024-04-25T00:33:17.925363+0000","last_became_active":"2024-04-25T00:30:48.532580+0000","last_became_peered":"2024-04-25T00:30:48.532580+0000","last_unstale":"2024-04-25T00:33:17.925363+0000","last_undegraded":"2024-04-25T00:33:17.925363+0000","last_fullsized":"2024-04-25T00:33:17.925363+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:42:22.887920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926476+0000","last_change":"2024-04-25T00:30:52.779757+0000","last_active":"2024-04-25T00:33:17.926476+0000","last_peered":"2024-04-25T00:33:17.926476+0000","last_clean":"2024-04-25T00:33:17.926476+0000","last_became_active":"2024-04-25T00:30:52.779324+0000","last_became_peered":"2024-04-25T00:30:52.779324+0000","last_unstale":"2024-04-25T00:33:17.926476+0000","last_undegraded":"2024-04-25T00:33:17.926476+0000","last_fullsized":"2024-04-25T00:33:17.926476+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:22:39.062000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4,7],"acting":[2,1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"89'4","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926614+0000","last_change":"2024-04-25T00:30:47.518212+0000","last_active":"2024-04-25T00:33:17.926614+0000","last_peered":"2024-04-25T00:33:17.926614+0000","last_clean":"2024-04-25T00:33:17.926614+0000","last_became_active":"2024-04-25T00:30:47.517610+0000","last_became_peered":"2024-04-25T00:30:47.517610+0000","last_unstale":"2024-04-25T00:33:17.926614+0000","last_undegraded":"2024-04-25T00:33:17.926614+0000","last_fullsized":"2024-04-25T00:33:17.926614+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:13:58.904020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"91'7","reported_seq":45,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305983+0000","last_change":"2024-04-25T00:31:45.028411+0000","last_active":"2024-04-25T00:33:18.305983+0000","last_peered":"2024-04-25T00:33:18.305983+0000","last_clean":"2024-04-25T00:33:18.305983+0000","last_became_active":"2024-04-25T00:31:45.028066+0000","last_became_peered":"2024-04-25T00:31:45.028066+0000","last_unstale":"2024-04-25T00:33:18.305983+0000","last_undegraded":"2024-04-25T00:33:18.305983+0000","last_fullsized":"2024-04-25T00:33:18.305983+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:33:46.556862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926345+0000","last_change":"2024-04-25T00:30:52.804915+0000","last_active":"2024-04-25T00:33:17.926345+0000","last_peered":"2024-04-25T00:33:17.926345+0000","last_clean":"2024-04-25T00:33:17.926345+0000","last_became_active":"2024-04-25T00:30:52.804491+0000","last_became_peered":"2024-04-25T00:30:52.804491+0000","last_unstale":"2024-04-25T00:33:17.926345+0000","last_undegraded":"2024-04-25T00:33:17.926345+0000","last_fullsized":"2024-04-25T00:33:17.926345+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:51:50.219556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,10,9],"acting":[4,5,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.8","version":"89'2","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.187520+0000","last_change":"2024-04-25T00:30:47.521909+0000","last_active":"2024-04-25T00:33:18.187520+0000","last_peered":"2024-04-25T00:33:18.187520+0000","last_clean":"2024-04-25T00:33:18.187520+0000","last_became_active":"2024-04-25T00:30:47.521551+0000","last_became_peered":"2024-04-25T00:30:47.521551+0000","last_unstale":"2024-04-25T00:33:18.187520+0000","last_undegraded":"2024-04-25T00:33:18.187520+0000","last_fullsized":"2024-04-25T00:33:18.187520+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:51:16.368609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"91'2","reported_seq":34,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.030469+0000","last_change":"2024-04-25T00:30:48.530514+0000","last_active":"2024-04-25T00:33:18.030469+0000","last_peered":"2024-04-25T00:33:18.030469+0000","last_clean":"2024-04-25T00:33:18.030469+0000","last_became_active":"2024-04-25T00:30:48.530129+0000","last_became_peered":"2024-04-25T00:30:48.530129+0000","last_unstale":"2024-04-25T00:33:18.030469+0000","last_undegraded":"2024-04-25T00:33:18.030469+0000","last_fullsized":"2024-04-25T00:33:18.030469+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:51:25.623186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.028315+0000","last_change":"2024-04-25T00:30:52.785767+0000","last_active":"2024-04-25T00:33:18.028315+0000","last_peered":"2024-04-25T00:33:18.028315+0000","last_clean":"2024-04-25T00:33:18.028315+0000","last_became_active":"2024-04-25T00:30:52.785301+0000","last_became_peered":"2024-04-25T00:30:52.785301+0000","last_unstale":"2024-04-25T00:33:18.028315+0000","last_undegraded":"2024-04-25T00:33:18.028315+0000","last_fullsized":"2024-04-25T00:33:18.028315+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:10:14.071733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,2,5],"acting":[9,3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.9","version":"89'3","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.924598+0000","last_change":"2024-04-25T00:30:47.526364+0000","last_active":"2024-04-25T00:33:17.924598+0000","last_peered":"2024-04-25T00:33:17.924598+0000","last_clean":"2024-04-25T00:33:17.924598+0000","last_became_active":"2024-04-25T00:30:47.526009+0000","last_became_peered":"2024-04-25T00:30:47.526009+0000","last_unstale":"2024-04-25T00:33:17.924598+0000","last_undegraded":"2024-04-25T00:33:17.924598+0000","last_fullsized":"2024-04-25T00:33:17.924598+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:17:38.471878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"91'9","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.304758+0000","last_change":"2024-04-25T00:30:48.532283+0000","last_active":"2024-04-25T00:33:18.304758+0000","last_peered":"2024-04-25T00:33:18.304758+0000","last_clean":"2024-04-25T00:33:18.304758+0000","last_became_active":"2024-04-25T00:30:48.531935+0000","last_became_peered":"2024-04-25T00:30:48.531935+0000","last_unstale":"2024-04-25T00:33:18.304758+0000","last_undegraded":"2024-04-25T00:33:18.304758+0000","last_fullsized":"2024-04-25T00:33:18.304758+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:01:15.965611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.c","version":"89'1","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926179+0000","last_change":"2024-04-25T00:31:44.717218+0000","last_active":"2024-04-25T00:33:17.926179+0000","last_peered":"2024-04-25T00:33:17.926179+0000","last_clean":"2024-04-25T00:33:17.926179+0000","last_became_active":"2024-04-25T00:31:44.712770+0000","last_became_peered":"2024-04-25T00:31:44.712770+0000","last_unstale":"2024-04-25T00:33:17.926179+0000","last_undegraded":"2024-04-25T00:33:17.926179+0000","last_fullsized":"2024-04-25T00:33:17.926179+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:06:58.496133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":117,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,7,8],"acting":[4,10,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.a","version":"89'1","reported_seq":35,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926611+0000","last_change":"2024-04-25T00:30:47.520608+0000","last_active":"2024-04-25T00:33:17.926611+0000","last_peered":"2024-04-25T00:33:17.926611+0000","last_clean":"2024-04-25T00:33:17.926611+0000","last_became_active":"2024-04-25T00:30:47.520063+0000","last_became_peered":"2024-04-25T00:30:47.520063+0000","last_unstale":"2024-04-25T00:33:17.926611+0000","last_undegraded":"2024-04-25T00:33:17.926611+0000","last_fullsized":"2024-04-25T00:33:17.926611+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:58:47.823712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.925247+0000","last_change":"2024-04-25T00:30:48.535073+0000","last_active":"2024-04-25T00:33:17.925247+0000","last_peered":"2024-04-25T00:33:17.925247+0000","last_clean":"2024-04-25T00:33:17.925247+0000","last_became_active":"2024-04-25T00:30:48.534247+0000","last_became_peered":"2024-04-25T00:30:48.534247+0000","last_unstale":"2024-04-25T00:33:17.925247+0000","last_undegraded":"2024-04-25T00:33:17.925247+0000","last_fullsized":"2024-04-25T00:33:17.925247+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:46:18.613705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926144+0000","last_change":"2024-04-25T00:30:52.804519+0000","last_active":"2024-04-25T00:33:17.926144+0000","last_peered":"2024-04-25T00:33:17.926144+0000","last_clean":"2024-04-25T00:33:17.926144+0000","last_became_active":"2024-04-25T00:30:52.804074+0000","last_became_peered":"2024-04-25T00:30:52.804074+0000","last_unstale":"2024-04-25T00:33:17.926144+0000","last_undegraded":"2024-04-25T00:33:17.926144+0000","last_fullsized":"2024-04-25T00:33:17.926144+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:48:45.058873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,10,3],"acting":[4,2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.b","version":"89'4","reported_seq":19,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.304778+0000","last_change":"2024-04-25T00:32:45.477574+0000","last_active":"2024-04-25T00:33:18.304778+0000","last_peered":"2024-04-25T00:33:18.304778+0000","last_clean":"2024-04-25T00:33:18.304778+0000","last_became_active":"2024-04-25T00:32:45.472622+0000","last_became_peered":"2024-04-25T00:32:45.472622+0000","last_unstale":"2024-04-25T00:33:18.304778+0000","last_undegraded":"2024-04-25T00:33:18.304778+0000","last_fullsized":"2024-04-25T00:33:18.304778+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:18:56.072904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,7],"acting":[11,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.a","version":"91'5","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055542+0000","last_change":"2024-04-25T00:30:48.537152+0000","last_active":"2024-04-25T00:33:18.055542+0000","last_peered":"2024-04-25T00:33:18.055542+0000","last_clean":"2024-04-25T00:33:18.055542+0000","last_became_active":"2024-04-25T00:30:48.536256+0000","last_became_peered":"2024-04-25T00:30:48.536256+0000","last_unstale":"2024-04-25T00:33:18.055542+0000","last_undegraded":"2024-04-25T00:33:18.055542+0000","last_fullsized":"2024-04-25T00:33:18.055542+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:57:53.870212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.187504+0000","last_change":"2024-04-25T00:30:52.781022+0000","last_active":"2024-04-25T00:33:18.187504+0000","last_peered":"2024-04-25T00:33:18.187504+0000","last_clean":"2024-04-25T00:33:18.187504+0000","last_became_active":"2024-04-25T00:30:52.780595+0000","last_became_peered":"2024-04-25T00:30:52.780595+0000","last_unstale":"2024-04-25T00:33:18.187504+0000","last_undegraded":"2024-04-25T00:33:18.187504+0000","last_fullsized":"2024-04-25T00:33:18.187504+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:27:02.580870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6,7],"acting":[1,5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.c","version":"89'3","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926971+0000","last_change":"2024-04-25T00:30:47.519118+0000","last_active":"2024-04-25T00:33:17.926971+0000","last_peered":"2024-04-25T00:33:17.926971+0000","last_clean":"2024-04-25T00:33:17.926971+0000","last_became_active":"2024-04-25T00:30:47.518639+0000","last_became_peered":"2024-04-25T00:30:47.518639+0000","last_unstale":"2024-04-25T00:33:17.926971+0000","last_undegraded":"2024-04-25T00:33:17.926971+0000","last_fullsized":"2024-04-25T00:33:17.926971+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:32:29.747903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"91'7","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055812+0000","last_change":"2024-04-25T00:30:48.542073+0000","last_active":"2024-04-25T00:33:18.055812+0000","last_peered":"2024-04-25T00:33:18.055812+0000","last_clean":"2024-04-25T00:33:18.055812+0000","last_became_active":"2024-04-25T00:30:48.541722+0000","last_became_peered":"2024-04-25T00:30:48.541722+0000","last_unstale":"2024-04-25T00:33:18.055812+0000","last_undegraded":"2024-04-25T00:33:18.055812+0000","last_fullsized":"2024-04-25T00:33:18.055812+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:34:03.397539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305021+0000","last_change":"2024-04-25T00:30:52.790234+0000","last_active":"2024-04-25T00:33:18.305021+0000","last_peered":"2024-04-25T00:33:18.305021+0000","last_clean":"2024-04-25T00:33:18.305021+0000","last_became_active":"2024-04-25T00:30:52.789788+0000","last_became_peered":"2024-04-25T00:30:52.789788+0000","last_unstale":"2024-04-25T00:33:18.305021+0000","last_undegraded":"2024-04-25T00:33:18.305021+0000","last_fullsized":"2024-04-25T00:33:18.305021+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:44:43.399771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,9,1,4],"acting":[0,9,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.d","version":"94'39","reported_seq":81,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.142633+0000","last_change":"2024-04-25T00:30:47.523597+0000","last_active":"2024-04-25T00:33:18.142633+0000","last_peered":"2024-04-25T00:33:18.142633+0000","last_clean":"2024-04-25T00:33:18.142633+0000","last_became_active":"2024-04-25T00:30:47.523225+0000","last_became_peered":"2024-04-25T00:30:47.523225+0000","last_unstale":"2024-04-25T00:33:18.142633+0000","last_undegraded":"2024-04-25T00:33:18.142633+0000","last_fullsized":"2024-04-25T00:33:18.142633+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":39,"log_dups_size":0,"ondisk_log_size":39,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:51:12.341015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131670,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":11,"num_read_kb":10,"num_write":49,"num_write_kb":475,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"91'6","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055778+0000","last_change":"2024-04-25T00:30:48.541417+0000","last_active":"2024-04-25T00:33:18.055778+0000","last_peered":"2024-04-25T00:33:18.055778+0000","last_clean":"2024-04-25T00:33:18.055778+0000","last_became_active":"2024-04-25T00:30:48.541015+0000","last_became_peered":"2024-04-25T00:30:48.541015+0000","last_unstale":"2024-04-25T00:33:18.055778+0000","last_undegraded":"2024-04-25T00:33:18.055778+0000","last_fullsized":"2024-04-25T00:33:18.055778+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:21:35.299823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.036816+0000","last_change":"2024-04-25T00:30:52.782230+0000","last_active":"2024-04-25T00:33:18.036816+0000","last_peered":"2024-04-25T00:33:18.036816+0000","last_clean":"2024-04-25T00:33:18.036816+0000","last_became_active":"2024-04-25T00:30:52.781708+0000","last_became_peered":"2024-04-25T00:30:52.781708+0000","last_unstale":"2024-04-25T00:33:18.036816+0000","last_undegraded":"2024-04-25T00:33:18.036816+0000","last_fullsized":"2024-04-25T00:33:18.036816+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:58:53.260985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7,1],"acting":[5,2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.e","version":"84'2","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.167506+0000","last_change":"2024-04-25T00:32:45.553977+0000","last_active":"2024-04-25T00:33:18.167506+0000","last_peered":"2024-04-25T00:33:18.167506+0000","last_clean":"2024-04-25T00:33:18.167506+0000","last_became_active":"2024-04-25T00:32:45.551280+0000","last_became_peered":"2024-04-25T00:32:45.551280+0000","last_unstale":"2024-04-25T00:33:18.167506+0000","last_undegraded":"2024-04-25T00:33:18.167506+0000","last_fullsized":"2024-04-25T00:33:18.167506+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:54:12.307459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"91'9","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055986+0000","last_change":"2024-04-25T00:30:48.532730+0000","last_active":"2024-04-25T00:33:18.055986+0000","last_peered":"2024-04-25T00:33:18.055986+0000","last_clean":"2024-04-25T00:33:18.055986+0000","last_became_active":"2024-04-25T00:30:48.532129+0000","last_became_peered":"2024-04-25T00:30:48.532129+0000","last_unstale":"2024-04-25T00:33:18.055986+0000","last_undegraded":"2024-04-25T00:33:18.055986+0000","last_fullsized":"2024-04-25T00:33:18.055986+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:36:30.115125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"91'9","reported_seq":47,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.304840+0000","last_change":"2024-04-25T00:31:44.712173+0000","last_active":"2024-04-25T00:33:18.304840+0000","last_peered":"2024-04-25T00:33:18.304840+0000","last_clean":"2024-04-25T00:33:18.304840+0000","last_became_active":"2024-04-25T00:31:44.711738+0000","last_became_peered":"2024-04-25T00:31:44.711738+0000","last_unstale":"2024-04-25T00:33:18.304840+0000","last_undegraded":"2024-04-25T00:33:18.304840+0000","last_fullsized":"2024-04-25T00:33:18.304840+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:06:59.466566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,9],"acting":[11,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"91'12","reported_seq":44,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926692+0000","last_change":"2024-04-25T00:30:48.540676+0000","last_active":"2024-04-25T00:33:17.926692+0000","last_peered":"2024-04-25T00:33:17.926692+0000","last_clean":"2024-04-25T00:33:17.926692+0000","last_became_active":"2024-04-25T00:30:48.540258+0000","last_became_peered":"2024-04-25T00:30:48.540258+0000","last_unstale":"2024-04-25T00:33:17.926692+0000","last_undegraded":"2024-04-25T00:33:17.926692+0000","last_fullsized":"2024-04-25T00:33:17.926692+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:10:29.930897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926531+0000","last_change":"2024-04-25T00:30:48.542191+0000","last_active":"2024-04-25T00:33:17.926531+0000","last_peered":"2024-04-25T00:33:17.926531+0000","last_clean":"2024-04-25T00:33:17.926531+0000","last_became_active":"2024-04-25T00:30:48.540433+0000","last_became_peered":"2024-04-25T00:30:48.540433+0000","last_unstale":"2024-04-25T00:33:17.926531+0000","last_undegraded":"2024-04-25T00:33:17.926531+0000","last_fullsized":"2024-04-25T00:33:17.926531+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:21:55.694555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"91'7","reported_seq":45,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305410+0000","last_change":"2024-04-25T00:31:45.028445+0000","last_active":"2024-04-25T00:33:18.305410+0000","last_peered":"2024-04-25T00:33:18.305410+0000","last_clean":"2024-04-25T00:33:18.305410+0000","last_became_active":"2024-04-25T00:31:45.028048+0000","last_became_peered":"2024-04-25T00:31:45.028048+0000","last_unstale":"2024-04-25T00:33:18.305410+0000","last_undegraded":"2024-04-25T00:33:18.305410+0000","last_fullsized":"2024-04-25T00:33:18.305410+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:04:02.714111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,8],"acting":[0,6,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"91'4","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.167135+0000","last_change":"2024-04-25T00:30:48.527892+0000","last_active":"2024-04-25T00:33:18.167135+0000","last_peered":"2024-04-25T00:33:18.167135+0000","last_clean":"2024-04-25T00:33:18.167135+0000","last_became_active":"2024-04-25T00:30:48.527522+0000","last_became_peered":"2024-04-25T00:30:48.527522+0000","last_unstale":"2024-04-25T00:33:18.167135+0000","last_undegraded":"2024-04-25T00:33:18.167135+0000","last_fullsized":"2024-04-25T00:33:18.167135+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:21:22.779470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"91'6","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305133+0000","last_change":"2024-04-25T00:30:48.535066+0000","last_active":"2024-04-25T00:33:18.305133+0000","last_peered":"2024-04-25T00:33:18.305133+0000","last_clean":"2024-04-25T00:33:18.305133+0000","last_became_active":"2024-04-25T00:30:48.534710+0000","last_became_peered":"2024-04-25T00:30:48.534710+0000","last_unstale":"2024-04-25T00:33:18.305133+0000","last_undegraded":"2024-04-25T00:33:18.305133+0000","last_fullsized":"2024-04-25T00:33:18.305133+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:57:18.582569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"91'15","reported_seq":47,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.927199+0000","last_change":"2024-04-25T00:30:48.541921+0000","last_active":"2024-04-25T00:33:17.927199+0000","last_peered":"2024-04-25T00:33:17.927199+0000","last_clean":"2024-04-25T00:33:17.927199+0000","last_became_active":"2024-04-25T00:30:48.541348+0000","last_became_peered":"2024-04-25T00:30:48.541348+0000","last_unstale":"2024-04-25T00:33:17.927199+0000","last_undegraded":"2024-04-25T00:33:17.927199+0000","last_fullsized":"2024-04-25T00:33:17.927199+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:07:00.001504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"91'2","reported_seq":34,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.925569+0000","last_change":"2024-04-25T00:30:48.536902+0000","last_active":"2024-04-25T00:33:17.925569+0000","last_peered":"2024-04-25T00:33:17.925569+0000","last_clean":"2024-04-25T00:33:17.925569+0000","last_became_active":"2024-04-25T00:30:48.536008+0000","last_became_peered":"2024-04-25T00:30:48.536008+0000","last_unstale":"2024-04-25T00:33:17.925569+0000","last_undegraded":"2024-04-25T00:33:17.925569+0000","last_fullsized":"2024-04-25T00:33:17.925569+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:19:15.050142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"91'6","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.036972+0000","last_change":"2024-04-25T00:30:48.533104+0000","last_active":"2024-04-25T00:33:18.036972+0000","last_peered":"2024-04-25T00:33:18.036972+0000","last_clean":"2024-04-25T00:33:18.036972+0000","last_became_active":"2024-04-25T00:30:48.532124+0000","last_became_peered":"2024-04-25T00:30:48.532124+0000","last_unstale":"2024-04-25T00:33:18.036972+0000","last_undegraded":"2024-04-25T00:33:18.036972+0000","last_fullsized":"2024-04-25T00:33:18.036972+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:54:57.391433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.924561+0000","last_change":"2024-04-25T00:30:48.542074+0000","last_active":"2024-04-25T00:33:17.924561+0000","last_peered":"2024-04-25T00:33:17.924561+0000","last_clean":"2024-04-25T00:33:17.924561+0000","last_became_active":"2024-04-25T00:30:48.538752+0000","last_became_peered":"2024-04-25T00:30:48.538752+0000","last_unstale":"2024-04-25T00:33:17.924561+0000","last_undegraded":"2024-04-25T00:33:17.924561+0000","last_fullsized":"2024-04-25T00:33:17.924561+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:55:30.613616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"91'10","reported_seq":42,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.187829+0000","last_change":"2024-04-25T00:30:48.534158+0000","last_active":"2024-04-25T00:33:18.187829+0000","last_peered":"2024-04-25T00:33:18.187829+0000","last_clean":"2024-04-25T00:33:18.187829+0000","last_became_active":"2024-04-25T00:30:48.533821+0000","last_became_peered":"2024-04-25T00:30:48.533821+0000","last_unstale":"2024-04-25T00:33:18.187829+0000","last_undegraded":"2024-04-25T00:33:18.187829+0000","last_fullsized":"2024-04-25T00:33:18.187829+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:05:19.327608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"91'11","reported_seq":43,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.167481+0000","last_change":"2024-04-25T00:30:48.531718+0000","last_active":"2024-04-25T00:33:18.167481+0000","last_peered":"2024-04-25T00:33:18.167481+0000","last_clean":"2024-04-25T00:33:18.167481+0000","last_became_active":"2024-04-25T00:30:48.531291+0000","last_became_peered":"2024-04-25T00:30:48.531291+0000","last_unstale":"2024-04-25T00:33:18.167481+0000","last_undegraded":"2024-04-25T00:33:18.167481+0000","last_fullsized":"2024-04-25T00:33:18.167481+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:56:49.032937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"91'8","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.304987+0000","last_change":"2024-04-25T00:31:44.713036+0000","last_active":"2024-04-25T00:33:18.304987+0000","last_peered":"2024-04-25T00:33:18.304987+0000","last_clean":"2024-04-25T00:33:18.304987+0000","last_became_active":"2024-04-25T00:31:44.712651+0000","last_became_peered":"2024-04-25T00:31:44.712651+0000","last_unstale":"2024-04-25T00:33:18.304987+0000","last_undegraded":"2024-04-25T00:33:18.304987+0000","last_fullsized":"2024-04-25T00:33:18.304987+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:05:04.811510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,9,8],"acting":[11,9,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":234,"num_objects":2,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":117,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":32768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2,"ondisk_log_size":2,"up":128,"acting":128,"num_store_stats":12},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":500,"ondisk_log_size":500,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":597867,"num_objects":62,"num_object_clones":0,"num_object_copies":186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":62,"num_whiteouts":0,"num_read":864,"num_read_kb":540,"num_write":289,"num_write_kb":1441,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":134,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2027520,"data_stored":1793601,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":229,"ondisk_log_size":229,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":80,"seq":343597383723,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32232,"kb_used_data":1052,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712920,"statfs":{"total":95995035648,"available":95962030080,"internally_reserved":0,"allocated":1077248,"data_stored":529717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1596,"internal_metadata":31914436},"hb_peers":[0,1,2,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":10,"up_from":72,"seq":309237645360,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28880,"kb_used_data":1864,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716272,"statfs":{"total":95995035648,"available":95965462528,"internally_reserved":0,"allocated":1908736,"data_stored":1321709,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1600,"internal_metadata":27654592},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":9,"up_from":67,"seq":287762808885,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36788,"kb_used_data":1448,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708364,"statfs":{"total":95995035648,"available":95957364736,"internally_reserved":0,"allocated":1482752,"data_stored":923147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1600,"internal_metadata":36174272},"hb_peers":[0,1,2,3,4,5,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":8,"up_from":59,"seq":253403070522,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32620,"kb_used_data":1444,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712532,"statfs":{"total":95995035648,"available":95961632768,"internally_reserved":0,"allocated":1478656,"data_stored":923045,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":7,"up_from":51,"seq":219043332159,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36984,"kb_used_data":1644,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708168,"statfs":{"total":95995035648,"available":95957164032,"internally_reserved":0,"allocated":1683456,"data_stored":1127382,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":36174282},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495685,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36384,"kb_used_data":1044,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708768,"statfs":{"total":95995035648,"available":95957778432,"internally_reserved":0,"allocated":1069056,"data_stored":525107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":36174278},"hb_peers":[0,1,2,3,4,5,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574940,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28116,"kb_used_data":1100,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717036,"statfs":{"total":95995035648,"available":95966244864,"internally_reserved":0,"allocated":1126400,"data_stored":581739,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738467,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28076,"kb_used_data":1052,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717076,"statfs":{"total":95995035648,"available":95966285824,"internally_reserved":0,"allocated":1077248,"data_stored":516571,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":18,"seq":77309411416,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28152,"kb_used_data":1132,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717000,"statfs":{"total":95995035648,"available":95966208000,"internally_reserved":0,"allocated":1159168,"data_stored":594074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":25,"seq":107374182483,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36592,"kb_used_data":1252,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708560,"statfs":{"total":95995035648,"available":95957565440,"internally_reserved":0,"allocated":1282048,"data_stored":722421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":36174279},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":33,"seq":141733920847,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28084,"kb_used_data":1064,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717068,"statfs":{"total":95995035648,"available":95966277632,"internally_reserved":0,"allocated":1089536,"data_stored":530709,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757322,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28800,"kb_used_data":1784,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716352,"statfs":{"total":95995035648,"available":95965544448,"internally_reserved":0,"allocated":1826816,"data_stored":1252948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":77824,"data_stored":67428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":94208,"data_stored":69057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":544768,"data_stored":525100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":2378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":163840,"data_stored":133467,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":3805,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":409600,"data_stored":393932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":394034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":245760,"data_stored":202228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":8192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-25T00:33:20.582 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-04-25T00:33:20.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.722+0000 7f176a8bd640 1 -- 172.21.15.32:0/3962079933 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1764075720 msgr2=0x7f1764073b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:20.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.722+0000 7f176a8bd640 1 --2- 172.21.15.32:0/3962079933 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1764075720 0x7f1764073b70 secure :-1 s=READY pgs=485 cs=0 l=1 rev1=1 crypto rx=0x7f1754009900 tx=0x7f175402a480 comp rx=0 tx=0).stop 2024-04-25T00:33:20.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.722+0000 7f176a8bd640 1 -- 172.21.15.32:0/3962079933 shutdown_connections 2024-04-25T00:33:20.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.722+0000 7f176a8bd640 1 --2- 172.21.15.32:0/3962079933 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1764109640 0x7f176410bae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:20.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.722+0000 7f176a8bd640 1 --2- 172.21.15.32:0/3962079933 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1764074140 0x7f17640745a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:20.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.722+0000 7f176a8bd640 1 --2- 172.21.15.32:0/3962079933 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1764075720 0x7f1764073b70 unknown :-1 s=CLOSED pgs=485 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:20.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.722+0000 7f176a8bd640 1 -- 172.21.15.32:0/3962079933 >> 172.21.15.32:0/3962079933 conn(0x7f17640fcd40 msgr2=0x7f17640ff160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:20.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.722+0000 7f176a8bd640 1 -- 172.21.15.32:0/3962079933 shutdown_connections 2024-04-25T00:33:20.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.722+0000 7f176a8bd640 1 -- 172.21.15.32:0/3962079933 wait complete. 2024-04-25T00:33:20.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.722+0000 7f176a8bd640 1 Processor -- start 2024-04-25T00:33:20.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.723+0000 7f176a8bd640 1 -- start start 2024-04-25T00:33:20.726 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.723+0000 7f176a8bd640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1764074140 0x7f1764199c90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:20.726 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.723+0000 7f176a8bd640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1764075720 0x7f176419a1d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:20.726 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.723+0000 7f176a8bd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1764109640 0x7f17641a1230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:20.726 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.723+0000 7f176a8bd640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f176410f600 con 0x7f1764109640 2024-04-25T00:33:20.726 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.723+0000 7f176a8bd640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f176410f480 con 0x7f1764074140 2024-04-25T00:33:20.726 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.723+0000 7f176a8bd640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f176410f780 con 0x7f1764075720 2024-04-25T00:33:20.726 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.723+0000 7f1768e5a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1764109640 0x7f17641a1230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:20.726 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.723+0000 7f1768e5a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1764109640 0x7f17641a1230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44394/0 (socket says 172.21.15.32:44394) 2024-04-25T00:33:20.726 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.723+0000 7f1768e5a640 1 -- 172.21.15.32:0/1529654349 learned_addr learned my addr 172.21.15.32:0/1529654349 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:20.726 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.723+0000 7f1763fff640 1 --2- 172.21.15.32:0/1529654349 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1764074140 0x7f1764199c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:20.726 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.723+0000 7f17637fe640 1 --2- 172.21.15.32:0/1529654349 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1764075720 0x7f176419a1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:20.726 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.723+0000 7f1768e5a640 1 -- 172.21.15.32:0/1529654349 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1764075720 msgr2=0x7f176419a1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:20.727 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.723+0000 7f1768e5a640 1 --2- 172.21.15.32:0/1529654349 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1764075720 0x7f176419a1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:20.727 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.723+0000 7f1768e5a640 1 -- 172.21.15.32:0/1529654349 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1764074140 msgr2=0x7f1764199c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:20.727 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.723+0000 7f1768e5a640 1 --2- 172.21.15.32:0/1529654349 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1764074140 0x7f1764199c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:20.727 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.723+0000 7f1768e5a640 1 -- 172.21.15.32:0/1529654349 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f17641a18d0 con 0x7f1764109640 2024-04-25T00:33:20.727 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.723+0000 7f1763fff640 1 --2- 172.21.15.32:0/1529654349 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1764074140 0x7f1764199c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:20.727 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.724+0000 7f1768e5a640 1 --2- 172.21.15.32:0/1529654349 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1764109640 0x7f17641a1230 secure :-1 s=READY pgs=486 cs=0 l=1 rev1=1 crypto rx=0x7f175c009fd0 tx=0x7f175c00b260 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:20.727 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.724+0000 7f17617fa640 1 -- 172.21.15.32:0/1529654349 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f175c017070 con 0x7f1764109640 2024-04-25T00:33:20.727 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.724+0000 7f176a8bd640 1 -- 172.21.15.32:0/1529654349 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f17641a1bc0 con 0x7f1764109640 2024-04-25T00:33:20.727 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.724+0000 7f17617fa640 1 -- 172.21.15.32:0/1529654349 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f175c003e50 con 0x7f1764109640 2024-04-25T00:33:20.727 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.724+0000 7f17617fa640 1 -- 172.21.15.32:0/1529654349 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f175c0106c0 con 0x7f1764109640 2024-04-25T00:33:20.727 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.724+0000 7f176a8bd640 1 -- 172.21.15.32:0/1529654349 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f17641a2110 con 0x7f1764109640 2024-04-25T00:33:20.727 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.725+0000 7f176a8bd640 1 -- 172.21.15.32:0/1529654349 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1764108400 con 0x7f1764109640 2024-04-25T00:33:20.728 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.727+0000 7f17617fa640 1 -- 172.21.15.32:0/1529654349 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 23) ==== 137953+0+0 (secure 0 0 0) 0x7f175c003970 con 0x7f1764109640 2024-04-25T00:33:20.728 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.727+0000 7f17617fa640 1 --2- 172.21.15.32:0/1529654349 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f174c0a2d00 0x7f174c0a51c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:20.728 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.728+0000 7f17617fa640 1 -- 172.21.15.32:0/1529654349 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f175c0cf4f0 con 0x7f1764109640 2024-04-25T00:33:20.728 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.728+0000 7f1763fff640 1 --2- 172.21.15.32:0/1529654349 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f174c0a2d00 0x7f174c0a51c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:20.729 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.729+0000 7f1763fff640 1 --2- 172.21.15.32:0/1529654349 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f174c0a2d00 0x7f174c0a51c0 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f176419ab60 tx=0x7f1754000bc0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:20.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.732+0000 7f17617fa640 1 -- 172.21.15.32:0/1529654349 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f175c09b710 con 0x7f1764109640 2024-04-25T00:33:20.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:20 smithi121 ceph-mon[32231]: [25/Apr/2024:00:33:18] ENGINE Serving on http://172.21.15.32:8765 2024-04-25T00:33:20.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:20 smithi121 ceph-mon[32231]: [25/Apr/2024:00:33:18] ENGINE Serving on https://172.21.15.32:7150 2024-04-25T00:33:20.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:20 smithi121 ceph-mon[32231]: [25/Apr/2024:00:33:18] ENGINE Bus STARTED 2024-04-25T00:33:20.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:20 smithi121 ceph-mon[32231]: pgmap v3: 129 pgs: 129 active+clean; 1.1 MiB data, 373 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:33:20.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:20 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1898524971' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:33:20.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:20 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2675413030' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:33:20.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:20 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:20.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:20 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:20.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:20 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:20.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:20 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:20.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:20 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/4162392750' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:33:20.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:20 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:20.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:20 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:20.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:20 smithi130 ceph-mon[32525]: [25/Apr/2024:00:33:18] ENGINE Serving on http://172.21.15.32:8765 2024-04-25T00:33:20.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:20 smithi130 ceph-mon[32525]: [25/Apr/2024:00:33:18] ENGINE Serving on https://172.21.15.32:7150 2024-04-25T00:33:20.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:20 smithi130 ceph-mon[32525]: [25/Apr/2024:00:33:18] ENGINE Bus STARTED 2024-04-25T00:33:20.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:20 smithi130 ceph-mon[32525]: pgmap v3: 129 pgs: 129 active+clean; 1.1 MiB data, 373 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:33:20.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:20 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1898524971' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:33:20.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:20 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2675413030' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:33:20.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:20 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:20.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:20 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:20.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:20 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:20.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:20 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:20.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:20 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/4162392750' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:33:20.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:20 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:20.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:20 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:20.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.915+0000 7f176a8bd640 1 -- 172.21.15.32:0/1529654349 --> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f1764061b80 con 0x7f174c0a2d00 2024-04-25T00:33:20.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.925+0000 7f17617fa640 1 -- 172.21.15.32:0/1529654349 <== mgr.15483 v2:172.21.15.32:6800/3307874911 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+339393 (secure 0 0 0) 0x7f1764061b80 con 0x7f174c0a2d00 2024-04-25T00:33:20.929 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:33:20.930 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-04-25T00:33:20.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.928+0000 7f176a8bd640 1 -- 172.21.15.32:0/1529654349 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f174c0a2d00 msgr2=0x7f174c0a51c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:20.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.928+0000 7f176a8bd640 1 --2- 172.21.15.32:0/1529654349 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f174c0a2d00 0x7f174c0a51c0 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f176419ab60 tx=0x7f1754000bc0 comp rx=0 tx=0).stop 2024-04-25T00:33:20.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.928+0000 7f176a8bd640 1 -- 172.21.15.32:0/1529654349 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1764109640 msgr2=0x7f17641a1230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:20.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.928+0000 7f176a8bd640 1 --2- 172.21.15.32:0/1529654349 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1764109640 0x7f17641a1230 secure :-1 s=READY pgs=486 cs=0 l=1 rev1=1 crypto rx=0x7f175c009fd0 tx=0x7f175c00b260 comp rx=0 tx=0).stop 2024-04-25T00:33:20.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.928+0000 7f176a8bd640 1 -- 172.21.15.32:0/1529654349 shutdown_connections 2024-04-25T00:33:20.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.928+0000 7f176a8bd640 1 --2- 172.21.15.32:0/1529654349 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f174c0a2d00 0x7f174c0a51c0 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:20.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.928+0000 7f176a8bd640 1 --2- 172.21.15.32:0/1529654349 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1764109640 0x7f17641a1230 unknown :-1 s=CLOSED pgs=486 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:20.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.928+0000 7f176a8bd640 1 --2- 172.21.15.32:0/1529654349 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1764075720 0x7f176419a1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:20.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.928+0000 7f176a8bd640 1 --2- 172.21.15.32:0/1529654349 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1764074140 0x7f1764199c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:20.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.928+0000 7f176a8bd640 1 -- 172.21.15.32:0/1529654349 >> 172.21.15.32:0/1529654349 conn(0x7f17640fcd40 msgr2=0x7f17641055d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:20.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.928+0000 7f176a8bd640 1 -- 172.21.15.32:0/1529654349 shutdown_connections 2024-04-25T00:33:20.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:20.928+0000 7f176a8bd640 1 -- 172.21.15.32:0/1529654349 wait complete. 2024-04-25T00:33:20.940 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":4,"stamp":"2024-04-25T00:33:19.938647+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1188469,"num_objects":66,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":66,"num_whiteouts":0,"num_read":1000,"num_read_kb":659,"num_write":472,"num_write_kb":4381,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":1180987,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":868,"ondisk_log_size":868,"up":419,"acting":419,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":419,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":381708,"kb_used_data":15880,"kb_used_omap":18,"kb_used_meta":365677,"kb_avail":1124560116,"statfs":{"total":1151940427776,"available":1151549558784,"internally_reserved":0,"allocated":16261120,"data_stored":9548569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19142,"internal_metadata":374453562},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.990637"},"pg_stats":[{"pgid":"3.3f","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305395+0000","last_change":"2024-04-25T00:30:48.538719+0000","last_active":"2024-04-25T00:33:18.305395+0000","last_peered":"2024-04-25T00:33:18.305395+0000","last_clean":"2024-04-25T00:33:18.305395+0000","last_became_active":"2024-04-25T00:30:48.538356+0000","last_became_peered":"2024-04-25T00:30:48.538356+0000","last_unstale":"2024-04-25T00:33:18.305395+0000","last_undegraded":"2024-04-25T00:33:18.305395+0000","last_fullsized":"2024-04-25T00:33:18.305395+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:39:08.974838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"91'4","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305901+0000","last_change":"2024-04-25T00:30:48.532262+0000","last_active":"2024-04-25T00:33:18.305901+0000","last_peered":"2024-04-25T00:33:18.305901+0000","last_clean":"2024-04-25T00:33:18.305901+0000","last_became_active":"2024-04-25T00:30:48.531926+0000","last_became_peered":"2024-04-25T00:30:48.531926+0000","last_unstale":"2024-04-25T00:33:18.305901+0000","last_undegraded":"2024-04-25T00:33:18.305901+0000","last_fullsized":"2024-04-25T00:33:18.305901+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:12:06.527649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"91'7","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305705+0000","last_change":"2024-04-25T00:30:48.536383+0000","last_active":"2024-04-25T00:33:18.305705+0000","last_peered":"2024-04-25T00:33:18.305705+0000","last_clean":"2024-04-25T00:33:18.305705+0000","last_became_active":"2024-04-25T00:30:48.536048+0000","last_became_peered":"2024-04-25T00:30:48.536048+0000","last_unstale":"2024-04-25T00:33:18.305705+0000","last_undegraded":"2024-04-25T00:33:18.305705+0000","last_fullsized":"2024-04-25T00:33:18.305705+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:32:26.728441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926366+0000","last_change":"2024-04-25T00:30:48.532195+0000","last_active":"2024-04-25T00:33:17.926366+0000","last_peered":"2024-04-25T00:33:17.926366+0000","last_clean":"2024-04-25T00:33:17.926366+0000","last_became_active":"2024-04-25T00:30:48.531851+0000","last_became_peered":"2024-04-25T00:30:48.531851+0000","last_unstale":"2024-04-25T00:33:17.926366+0000","last_undegraded":"2024-04-25T00:33:17.926366+0000","last_fullsized":"2024-04-25T00:33:17.926366+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:15:34.704958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"91'4","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.030694+0000","last_change":"2024-04-25T00:30:48.534024+0000","last_active":"2024-04-25T00:33:18.030694+0000","last_peered":"2024-04-25T00:33:18.030694+0000","last_clean":"2024-04-25T00:33:18.030694+0000","last_became_active":"2024-04-25T00:30:48.533681+0000","last_became_peered":"2024-04-25T00:30:48.533681+0000","last_unstale":"2024-04-25T00:33:18.030694+0000","last_undegraded":"2024-04-25T00:33:18.030694+0000","last_fullsized":"2024-04-25T00:33:18.030694+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:07:35.816816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"91'6","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.040282+0000","last_change":"2024-04-25T00:30:48.541263+0000","last_active":"2024-04-25T00:33:18.040282+0000","last_peered":"2024-04-25T00:33:18.040282+0000","last_clean":"2024-04-25T00:33:18.040282+0000","last_became_active":"2024-04-25T00:30:48.540889+0000","last_became_peered":"2024-04-25T00:30:48.540889+0000","last_unstale":"2024-04-25T00:33:18.040282+0000","last_undegraded":"2024-04-25T00:33:18.040282+0000","last_fullsized":"2024-04-25T00:33:18.040282+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:31:42.067004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,4],"acting":[10,9,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"91'5","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055606+0000","last_change":"2024-04-25T00:30:48.533194+0000","last_active":"2024-04-25T00:33:18.055606+0000","last_peered":"2024-04-25T00:33:18.055606+0000","last_clean":"2024-04-25T00:33:18.055606+0000","last_became_active":"2024-04-25T00:30:48.532424+0000","last_became_peered":"2024-04-25T00:30:48.532424+0000","last_unstale":"2024-04-25T00:33:18.055606+0000","last_undegraded":"2024-04-25T00:33:18.055606+0000","last_fullsized":"2024-04-25T00:33:18.055606+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:06:50.187796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"91'9","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.924700+0000","last_change":"2024-04-25T00:30:48.542829+0000","last_active":"2024-04-25T00:33:17.924700+0000","last_peered":"2024-04-25T00:33:17.924700+0000","last_clean":"2024-04-25T00:33:17.924700+0000","last_became_active":"2024-04-25T00:30:48.542204+0000","last_became_peered":"2024-04-25T00:30:48.542204+0000","last_unstale":"2024-04-25T00:33:17.924700+0000","last_undegraded":"2024-04-25T00:33:17.924700+0000","last_fullsized":"2024-04-25T00:33:17.924700+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:56:47.869075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"91'11","reported_seq":43,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.188149+0000","last_change":"2024-04-25T00:30:48.537924+0000","last_active":"2024-04-25T00:33:18.188149+0000","last_peered":"2024-04-25T00:33:18.188149+0000","last_clean":"2024-04-25T00:33:18.188149+0000","last_became_active":"2024-04-25T00:30:48.537547+0000","last_became_peered":"2024-04-25T00:30:48.537547+0000","last_unstale":"2024-04-25T00:33:18.188149+0000","last_undegraded":"2024-04-25T00:33:18.188149+0000","last_fullsized":"2024-04-25T00:33:18.188149+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:26:45.590923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"91'16","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.040641+0000","last_change":"2024-04-25T00:30:48.539717+0000","last_active":"2024-04-25T00:33:18.040641+0000","last_peered":"2024-04-25T00:33:18.040641+0000","last_clean":"2024-04-25T00:33:18.040641+0000","last_became_active":"2024-04-25T00:30:48.539242+0000","last_became_peered":"2024-04-25T00:30:48.539242+0000","last_unstale":"2024-04-25T00:33:18.040641+0000","last_undegraded":"2024-04-25T00:33:18.040641+0000","last_fullsized":"2024-04-25T00:33:18.040641+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":16,"log_dups_size":0,"ondisk_log_size":16,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:58:15.353959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"91'6","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055839+0000","last_change":"2024-04-25T00:30:48.542767+0000","last_active":"2024-04-25T00:33:18.055839+0000","last_peered":"2024-04-25T00:33:18.055839+0000","last_clean":"2024-04-25T00:33:18.055839+0000","last_became_active":"2024-04-25T00:30:48.542252+0000","last_became_peered":"2024-04-25T00:30:48.542252+0000","last_unstale":"2024-04-25T00:33:18.055839+0000","last_undegraded":"2024-04-25T00:33:18.055839+0000","last_fullsized":"2024-04-25T00:33:18.055839+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:07:49.414660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"91'7","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926388+0000","last_change":"2024-04-25T00:30:48.532248+0000","last_active":"2024-04-25T00:33:17.926388+0000","last_peered":"2024-04-25T00:33:17.926388+0000","last_clean":"2024-04-25T00:33:17.926388+0000","last_became_active":"2024-04-25T00:30:48.531876+0000","last_became_peered":"2024-04-25T00:30:48.531876+0000","last_unstale":"2024-04-25T00:33:17.926388+0000","last_undegraded":"2024-04-25T00:33:17.926388+0000","last_fullsized":"2024-04-25T00:33:17.926388+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:00:35.711693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"91'10","reported_seq":42,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.167693+0000","last_change":"2024-04-25T00:30:48.526871+0000","last_active":"2024-04-25T00:33:18.167693+0000","last_peered":"2024-04-25T00:33:18.167693+0000","last_clean":"2024-04-25T00:33:18.167693+0000","last_became_active":"2024-04-25T00:30:48.526477+0000","last_became_peered":"2024-04-25T00:30:48.526477+0000","last_unstale":"2024-04-25T00:33:18.167693+0000","last_undegraded":"2024-04-25T00:33:18.167693+0000","last_fullsized":"2024-04-25T00:33:18.167693+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:17:59.447932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"91'7","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305099+0000","last_change":"2024-04-25T00:30:48.530297+0000","last_active":"2024-04-25T00:33:18.305099+0000","last_peered":"2024-04-25T00:33:18.305099+0000","last_clean":"2024-04-25T00:33:18.305099+0000","last_became_active":"2024-04-25T00:30:48.529091+0000","last_became_peered":"2024-04-25T00:30:48.529091+0000","last_unstale":"2024-04-25T00:33:18.305099+0000","last_undegraded":"2024-04-25T00:33:18.305099+0000","last_fullsized":"2024-04-25T00:33:18.305099+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:54:52.206881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"91'7","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305119+0000","last_change":"2024-04-25T00:30:48.535629+0000","last_active":"2024-04-25T00:33:18.305119+0000","last_peered":"2024-04-25T00:33:18.305119+0000","last_clean":"2024-04-25T00:33:18.305119+0000","last_became_active":"2024-04-25T00:30:48.535190+0000","last_became_peered":"2024-04-25T00:30:48.535190+0000","last_unstale":"2024-04-25T00:33:18.305119+0000","last_undegraded":"2024-04-25T00:33:18.305119+0000","last_fullsized":"2024-04-25T00:33:18.305119+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:11:27.127964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"91'9","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.187509+0000","last_change":"2024-04-25T00:30:48.532988+0000","last_active":"2024-04-25T00:33:18.187509+0000","last_peered":"2024-04-25T00:33:18.187509+0000","last_clean":"2024-04-25T00:33:18.187509+0000","last_became_active":"2024-04-25T00:30:48.532670+0000","last_became_peered":"2024-04-25T00:30:48.532670+0000","last_unstale":"2024-04-25T00:33:18.187509+0000","last_undegraded":"2024-04-25T00:33:18.187509+0000","last_fullsized":"2024-04-25T00:33:18.187509+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:03:35.336175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305254+0000","last_change":"2024-04-25T00:30:48.539499+0000","last_active":"2024-04-25T00:33:18.305254+0000","last_peered":"2024-04-25T00:33:18.305254+0000","last_clean":"2024-04-25T00:33:18.305254+0000","last_became_active":"2024-04-25T00:30:48.538516+0000","last_became_peered":"2024-04-25T00:30:48.538516+0000","last_unstale":"2024-04-25T00:33:18.305254+0000","last_undegraded":"2024-04-25T00:33:18.305254+0000","last_fullsized":"2024-04-25T00:33:18.305254+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:03:16.781091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"91'10","reported_seq":42,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.028510+0000","last_change":"2024-04-25T00:30:48.534204+0000","last_active":"2024-04-25T00:33:18.028510+0000","last_peered":"2024-04-25T00:33:18.028510+0000","last_clean":"2024-04-25T00:33:18.028510+0000","last_became_active":"2024-04-25T00:30:48.533818+0000","last_became_peered":"2024-04-25T00:30:48.533818+0000","last_unstale":"2024-04-25T00:33:18.028510+0000","last_undegraded":"2024-04-25T00:33:18.028510+0000","last_fullsized":"2024-04-25T00:33:18.028510+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:08:42.964056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"91'7","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.927103+0000","last_change":"2024-04-25T00:30:48.542756+0000","last_active":"2024-04-25T00:33:17.927103+0000","last_peered":"2024-04-25T00:33:17.927103+0000","last_clean":"2024-04-25T00:33:17.927103+0000","last_became_active":"2024-04-25T00:30:48.541824+0000","last_became_peered":"2024-04-25T00:30:48.541824+0000","last_unstale":"2024-04-25T00:33:17.927103+0000","last_undegraded":"2024-04-25T00:33:17.927103+0000","last_fullsized":"2024-04-25T00:33:17.927103+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:31:40.996474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926670+0000","last_change":"2024-04-25T00:30:52.788477+0000","last_active":"2024-04-25T00:33:17.926670+0000","last_peered":"2024-04-25T00:33:17.926670+0000","last_clean":"2024-04-25T00:33:17.926670+0000","last_became_active":"2024-04-25T00:30:52.788073+0000","last_became_peered":"2024-04-25T00:30:52.788073+0000","last_unstale":"2024-04-25T00:33:17.926670+0000","last_undegraded":"2024-04-25T00:33:17.926670+0000","last_fullsized":"2024-04-25T00:33:17.926670+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:29:34.781119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,9,0],"acting":[2,3,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"89'9","reported_seq":44,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.028297+0000","last_change":"2024-04-25T00:30:47.524580+0000","last_active":"2024-04-25T00:33:18.028297+0000","last_peered":"2024-04-25T00:33:18.028297+0000","last_clean":"2024-04-25T00:33:18.028297+0000","last_became_active":"2024-04-25T00:30:47.524170+0000","last_became_peered":"2024-04-25T00:30:47.524170+0000","last_unstale":"2024-04-25T00:33:18.028297+0000","last_undegraded":"2024-04-25T00:33:18.028297+0000","last_fullsized":"2024-04-25T00:33:18.028297+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:53:08.734842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"91'5","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.925270+0000","last_change":"2024-04-25T00:30:48.542077+0000","last_active":"2024-04-25T00:33:17.925270+0000","last_peered":"2024-04-25T00:33:17.925270+0000","last_clean":"2024-04-25T00:33:17.925270+0000","last_became_active":"2024-04-25T00:30:48.539043+0000","last_became_peered":"2024-04-25T00:30:48.539043+0000","last_unstale":"2024-04-25T00:33:17.925270+0000","last_undegraded":"2024-04-25T00:33:17.925270+0000","last_fullsized":"2024-04-25T00:33:17.925270+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:48:52.254800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926432+0000","last_change":"2024-04-25T00:30:52.786091+0000","last_active":"2024-04-25T00:33:17.926432+0000","last_peered":"2024-04-25T00:33:17.926432+0000","last_clean":"2024-04-25T00:33:17.926432+0000","last_became_active":"2024-04-25T00:30:52.785511+0000","last_became_peered":"2024-04-25T00:30:52.785511+0000","last_unstale":"2024-04-25T00:33:17.926432+0000","last_undegraded":"2024-04-25T00:33:17.926432+0000","last_fullsized":"2024-04-25T00:33:17.926432+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:19:17.992033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,9,3],"acting":[4,6,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1e","version":"89'2","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.925488+0000","last_change":"2024-04-25T00:30:47.529293+0000","last_active":"2024-04-25T00:33:17.925488+0000","last_peered":"2024-04-25T00:33:17.925488+0000","last_clean":"2024-04-25T00:33:17.925488+0000","last_became_active":"2024-04-25T00:30:47.527975+0000","last_became_peered":"2024-04-25T00:30:47.527975+0000","last_unstale":"2024-04-25T00:33:17.925488+0000","last_undegraded":"2024-04-25T00:33:17.925488+0000","last_fullsized":"2024-04-25T00:33:17.925488+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:17:40.057962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"91'6","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305797+0000","last_change":"2024-04-25T00:30:48.530993+0000","last_active":"2024-04-25T00:33:18.305797+0000","last_peered":"2024-04-25T00:33:18.305797+0000","last_clean":"2024-04-25T00:33:18.305797+0000","last_became_active":"2024-04-25T00:30:48.530550+0000","last_became_peered":"2024-04-25T00:30:48.530550+0000","last_unstale":"2024-04-25T00:33:18.305797+0000","last_undegraded":"2024-04-25T00:33:18.305797+0000","last_fullsized":"2024-04-25T00:33:18.305797+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:11:35.308456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":24,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926274+0000","last_change":"2024-04-25T00:30:52.787774+0000","last_active":"2024-04-25T00:33:17.926274+0000","last_peered":"2024-04-25T00:33:17.926274+0000","last_clean":"2024-04-25T00:33:17.926274+0000","last_became_active":"2024-04-25T00:30:52.786988+0000","last_became_peered":"2024-04-25T00:30:52.786988+0000","last_unstale":"2024-04-25T00:33:17.926274+0000","last_undegraded":"2024-04-25T00:33:17.926274+0000","last_fullsized":"2024-04-25T00:33:17.926274+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:58:28.081767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,9,5],"acting":[4,3,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1d","version":"89'4","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.030776+0000","last_change":"2024-04-25T00:30:47.523477+0000","last_active":"2024-04-25T00:33:18.030776+0000","last_peered":"2024-04-25T00:33:18.030776+0000","last_clean":"2024-04-25T00:33:18.030776+0000","last_became_active":"2024-04-25T00:30:47.522961+0000","last_became_peered":"2024-04-25T00:30:47.522961+0000","last_unstale":"2024-04-25T00:33:18.030776+0000","last_undegraded":"2024-04-25T00:33:18.030776+0000","last_fullsized":"2024-04-25T00:33:18.030776+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:39:39.793477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"91'9","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.187779+0000","last_change":"2024-04-25T00:30:48.527746+0000","last_active":"2024-04-25T00:33:18.187779+0000","last_peered":"2024-04-25T00:33:18.187779+0000","last_clean":"2024-04-25T00:33:18.187779+0000","last_became_active":"2024-04-25T00:30:48.527374+0000","last_became_peered":"2024-04-25T00:30:48.527374+0000","last_unstale":"2024-04-25T00:33:18.187779+0000","last_undegraded":"2024-04-25T00:33:18.187779+0000","last_fullsized":"2024-04-25T00:33:18.187779+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:28:36.822220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926088+0000","last_change":"2024-04-25T00:30:52.782368+0000","last_active":"2024-04-25T00:33:17.926088+0000","last_peered":"2024-04-25T00:33:17.926088+0000","last_clean":"2024-04-25T00:33:17.926088+0000","last_became_active":"2024-04-25T00:30:52.781875+0000","last_became_peered":"2024-04-25T00:30:52.781875+0000","last_unstale":"2024-04-25T00:33:17.926088+0000","last_undegraded":"2024-04-25T00:33:17.926088+0000","last_fullsized":"2024-04-25T00:33:17.926088+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:07:55.509131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5,3],"acting":[4,6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1c","version":"84'2","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.031019+0000","last_change":"2024-04-25T00:30:47.523707+0000","last_active":"2024-04-25T00:33:18.031019+0000","last_peered":"2024-04-25T00:33:18.031019+0000","last_clean":"2024-04-25T00:33:18.031019+0000","last_became_active":"2024-04-25T00:30:47.523216+0000","last_became_peered":"2024-04-25T00:30:47.523216+0000","last_unstale":"2024-04-25T00:33:18.031019+0000","last_undegraded":"2024-04-25T00:33:18.031019+0000","last_fullsized":"2024-04-25T00:33:18.031019+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:21:21.006368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"91'15","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.037238+0000","last_change":"2024-04-25T00:30:48.529235+0000","last_active":"2024-04-25T00:33:18.037238+0000","last_peered":"2024-04-25T00:33:18.037238+0000","last_clean":"2024-04-25T00:33:18.037238+0000","last_became_active":"2024-04-25T00:30:48.528887+0000","last_became_peered":"2024-04-25T00:30:48.528887+0000","last_unstale":"2024-04-25T00:33:18.037238+0000","last_undegraded":"2024-04-25T00:33:18.037238+0000","last_fullsized":"2024-04-25T00:33:18.037238+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:18:05.290578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":31,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926986+0000","last_change":"2024-04-25T00:31:44.714433+0000","last_active":"2024-04-25T00:33:17.926986+0000","last_peered":"2024-04-25T00:33:17.926986+0000","last_clean":"2024-04-25T00:33:17.926986+0000","last_became_active":"2024-04-25T00:31:44.713810+0000","last_became_peered":"2024-04-25T00:31:44.713810+0000","last_unstale":"2024-04-25T00:33:17.926986+0000","last_undegraded":"2024-04-25T00:33:17.926986+0000","last_fullsized":"2024-04-25T00:33:17.926986+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:00:49.815134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,7,11],"acting":[2,8,7,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1b","version":"94'6","reported_seq":166,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.185075+0000","last_change":"2024-04-25T00:30:47.518472+0000","last_active":"2024-04-25T00:33:18.185075+0000","last_peered":"2024-04-25T00:33:18.185075+0000","last_clean":"2024-04-25T00:33:18.185075+0000","last_became_active":"2024-04-25T00:30:47.518140+0000","last_became_peered":"2024-04-25T00:30:47.518140+0000","last_unstale":"2024-04-25T00:33:18.185075+0000","last_undegraded":"2024-04-25T00:33:18.185075+0000","last_fullsized":"2024-04-25T00:33:18.185075+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:19:07.780949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1013,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":2,"num_write":3,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"91'7","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.030957+0000","last_change":"2024-04-25T00:30:48.533757+0000","last_active":"2024-04-25T00:33:18.030957+0000","last_peered":"2024-04-25T00:33:18.030957+0000","last_clean":"2024-04-25T00:33:18.030957+0000","last_became_active":"2024-04-25T00:30:48.533421+0000","last_became_peered":"2024-04-25T00:30:48.533421+0000","last_unstale":"2024-04-25T00:33:18.030957+0000","last_undegraded":"2024-04-25T00:33:18.030957+0000","last_fullsized":"2024-04-25T00:33:18.030957+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:11:38.190783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":31,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926818+0000","last_change":"2024-04-25T00:31:45.031412+0000","last_active":"2024-04-25T00:33:17.926818+0000","last_peered":"2024-04-25T00:33:17.926818+0000","last_clean":"2024-04-25T00:33:17.926818+0000","last_became_active":"2024-04-25T00:31:45.030822+0000","last_became_peered":"2024-04-25T00:31:45.030822+0000","last_unstale":"2024-04-25T00:33:17.926818+0000","last_undegraded":"2024-04-25T00:33:17.926818+0000","last_fullsized":"2024-04-25T00:33:17.926818+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:34:26.758966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,8,5],"acting":[2,6,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1a","version":"89'3","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.040284+0000","last_change":"2024-04-25T00:30:47.527306+0000","last_active":"2024-04-25T00:33:18.040284+0000","last_peered":"2024-04-25T00:33:18.040284+0000","last_clean":"2024-04-25T00:33:18.040284+0000","last_became_active":"2024-04-25T00:30:47.526935+0000","last_became_peered":"2024-04-25T00:30:47.526935+0000","last_unstale":"2024-04-25T00:33:18.040284+0000","last_undegraded":"2024-04-25T00:33:18.040284+0000","last_fullsized":"2024-04-25T00:33:18.040284+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:41:37.951143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"91'8","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.027475+0000","last_change":"2024-04-25T00:31:45.170610+0000","last_active":"2024-04-25T00:33:18.027475+0000","last_peered":"2024-04-25T00:33:18.027475+0000","last_clean":"2024-04-25T00:33:18.027475+0000","last_became_active":"2024-04-25T00:31:45.170183+0000","last_became_peered":"2024-04-25T00:31:45.170183+0000","last_unstale":"2024-04-25T00:33:18.027475+0000","last_undegraded":"2024-04-25T00:33:18.027475+0000","last_fullsized":"2024-04-25T00:33:18.027475+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:39:39.781997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,4,3],"acting":[9,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305164+0000","last_change":"2024-04-25T00:30:52.784986+0000","last_active":"2024-04-25T00:33:18.305164+0000","last_peered":"2024-04-25T00:33:18.305164+0000","last_clean":"2024-04-25T00:33:18.305164+0000","last_became_active":"2024-04-25T00:30:52.784593+0000","last_became_peered":"2024-04-25T00:30:52.784593+0000","last_unstale":"2024-04-25T00:33:18.305164+0000","last_undegraded":"2024-04-25T00:33:18.305164+0000","last_fullsized":"2024-04-25T00:33:18.305164+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:04:05.833730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,9,0],"acting":[11,4,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.19","version":"89'2","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.166738+0000","last_change":"2024-04-25T00:30:47.523663+0000","last_active":"2024-04-25T00:33:18.166738+0000","last_peered":"2024-04-25T00:33:18.166738+0000","last_clean":"2024-04-25T00:33:18.166738+0000","last_became_active":"2024-04-25T00:30:47.523324+0000","last_became_peered":"2024-04-25T00:30:47.523324+0000","last_unstale":"2024-04-25T00:33:18.166738+0000","last_undegraded":"2024-04-25T00:33:18.166738+0000","last_fullsized":"2024-04-25T00:33:18.166738+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:51:23.593579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.924628+0000","last_change":"2024-04-25T00:30:48.529738+0000","last_active":"2024-04-25T00:33:17.924628+0000","last_peered":"2024-04-25T00:33:17.924628+0000","last_clean":"2024-04-25T00:33:17.924628+0000","last_became_active":"2024-04-25T00:30:48.529350+0000","last_became_peered":"2024-04-25T00:30:48.529350+0000","last_unstale":"2024-04-25T00:33:17.924628+0000","last_undegraded":"2024-04-25T00:33:17.924628+0000","last_fullsized":"2024-04-25T00:33:17.924628+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:51:29.238293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305792+0000","last_change":"2024-04-25T00:30:52.788480+0000","last_active":"2024-04-25T00:33:18.305792+0000","last_peered":"2024-04-25T00:33:18.305792+0000","last_clean":"2024-04-25T00:33:18.305792+0000","last_became_active":"2024-04-25T00:30:52.788065+0000","last_became_peered":"2024-04-25T00:30:52.788065+0000","last_unstale":"2024-04-25T00:33:18.305792+0000","last_undegraded":"2024-04-25T00:33:18.305792+0000","last_fullsized":"2024-04-25T00:33:18.305792+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:16:50.179177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6,1],"acting":[0,7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.18","version":"89'1","reported_seq":35,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.036951+0000","last_change":"2024-04-25T00:30:47.524706+0000","last_active":"2024-04-25T00:33:18.036951+0000","last_peered":"2024-04-25T00:33:18.036951+0000","last_clean":"2024-04-25T00:33:18.036951+0000","last_became_active":"2024-04-25T00:30:47.524355+0000","last_became_peered":"2024-04-25T00:30:47.524355+0000","last_unstale":"2024-04-25T00:33:18.036951+0000","last_undegraded":"2024-04-25T00:33:18.036951+0000","last_fullsized":"2024-04-25T00:33:18.036951+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:22:38.093362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"91'11","reported_seq":43,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.027727+0000","last_change":"2024-04-25T00:30:48.533266+0000","last_active":"2024-04-25T00:33:18.027727+0000","last_peered":"2024-04-25T00:33:18.027727+0000","last_clean":"2024-04-25T00:33:18.027727+0000","last_became_active":"2024-04-25T00:30:48.532932+0000","last_became_peered":"2024-04-25T00:30:48.532932+0000","last_unstale":"2024-04-25T00:33:18.027727+0000","last_undegraded":"2024-04-25T00:33:18.027727+0000","last_fullsized":"2024-04-25T00:33:18.027727+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:00:54.147725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.924818+0000","last_change":"2024-04-25T00:30:52.780317+0000","last_active":"2024-04-25T00:33:17.924818+0000","last_peered":"2024-04-25T00:33:17.924818+0000","last_clean":"2024-04-25T00:33:17.924818+0000","last_became_active":"2024-04-25T00:30:52.778539+0000","last_became_peered":"2024-04-25T00:30:52.778539+0000","last_unstale":"2024-04-25T00:33:17.924818+0000","last_undegraded":"2024-04-25T00:33:17.924818+0000","last_fullsized":"2024-04-25T00:33:17.924818+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:34:49.527307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1,5],"acting":[3,7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.17","version":"94'4","reported_seq":162,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.185182+0000","last_change":"2024-04-25T00:30:47.525647+0000","last_active":"2024-04-25T00:33:18.185182+0000","last_peered":"2024-04-25T00:33:18.185182+0000","last_clean":"2024-04-25T00:33:18.185182+0000","last_became_active":"2024-04-25T00:30:47.525280+0000","last_became_peered":"2024-04-25T00:30:47.525280+0000","last_unstale":"2024-04-25T00:33:18.185182+0000","last_undegraded":"2024-04-25T00:33:18.185182+0000","last_fullsized":"2024-04-25T00:33:18.185182+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:56:15.761173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":124,"num_read_kb":124,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"91'10","reported_seq":42,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.188103+0000","last_change":"2024-04-25T00:30:48.527913+0000","last_active":"2024-04-25T00:33:18.188103+0000","last_peered":"2024-04-25T00:33:18.188103+0000","last_clean":"2024-04-25T00:33:18.188103+0000","last_became_active":"2024-04-25T00:30:48.527538+0000","last_became_peered":"2024-04-25T00:30:48.527538+0000","last_unstale":"2024-04-25T00:33:18.188103+0000","last_undegraded":"2024-04-25T00:33:18.188103+0000","last_fullsized":"2024-04-25T00:33:18.188103+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:48:38.646867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.027433+0000","last_change":"2024-04-25T00:30:52.788267+0000","last_active":"2024-04-25T00:33:18.027433+0000","last_peered":"2024-04-25T00:33:18.027433+0000","last_clean":"2024-04-25T00:33:18.027433+0000","last_became_active":"2024-04-25T00:30:52.785188+0000","last_became_peered":"2024-04-25T00:30:52.785188+0000","last_unstale":"2024-04-25T00:33:18.027433+0000","last_undegraded":"2024-04-25T00:33:18.027433+0000","last_fullsized":"2024-04-25T00:33:18.027433+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:38:25.194673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,0,2,4],"acting":[9,0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.16","version":"89'2","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055922+0000","last_change":"2024-04-25T00:30:47.522213+0000","last_active":"2024-04-25T00:33:18.055922+0000","last_peered":"2024-04-25T00:33:18.055922+0000","last_clean":"2024-04-25T00:33:18.055922+0000","last_became_active":"2024-04-25T00:30:47.521519+0000","last_became_peered":"2024-04-25T00:30:47.521519+0000","last_unstale":"2024-04-25T00:33:18.055922+0000","last_undegraded":"2024-04-25T00:33:18.055922+0000","last_fullsized":"2024-04-25T00:33:18.055922+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:16:10.001222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"91'9","reported_seq":47,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.306004+0000","last_change":"2024-04-25T00:31:45.029866+0000","last_active":"2024-04-25T00:33:18.306004+0000","last_peered":"2024-04-25T00:33:18.306004+0000","last_clean":"2024-04-25T00:33:18.306004+0000","last_became_active":"2024-04-25T00:31:45.029587+0000","last_became_peered":"2024-04-25T00:31:45.029587+0000","last_unstale":"2024-04-25T00:33:18.306004+0000","last_undegraded":"2024-04-25T00:33:18.306004+0000","last_fullsized":"2024-04-25T00:33:18.306004+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:09:09.434492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926486+0000","last_change":"2024-04-25T00:30:52.786818+0000","last_active":"2024-04-25T00:33:17.926486+0000","last_peered":"2024-04-25T00:33:17.926486+0000","last_clean":"2024-04-25T00:33:17.926486+0000","last_became_active":"2024-04-25T00:30:52.786293+0000","last_became_peered":"2024-04-25T00:30:52.786293+0000","last_unstale":"2024-04-25T00:33:17.926486+0000","last_undegraded":"2024-04-25T00:33:17.926486+0000","last_fullsized":"2024-04-25T00:33:17.926486+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:27:02.751296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2,9],"acting":[4,1,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.15","version":"89'4","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.037256+0000","last_change":"2024-04-25T00:30:47.520882+0000","last_active":"2024-04-25T00:33:18.037256+0000","last_peered":"2024-04-25T00:33:18.037256+0000","last_clean":"2024-04-25T00:33:18.037256+0000","last_became_active":"2024-04-25T00:30:47.520545+0000","last_became_peered":"2024-04-25T00:30:47.520545+0000","last_unstale":"2024-04-25T00:33:18.037256+0000","last_undegraded":"2024-04-25T00:33:18.037256+0000","last_fullsized":"2024-04-25T00:33:18.037256+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:46:04.715580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"91'11","reported_seq":43,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.187850+0000","last_change":"2024-04-25T00:30:48.536814+0000","last_active":"2024-04-25T00:33:18.187850+0000","last_peered":"2024-04-25T00:33:18.187850+0000","last_clean":"2024-04-25T00:33:18.187850+0000","last_became_active":"2024-04-25T00:30:48.536486+0000","last_became_peered":"2024-04-25T00:30:48.536486+0000","last_unstale":"2024-04-25T00:33:18.187850+0000","last_undegraded":"2024-04-25T00:33:18.187850+0000","last_fullsized":"2024-04-25T00:33:18.187850+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:58:23.861959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":18,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.040101+0000","last_change":"2024-04-25T00:31:45.031208+0000","last_active":"2024-04-25T00:33:18.040101+0000","last_peered":"2024-04-25T00:33:18.040101+0000","last_clean":"2024-04-25T00:33:18.040101+0000","last_became_active":"2024-04-25T00:31:45.030831+0000","last_became_peered":"2024-04-25T00:31:45.030831+0000","last_unstale":"2024-04-25T00:33:18.040101+0000","last_undegraded":"2024-04-25T00:33:18.040101+0000","last_fullsized":"2024-04-25T00:33:18.040101+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:42:01.869146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,11,5],"acting":[10,8,11,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.14","version":"94'39","reported_seq":331,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.187222+0000","last_change":"2024-04-25T00:30:47.526899+0000","last_active":"2024-04-25T00:33:18.187222+0000","last_peered":"2024-04-25T00:33:18.187222+0000","last_clean":"2024-04-25T00:33:18.187222+0000","last_became_active":"2024-04-25T00:30:47.526528+0000","last_became_peered":"2024-04-25T00:30:47.526528+0000","last_unstale":"2024-04-25T00:33:18.187222+0000","last_undegraded":"2024-04-25T00:33:18.187222+0000","last_fullsized":"2024-04-25T00:33:18.187222+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":39,"log_dups_size":0,"ondisk_log_size":39,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:13:34.004584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67428,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":258,"num_read_kb":195,"num_write":40,"num_write_kb":86,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"91'6","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.030904+0000","last_change":"2024-04-25T00:30:48.538225+0000","last_active":"2024-04-25T00:33:18.030904+0000","last_peered":"2024-04-25T00:33:18.030904+0000","last_clean":"2024-04-25T00:33:18.030904+0000","last_became_active":"2024-04-25T00:30:48.537807+0000","last_became_peered":"2024-04-25T00:30:48.537807+0000","last_unstale":"2024-04-25T00:33:18.030904+0000","last_undegraded":"2024-04-25T00:33:18.030904+0000","last_fullsized":"2024-04-25T00:33:18.030904+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:00:46.275584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.037669+0000","last_change":"2024-04-25T00:30:52.782266+0000","last_active":"2024-04-25T00:33:18.037669+0000","last_peered":"2024-04-25T00:33:18.037669+0000","last_clean":"2024-04-25T00:33:18.037669+0000","last_became_active":"2024-04-25T00:30:52.781133+0000","last_became_peered":"2024-04-25T00:30:52.781133+0000","last_unstale":"2024-04-25T00:33:18.037669+0000","last_undegraded":"2024-04-25T00:33:18.037669+0000","last_fullsized":"2024-04-25T00:33:18.037669+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:19:03.818560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,8,3,6],"acting":[5,8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.13","version":"89'6","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926797+0000","last_change":"2024-04-25T00:30:47.520909+0000","last_active":"2024-04-25T00:33:17.926797+0000","last_peered":"2024-04-25T00:33:17.926797+0000","last_clean":"2024-04-25T00:33:17.926797+0000","last_became_active":"2024-04-25T00:30:47.520287+0000","last_became_peered":"2024-04-25T00:30:47.520287+0000","last_unstale":"2024-04-25T00:33:17.926797+0000","last_undegraded":"2024-04-25T00:33:17.926797+0000","last_fullsized":"2024-04-25T00:33:17.926797+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:49:43.795115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"91'9","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.167123+0000","last_change":"2024-04-25T00:30:48.532270+0000","last_active":"2024-04-25T00:33:18.167123+0000","last_peered":"2024-04-25T00:33:18.167123+0000","last_clean":"2024-04-25T00:33:18.167123+0000","last_became_active":"2024-04-25T00:30:48.531882+0000","last_became_peered":"2024-04-25T00:30:48.531882+0000","last_unstale":"2024-04-25T00:33:18.167123+0000","last_undegraded":"2024-04-25T00:33:18.167123+0000","last_fullsized":"2024-04-25T00:33:18.167123+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:58:17.865635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.304924+0000","last_change":"2024-04-25T00:30:52.777708+0000","last_active":"2024-04-25T00:33:18.304924+0000","last_peered":"2024-04-25T00:33:18.304924+0000","last_clean":"2024-04-25T00:33:18.304924+0000","last_became_active":"2024-04-25T00:30:52.777254+0000","last_became_peered":"2024-04-25T00:30:52.777254+0000","last_unstale":"2024-04-25T00:33:18.304924+0000","last_undegraded":"2024-04-25T00:33:18.304924+0000","last_fullsized":"2024-04-25T00:33:18.304924+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:29:47.427190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,9,7,6],"acting":[11,9,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.12","version":"89'3","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.037312+0000","last_change":"2024-04-25T00:30:47.524667+0000","last_active":"2024-04-25T00:33:18.037312+0000","last_peered":"2024-04-25T00:33:18.037312+0000","last_clean":"2024-04-25T00:33:18.037312+0000","last_became_active":"2024-04-25T00:30:47.524361+0000","last_became_peered":"2024-04-25T00:30:47.524361+0000","last_unstale":"2024-04-25T00:33:18.037312+0000","last_undegraded":"2024-04-25T00:33:18.037312+0000","last_fullsized":"2024-04-25T00:33:18.037312+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:16:08.805720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"91'15","reported_seq":47,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055606+0000","last_change":"2024-04-25T00:30:48.538658+0000","last_active":"2024-04-25T00:33:18.055606+0000","last_peered":"2024-04-25T00:33:18.055606+0000","last_clean":"2024-04-25T00:33:18.055606+0000","last_became_active":"2024-04-25T00:30:48.538316+0000","last_became_peered":"2024-04-25T00:30:48.538316+0000","last_unstale":"2024-04-25T00:33:18.055606+0000","last_undegraded":"2024-04-25T00:33:18.055606+0000","last_fullsized":"2024-04-25T00:33:18.055606+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:24:47.303559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":24,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.028189+0000","last_change":"2024-04-25T00:30:52.789512+0000","last_active":"2024-04-25T00:33:18.028189+0000","last_peered":"2024-04-25T00:33:18.028189+0000","last_clean":"2024-04-25T00:33:18.028189+0000","last_became_active":"2024-04-25T00:30:52.788929+0000","last_became_peered":"2024-04-25T00:30:52.788929+0000","last_unstale":"2024-04-25T00:33:18.028189+0000","last_undegraded":"2024-04-25T00:33:18.028189+0000","last_fullsized":"2024-04-25T00:33:18.028189+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:21:05.195296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,0,3],"acting":[9,7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.11","version":"94'8","reported_seq":171,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.185195+0000","last_change":"2024-04-25T00:30:47.513627+0000","last_active":"2024-04-25T00:33:18.185195+0000","last_peered":"2024-04-25T00:33:18.185195+0000","last_clean":"2024-04-25T00:33:18.185195+0000","last_became_active":"2024-04-25T00:30:47.513215+0000","last_became_peered":"2024-04-25T00:30:47.513215+0000","last_unstale":"2024-04-25T00:33:18.185195+0000","last_undegraded":"2024-04-25T00:33:18.185195+0000","last_fullsized":"2024-04-25T00:33:18.185195+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:32:16.111590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2210,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":130,"num_read_kb":6,"num_write":7,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"91'6","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055536+0000","last_change":"2024-04-25T00:30:48.533056+0000","last_active":"2024-04-25T00:33:18.055536+0000","last_peered":"2024-04-25T00:33:18.055536+0000","last_clean":"2024-04-25T00:33:18.055536+0000","last_became_active":"2024-04-25T00:30:48.532698+0000","last_became_peered":"2024-04-25T00:30:48.532698+0000","last_unstale":"2024-04-25T00:33:18.055536+0000","last_undegraded":"2024-04-25T00:33:18.055536+0000","last_fullsized":"2024-04-25T00:33:18.055536+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:36:24.987784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.167620+0000","last_change":"2024-04-25T00:30:52.780669+0000","last_active":"2024-04-25T00:33:18.167620+0000","last_peered":"2024-04-25T00:33:18.167620+0000","last_clean":"2024-04-25T00:33:18.167620+0000","last_became_active":"2024-04-25T00:30:52.780222+0000","last_became_peered":"2024-04-25T00:30:52.780222+0000","last_unstale":"2024-04-25T00:33:18.167620+0000","last_undegraded":"2024-04-25T00:33:18.167620+0000","last_fullsized":"2024-04-25T00:33:18.167620+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:50:27.415077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,9,11],"acting":[6,3,9,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"89'15","reported_seq":74,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.030496+0000","last_change":"2024-04-25T00:32:45.486412+0000","last_active":"2024-04-25T00:33:18.030496+0000","last_peered":"2024-04-25T00:33:18.030496+0000","last_clean":"2024-04-25T00:33:18.030496+0000","last_became_active":"2024-04-25T00:32:45.474176+0000","last_became_peered":"2024-04-25T00:32:45.474176+0000","last_unstale":"2024-04-25T00:33:18.030496+0000","last_undegraded":"2024-04-25T00:33:18.030496+0000","last_fullsized":"2024-04-25T00:33:18.030496+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:59:50.867374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":112,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,4],"acting":[8,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"91'6","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.037133+0000","last_change":"2024-04-25T00:30:48.534363+0000","last_active":"2024-04-25T00:33:18.037133+0000","last_peered":"2024-04-25T00:33:18.037133+0000","last_clean":"2024-04-25T00:33:18.037133+0000","last_became_active":"2024-04-25T00:30:48.533855+0000","last_became_peered":"2024-04-25T00:30:48.533855+0000","last_unstale":"2024-04-25T00:33:18.037133+0000","last_undegraded":"2024-04-25T00:33:18.037133+0000","last_fullsized":"2024-04-25T00:33:18.037133+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:28:29.374501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.037011+0000","last_change":"2024-04-25T00:30:52.790323+0000","last_active":"2024-04-25T00:33:18.037011+0000","last_peered":"2024-04-25T00:33:18.037011+0000","last_clean":"2024-04-25T00:33:18.037011+0000","last_became_active":"2024-04-25T00:30:52.789905+0000","last_became_peered":"2024-04-25T00:30:52.789905+0000","last_unstale":"2024-04-25T00:33:18.037011+0000","last_undegraded":"2024-04-25T00:33:18.037011+0000","last_fullsized":"2024-04-25T00:33:18.037011+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:19:03.687318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3,1],"acting":[5,0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"93'3","reported_seq":20,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305099+0000","last_change":"2024-04-25T00:32:45.490186+0000","last_active":"2024-04-25T00:33:18.305099+0000","last_peered":"2024-04-25T00:33:18.305099+0000","last_clean":"2024-04-25T00:33:18.305099+0000","last_became_active":"2024-04-25T00:32:45.474069+0000","last_became_peered":"2024-04-25T00:32:45.474069+0000","last_unstale":"2024-04-25T00:33:18.305099+0000","last_undegraded":"2024-04-25T00:33:18.305099+0000","last_fullsized":"2024-04-25T00:33:18.305099+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:58:34.104225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.e","version":"91'5","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305056+0000","last_change":"2024-04-25T00:30:48.539679+0000","last_active":"2024-04-25T00:33:18.305056+0000","last_peered":"2024-04-25T00:33:18.305056+0000","last_clean":"2024-04-25T00:33:18.305056+0000","last_became_active":"2024-04-25T00:30:48.538065+0000","last_became_peered":"2024-04-25T00:30:48.538065+0000","last_unstale":"2024-04-25T00:33:18.305056+0000","last_undegraded":"2024-04-25T00:33:18.305056+0000","last_fullsized":"2024-04-25T00:33:18.305056+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:05:16.435010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.030891+0000","last_change":"2024-04-25T00:30:52.804824+0000","last_active":"2024-04-25T00:33:18.030891+0000","last_peered":"2024-04-25T00:33:18.030891+0000","last_clean":"2024-04-25T00:33:18.030891+0000","last_became_active":"2024-04-25T00:30:52.804322+0000","last_became_peered":"2024-04-25T00:30:52.804322+0000","last_unstale":"2024-04-25T00:33:18.030891+0000","last_undegraded":"2024-04-25T00:33:18.030891+0000","last_fullsized":"2024-04-25T00:33:18.030891+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:38:08.385170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,10,11,9],"acting":[8,10,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":34,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.027849+0000","last_change":"2024-04-25T00:30:47.516916+0000","last_active":"2024-04-25T00:33:18.027849+0000","last_peered":"2024-04-25T00:33:18.027849+0000","last_clean":"2024-04-25T00:33:18.027849+0000","last_became_active":"2024-04-25T00:30:47.516520+0000","last_became_peered":"2024-04-25T00:30:47.516520+0000","last_unstale":"2024-04-25T00:33:18.027849+0000","last_undegraded":"2024-04-25T00:33:18.027849+0000","last_fullsized":"2024-04-25T00:33:18.027849+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:29:19.099764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"91'4","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.040501+0000","last_change":"2024-04-25T00:30:48.537584+0000","last_active":"2024-04-25T00:33:18.040501+0000","last_peered":"2024-04-25T00:33:18.040501+0000","last_clean":"2024-04-25T00:33:18.040501+0000","last_became_active":"2024-04-25T00:30:48.537193+0000","last_became_peered":"2024-04-25T00:30:48.537193+0000","last_unstale":"2024-04-25T00:33:18.040501+0000","last_undegraded":"2024-04-25T00:33:18.040501+0000","last_fullsized":"2024-04-25T00:33:18.040501+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:15:22.075323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.166777+0000","last_change":"2024-04-25T00:30:52.784802+0000","last_active":"2024-04-25T00:33:18.166777+0000","last_peered":"2024-04-25T00:33:18.166777+0000","last_clean":"2024-04-25T00:33:18.166777+0000","last_became_active":"2024-04-25T00:30:52.784359+0000","last_became_peered":"2024-04-25T00:30:52.784359+0000","last_unstale":"2024-04-25T00:33:18.166777+0000","last_undegraded":"2024-04-25T00:33:18.166777+0000","last_fullsized":"2024-04-25T00:33:18.166777+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:19:57.723542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1,8],"acting":[6,0,1,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"89'1","reported_seq":35,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055523+0000","last_change":"2024-04-25T00:30:47.525494+0000","last_active":"2024-04-25T00:33:18.055523+0000","last_peered":"2024-04-25T00:33:18.055523+0000","last_clean":"2024-04-25T00:33:18.055523+0000","last_became_active":"2024-04-25T00:30:47.525158+0000","last_became_peered":"2024-04-25T00:30:47.525158+0000","last_unstale":"2024-04-25T00:33:18.055523+0000","last_undegraded":"2024-04-25T00:33:18.055523+0000","last_fullsized":"2024-04-25T00:33:18.055523+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:51:18.543352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"91'7","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.040217+0000","last_change":"2024-04-25T00:30:48.540926+0000","last_active":"2024-04-25T00:33:18.040217+0000","last_peered":"2024-04-25T00:33:18.040217+0000","last_clean":"2024-04-25T00:33:18.040217+0000","last_became_active":"2024-04-25T00:30:48.540204+0000","last_became_peered":"2024-04-25T00:30:48.540204+0000","last_unstale":"2024-04-25T00:33:18.040217+0000","last_undegraded":"2024-04-25T00:33:18.040217+0000","last_fullsized":"2024-04-25T00:33:18.040217+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:02:45.530266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.167067+0000","last_change":"2024-04-25T00:30:52.805780+0000","last_active":"2024-04-25T00:33:18.167067+0000","last_peered":"2024-04-25T00:33:18.167067+0000","last_clean":"2024-04-25T00:33:18.167067+0000","last_became_active":"2024-04-25T00:30:52.805222+0000","last_became_peered":"2024-04-25T00:30:52.805222+0000","last_unstale":"2024-04-25T00:33:18.167067+0000","last_undegraded":"2024-04-25T00:33:18.167067+0000","last_fullsized":"2024-04-25T00:33:18.167067+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:32:17.034630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,10,0],"acting":[6,5,10,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.0","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.037047+0000","last_change":"2024-04-25T00:30:48.534335+0000","last_active":"2024-04-25T00:33:18.037047+0000","last_peered":"2024-04-25T00:33:18.037047+0000","last_clean":"2024-04-25T00:33:18.037047+0000","last_became_active":"2024-04-25T00:30:48.532670+0000","last_became_peered":"2024-04-25T00:30:48.532670+0000","last_unstale":"2024-04-25T00:33:18.037047+0000","last_undegraded":"2024-04-25T00:33:18.037047+0000","last_fullsized":"2024-04-25T00:33:18.037047+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:26:23.145918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"93'6","reported_seq":164,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.166569+0000","last_change":"2024-04-25T00:30:47.527695+0000","last_active":"2024-04-25T00:33:18.166569+0000","last_peered":"2024-04-25T00:33:18.166569+0000","last_clean":"2024-04-25T00:33:18.166569+0000","last_became_active":"2024-04-25T00:30:47.527239+0000","last_became_peered":"2024-04-25T00:30:47.527239+0000","last_unstale":"2024-04-25T00:33:18.166569+0000","last_undegraded":"2024-04-25T00:33:18.166569+0000","last_fullsized":"2024-04-25T00:33:18.166569+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:54:30.911284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":124,"num_read_kb":124,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"89'1","reported_seq":26,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055558+0000","last_change":"2024-04-25T00:30:52.781091+0000","last_active":"2024-04-25T00:33:18.055558+0000","last_peered":"2024-04-25T00:33:18.055558+0000","last_clean":"2024-04-25T00:33:18.055558+0000","last_became_active":"2024-04-25T00:30:52.780538+0000","last_became_peered":"2024-04-25T00:30:52.780538+0000","last_unstale":"2024-04-25T00:33:18.055558+0000","last_undegraded":"2024-04-25T00:33:18.055558+0000","last_fullsized":"2024-04-25T00:33:18.055558+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:49:18.141802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,11,9],"acting":[7,5,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.924882+0000","last_change":"2024-04-25T00:30:48.538730+0000","last_active":"2024-04-25T00:33:17.924882+0000","last_peered":"2024-04-25T00:33:17.924882+0000","last_clean":"2024-04-25T00:33:17.924882+0000","last_became_active":"2024-04-25T00:30:48.538336+0000","last_became_peered":"2024-04-25T00:30:48.538336+0000","last_unstale":"2024-04-25T00:33:17.924882+0000","last_undegraded":"2024-04-25T00:33:17.924882+0000","last_fullsized":"2024-04-25T00:33:17.924882+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:44:22.370448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"89'3","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926327+0000","last_change":"2024-04-25T00:30:47.520450+0000","last_active":"2024-04-25T00:33:17.926327+0000","last_peered":"2024-04-25T00:33:17.926327+0000","last_clean":"2024-04-25T00:33:17.926327+0000","last_became_active":"2024-04-25T00:30:47.520073+0000","last_became_peered":"2024-04-25T00:30:47.520073+0000","last_unstale":"2024-04-25T00:33:17.926327+0000","last_undegraded":"2024-04-25T00:33:17.926327+0000","last_fullsized":"2024-04-25T00:33:17.926327+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:49:53.570421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":4,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"22'137","reported_seq":367,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055359+0000","last_change":"2024-04-25T00:29:34.720552+0000","last_active":"2024-04-25T00:33:18.055359+0000","last_peered":"2024-04-25T00:33:18.055359+0000","last_clean":"2024-04-25T00:33:18.055359+0000","last_became_active":"2024-04-25T00:29:34.720041+0000","last_became_peered":"2024-04-25T00:29:34.720041+0000","last_unstale":"2024-04-25T00:33:18.055359+0000","last_undegraded":"2024-04-25T00:33:18.055359+0000","last_fullsized":"2024-04-25T00:33:18.055359+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:26:07.691582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:26:07.691582+0000","last_clean_scrub_stamp":"2024-04-25T00:26:07.691582+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:15:27.995162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.037382+0000","last_change":"2024-04-25T00:30:52.805489+0000","last_active":"2024-04-25T00:33:18.037382+0000","last_peered":"2024-04-25T00:33:18.037382+0000","last_clean":"2024-04-25T00:33:18.037382+0000","last_became_active":"2024-04-25T00:30:52.804758+0000","last_became_peered":"2024-04-25T00:30:52.804758+0000","last_unstale":"2024-04-25T00:33:18.037382+0000","last_undegraded":"2024-04-25T00:33:18.037382+0000","last_fullsized":"2024-04-25T00:33:18.037382+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:17:31.325548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,6,10],"acting":[5,9,6,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1","version":"91'5","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.166495+0000","last_change":"2024-04-25T00:30:48.537699+0000","last_active":"2024-04-25T00:33:18.166495+0000","last_peered":"2024-04-25T00:33:18.166495+0000","last_clean":"2024-04-25T00:33:18.166495+0000","last_became_active":"2024-04-25T00:30:48.537291+0000","last_became_peered":"2024-04-25T00:30:48.537291+0000","last_unstale":"2024-04-25T00:33:18.166495+0000","last_undegraded":"2024-04-25T00:33:18.166495+0000","last_fullsized":"2024-04-25T00:33:18.166495+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:25:36.816499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"89'3","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055470+0000","last_change":"2024-04-25T00:30:47.524694+0000","last_active":"2024-04-25T00:33:18.055470+0000","last_peered":"2024-04-25T00:33:18.055470+0000","last_clean":"2024-04-25T00:33:18.055470+0000","last_became_active":"2024-04-25T00:30:47.524359+0000","last_became_peered":"2024-04-25T00:30:47.524359+0000","last_unstale":"2024-04-25T00:33:18.055470+0000","last_undegraded":"2024-04-25T00:33:18.055470+0000","last_fullsized":"2024-04-25T00:33:18.055470+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:55:16.677234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.304812+0000","last_change":"2024-04-25T00:30:52.780364+0000","last_active":"2024-04-25T00:33:18.304812+0000","last_peered":"2024-04-25T00:33:18.304812+0000","last_clean":"2024-04-25T00:33:18.304812+0000","last_became_active":"2024-04-25T00:30:52.779904+0000","last_became_peered":"2024-04-25T00:30:52.779904+0000","last_unstale":"2024-04-25T00:33:18.304812+0000","last_undegraded":"2024-04-25T00:33:18.304812+0000","last_fullsized":"2024-04-25T00:33:18.304812+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:11:26.738526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,7,6],"acting":[11,5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.5","version":"89'1","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.030818+0000","last_change":"2024-04-25T00:30:47.525205+0000","last_active":"2024-04-25T00:33:18.030818+0000","last_peered":"2024-04-25T00:33:18.030818+0000","last_clean":"2024-04-25T00:33:18.030818+0000","last_became_active":"2024-04-25T00:30:47.517729+0000","last_became_peered":"2024-04-25T00:30:47.517729+0000","last_unstale":"2024-04-25T00:33:18.030818+0000","last_undegraded":"2024-04-25T00:33:18.030818+0000","last_fullsized":"2024-04-25T00:33:18.030818+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:21:14.810119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"91'9","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.166967+0000","last_change":"2024-04-25T00:30:48.533822+0000","last_active":"2024-04-25T00:33:18.166967+0000","last_peered":"2024-04-25T00:33:18.166967+0000","last_clean":"2024-04-25T00:33:18.166967+0000","last_became_active":"2024-04-25T00:30:48.533479+0000","last_became_peered":"2024-04-25T00:30:48.533479+0000","last_unstale":"2024-04-25T00:33:18.166967+0000","last_undegraded":"2024-04-25T00:33:18.166967+0000","last_fullsized":"2024-04-25T00:33:18.166967+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:49:01.028124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.924543+0000","last_change":"2024-04-25T00:30:52.804770+0000","last_active":"2024-04-25T00:33:17.924543+0000","last_peered":"2024-04-25T00:33:17.924543+0000","last_clean":"2024-04-25T00:33:17.924543+0000","last_became_active":"2024-04-25T00:30:52.804242+0000","last_became_peered":"2024-04-25T00:30:52.804242+0000","last_unstale":"2024-04-25T00:33:17.924543+0000","last_undegraded":"2024-04-25T00:33:17.924543+0000","last_fullsized":"2024-04-25T00:33:17.924543+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:34:11.761478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,10,0,7],"acting":[3,10,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"94'44","reported_seq":101,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.144934+0000","last_change":"2024-04-25T00:30:47.521364+0000","last_active":"2024-04-25T00:33:18.144934+0000","last_peered":"2024-04-25T00:33:18.144934+0000","last_clean":"2024-04-25T00:33:18.144934+0000","last_became_active":"2024-04-25T00:30:47.521032+0000","last_became_peered":"2024-04-25T00:30:47.521032+0000","last_unstale":"2024-04-25T00:33:18.144934+0000","last_undegraded":"2024-04-25T00:33:18.144934+0000","last_fullsized":"2024-04-25T00:33:18.144934+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":44,"log_dups_size":0,"ondisk_log_size":44,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:33:26.156568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393216,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":82,"num_read_kb":70,"num_write":78,"num_write_kb":836,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,8,3],"acting":[9,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"91'5","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.925363+0000","last_change":"2024-04-25T00:30:48.532942+0000","last_active":"2024-04-25T00:33:17.925363+0000","last_peered":"2024-04-25T00:33:17.925363+0000","last_clean":"2024-04-25T00:33:17.925363+0000","last_became_active":"2024-04-25T00:30:48.532580+0000","last_became_peered":"2024-04-25T00:30:48.532580+0000","last_unstale":"2024-04-25T00:33:17.925363+0000","last_undegraded":"2024-04-25T00:33:17.925363+0000","last_fullsized":"2024-04-25T00:33:17.925363+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:42:22.887920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926476+0000","last_change":"2024-04-25T00:30:52.779757+0000","last_active":"2024-04-25T00:33:17.926476+0000","last_peered":"2024-04-25T00:33:17.926476+0000","last_clean":"2024-04-25T00:33:17.926476+0000","last_became_active":"2024-04-25T00:30:52.779324+0000","last_became_peered":"2024-04-25T00:30:52.779324+0000","last_unstale":"2024-04-25T00:33:17.926476+0000","last_undegraded":"2024-04-25T00:33:17.926476+0000","last_fullsized":"2024-04-25T00:33:17.926476+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:22:39.062000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4,7],"acting":[2,1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"89'4","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926614+0000","last_change":"2024-04-25T00:30:47.518212+0000","last_active":"2024-04-25T00:33:17.926614+0000","last_peered":"2024-04-25T00:33:17.926614+0000","last_clean":"2024-04-25T00:33:17.926614+0000","last_became_active":"2024-04-25T00:30:47.517610+0000","last_became_peered":"2024-04-25T00:30:47.517610+0000","last_unstale":"2024-04-25T00:33:17.926614+0000","last_undegraded":"2024-04-25T00:33:17.926614+0000","last_fullsized":"2024-04-25T00:33:17.926614+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:13:58.904020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"91'7","reported_seq":45,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305983+0000","last_change":"2024-04-25T00:31:45.028411+0000","last_active":"2024-04-25T00:33:18.305983+0000","last_peered":"2024-04-25T00:33:18.305983+0000","last_clean":"2024-04-25T00:33:18.305983+0000","last_became_active":"2024-04-25T00:31:45.028066+0000","last_became_peered":"2024-04-25T00:31:45.028066+0000","last_unstale":"2024-04-25T00:33:18.305983+0000","last_undegraded":"2024-04-25T00:33:18.305983+0000","last_fullsized":"2024-04-25T00:33:18.305983+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:33:46.556862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926345+0000","last_change":"2024-04-25T00:30:52.804915+0000","last_active":"2024-04-25T00:33:17.926345+0000","last_peered":"2024-04-25T00:33:17.926345+0000","last_clean":"2024-04-25T00:33:17.926345+0000","last_became_active":"2024-04-25T00:30:52.804491+0000","last_became_peered":"2024-04-25T00:30:52.804491+0000","last_unstale":"2024-04-25T00:33:17.926345+0000","last_undegraded":"2024-04-25T00:33:17.926345+0000","last_fullsized":"2024-04-25T00:33:17.926345+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:51:50.219556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,10,9],"acting":[4,5,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.8","version":"89'2","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.187520+0000","last_change":"2024-04-25T00:30:47.521909+0000","last_active":"2024-04-25T00:33:18.187520+0000","last_peered":"2024-04-25T00:33:18.187520+0000","last_clean":"2024-04-25T00:33:18.187520+0000","last_became_active":"2024-04-25T00:30:47.521551+0000","last_became_peered":"2024-04-25T00:30:47.521551+0000","last_unstale":"2024-04-25T00:33:18.187520+0000","last_undegraded":"2024-04-25T00:33:18.187520+0000","last_fullsized":"2024-04-25T00:33:18.187520+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:51:16.368609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"91'2","reported_seq":34,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.030469+0000","last_change":"2024-04-25T00:30:48.530514+0000","last_active":"2024-04-25T00:33:18.030469+0000","last_peered":"2024-04-25T00:33:18.030469+0000","last_clean":"2024-04-25T00:33:18.030469+0000","last_became_active":"2024-04-25T00:30:48.530129+0000","last_became_peered":"2024-04-25T00:30:48.530129+0000","last_unstale":"2024-04-25T00:33:18.030469+0000","last_undegraded":"2024-04-25T00:33:18.030469+0000","last_fullsized":"2024-04-25T00:33:18.030469+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:51:25.623186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.028315+0000","last_change":"2024-04-25T00:30:52.785767+0000","last_active":"2024-04-25T00:33:18.028315+0000","last_peered":"2024-04-25T00:33:18.028315+0000","last_clean":"2024-04-25T00:33:18.028315+0000","last_became_active":"2024-04-25T00:30:52.785301+0000","last_became_peered":"2024-04-25T00:30:52.785301+0000","last_unstale":"2024-04-25T00:33:18.028315+0000","last_undegraded":"2024-04-25T00:33:18.028315+0000","last_fullsized":"2024-04-25T00:33:18.028315+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:10:14.071733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,2,5],"acting":[9,3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.9","version":"89'3","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.924598+0000","last_change":"2024-04-25T00:30:47.526364+0000","last_active":"2024-04-25T00:33:17.924598+0000","last_peered":"2024-04-25T00:33:17.924598+0000","last_clean":"2024-04-25T00:33:17.924598+0000","last_became_active":"2024-04-25T00:30:47.526009+0000","last_became_peered":"2024-04-25T00:30:47.526009+0000","last_unstale":"2024-04-25T00:33:17.924598+0000","last_undegraded":"2024-04-25T00:33:17.924598+0000","last_fullsized":"2024-04-25T00:33:17.924598+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:17:38.471878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"91'9","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.304758+0000","last_change":"2024-04-25T00:30:48.532283+0000","last_active":"2024-04-25T00:33:18.304758+0000","last_peered":"2024-04-25T00:33:18.304758+0000","last_clean":"2024-04-25T00:33:18.304758+0000","last_became_active":"2024-04-25T00:30:48.531935+0000","last_became_peered":"2024-04-25T00:30:48.531935+0000","last_unstale":"2024-04-25T00:33:18.304758+0000","last_undegraded":"2024-04-25T00:33:18.304758+0000","last_fullsized":"2024-04-25T00:33:18.304758+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:01:15.965611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.c","version":"89'1","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926179+0000","last_change":"2024-04-25T00:31:44.717218+0000","last_active":"2024-04-25T00:33:17.926179+0000","last_peered":"2024-04-25T00:33:17.926179+0000","last_clean":"2024-04-25T00:33:17.926179+0000","last_became_active":"2024-04-25T00:31:44.712770+0000","last_became_peered":"2024-04-25T00:31:44.712770+0000","last_unstale":"2024-04-25T00:33:17.926179+0000","last_undegraded":"2024-04-25T00:33:17.926179+0000","last_fullsized":"2024-04-25T00:33:17.926179+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:06:58.496133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":117,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,7,8],"acting":[4,10,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.a","version":"89'1","reported_seq":35,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926611+0000","last_change":"2024-04-25T00:30:47.520608+0000","last_active":"2024-04-25T00:33:17.926611+0000","last_peered":"2024-04-25T00:33:17.926611+0000","last_clean":"2024-04-25T00:33:17.926611+0000","last_became_active":"2024-04-25T00:30:47.520063+0000","last_became_peered":"2024-04-25T00:30:47.520063+0000","last_unstale":"2024-04-25T00:33:17.926611+0000","last_undegraded":"2024-04-25T00:33:17.926611+0000","last_fullsized":"2024-04-25T00:33:17.926611+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:58:47.823712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.925247+0000","last_change":"2024-04-25T00:30:48.535073+0000","last_active":"2024-04-25T00:33:17.925247+0000","last_peered":"2024-04-25T00:33:17.925247+0000","last_clean":"2024-04-25T00:33:17.925247+0000","last_became_active":"2024-04-25T00:30:48.534247+0000","last_became_peered":"2024-04-25T00:30:48.534247+0000","last_unstale":"2024-04-25T00:33:17.925247+0000","last_undegraded":"2024-04-25T00:33:17.925247+0000","last_fullsized":"2024-04-25T00:33:17.925247+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:46:18.613705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926144+0000","last_change":"2024-04-25T00:30:52.804519+0000","last_active":"2024-04-25T00:33:17.926144+0000","last_peered":"2024-04-25T00:33:17.926144+0000","last_clean":"2024-04-25T00:33:17.926144+0000","last_became_active":"2024-04-25T00:30:52.804074+0000","last_became_peered":"2024-04-25T00:30:52.804074+0000","last_unstale":"2024-04-25T00:33:17.926144+0000","last_undegraded":"2024-04-25T00:33:17.926144+0000","last_fullsized":"2024-04-25T00:33:17.926144+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:48:45.058873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,10,3],"acting":[4,2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.b","version":"89'4","reported_seq":19,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.304778+0000","last_change":"2024-04-25T00:32:45.477574+0000","last_active":"2024-04-25T00:33:18.304778+0000","last_peered":"2024-04-25T00:33:18.304778+0000","last_clean":"2024-04-25T00:33:18.304778+0000","last_became_active":"2024-04-25T00:32:45.472622+0000","last_became_peered":"2024-04-25T00:32:45.472622+0000","last_unstale":"2024-04-25T00:33:18.304778+0000","last_undegraded":"2024-04-25T00:33:18.304778+0000","last_fullsized":"2024-04-25T00:33:18.304778+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:18:56.072904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,7],"acting":[11,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.a","version":"91'5","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055542+0000","last_change":"2024-04-25T00:30:48.537152+0000","last_active":"2024-04-25T00:33:18.055542+0000","last_peered":"2024-04-25T00:33:18.055542+0000","last_clean":"2024-04-25T00:33:18.055542+0000","last_became_active":"2024-04-25T00:30:48.536256+0000","last_became_peered":"2024-04-25T00:30:48.536256+0000","last_unstale":"2024-04-25T00:33:18.055542+0000","last_undegraded":"2024-04-25T00:33:18.055542+0000","last_fullsized":"2024-04-25T00:33:18.055542+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:57:53.870212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.187504+0000","last_change":"2024-04-25T00:30:52.781022+0000","last_active":"2024-04-25T00:33:18.187504+0000","last_peered":"2024-04-25T00:33:18.187504+0000","last_clean":"2024-04-25T00:33:18.187504+0000","last_became_active":"2024-04-25T00:30:52.780595+0000","last_became_peered":"2024-04-25T00:30:52.780595+0000","last_unstale":"2024-04-25T00:33:18.187504+0000","last_undegraded":"2024-04-25T00:33:18.187504+0000","last_fullsized":"2024-04-25T00:33:18.187504+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:27:02.580870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6,7],"acting":[1,5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.c","version":"89'3","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926971+0000","last_change":"2024-04-25T00:30:47.519118+0000","last_active":"2024-04-25T00:33:17.926971+0000","last_peered":"2024-04-25T00:33:17.926971+0000","last_clean":"2024-04-25T00:33:17.926971+0000","last_became_active":"2024-04-25T00:30:47.518639+0000","last_became_peered":"2024-04-25T00:30:47.518639+0000","last_unstale":"2024-04-25T00:33:17.926971+0000","last_undegraded":"2024-04-25T00:33:17.926971+0000","last_fullsized":"2024-04-25T00:33:17.926971+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:32:29.747903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"91'7","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055812+0000","last_change":"2024-04-25T00:30:48.542073+0000","last_active":"2024-04-25T00:33:18.055812+0000","last_peered":"2024-04-25T00:33:18.055812+0000","last_clean":"2024-04-25T00:33:18.055812+0000","last_became_active":"2024-04-25T00:30:48.541722+0000","last_became_peered":"2024-04-25T00:30:48.541722+0000","last_unstale":"2024-04-25T00:33:18.055812+0000","last_undegraded":"2024-04-25T00:33:18.055812+0000","last_fullsized":"2024-04-25T00:33:18.055812+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:34:03.397539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305021+0000","last_change":"2024-04-25T00:30:52.790234+0000","last_active":"2024-04-25T00:33:18.305021+0000","last_peered":"2024-04-25T00:33:18.305021+0000","last_clean":"2024-04-25T00:33:18.305021+0000","last_became_active":"2024-04-25T00:30:52.789788+0000","last_became_peered":"2024-04-25T00:30:52.789788+0000","last_unstale":"2024-04-25T00:33:18.305021+0000","last_undegraded":"2024-04-25T00:33:18.305021+0000","last_fullsized":"2024-04-25T00:33:18.305021+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:44:43.399771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,9,1,4],"acting":[0,9,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.d","version":"94'39","reported_seq":81,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.142633+0000","last_change":"2024-04-25T00:30:47.523597+0000","last_active":"2024-04-25T00:33:18.142633+0000","last_peered":"2024-04-25T00:33:18.142633+0000","last_clean":"2024-04-25T00:33:18.142633+0000","last_became_active":"2024-04-25T00:30:47.523225+0000","last_became_peered":"2024-04-25T00:30:47.523225+0000","last_unstale":"2024-04-25T00:33:18.142633+0000","last_undegraded":"2024-04-25T00:33:18.142633+0000","last_fullsized":"2024-04-25T00:33:18.142633+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":39,"log_dups_size":0,"ondisk_log_size":39,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:51:12.341015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131670,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":11,"num_read_kb":10,"num_write":49,"num_write_kb":475,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"91'6","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055778+0000","last_change":"2024-04-25T00:30:48.541417+0000","last_active":"2024-04-25T00:33:18.055778+0000","last_peered":"2024-04-25T00:33:18.055778+0000","last_clean":"2024-04-25T00:33:18.055778+0000","last_became_active":"2024-04-25T00:30:48.541015+0000","last_became_peered":"2024-04-25T00:30:48.541015+0000","last_unstale":"2024-04-25T00:33:18.055778+0000","last_undegraded":"2024-04-25T00:33:18.055778+0000","last_fullsized":"2024-04-25T00:33:18.055778+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:21:35.299823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.036816+0000","last_change":"2024-04-25T00:30:52.782230+0000","last_active":"2024-04-25T00:33:18.036816+0000","last_peered":"2024-04-25T00:33:18.036816+0000","last_clean":"2024-04-25T00:33:18.036816+0000","last_became_active":"2024-04-25T00:30:52.781708+0000","last_became_peered":"2024-04-25T00:30:52.781708+0000","last_unstale":"2024-04-25T00:33:18.036816+0000","last_undegraded":"2024-04-25T00:33:18.036816+0000","last_fullsized":"2024-04-25T00:33:18.036816+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:58:53.260985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7,1],"acting":[5,2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.e","version":"84'2","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.167506+0000","last_change":"2024-04-25T00:32:45.553977+0000","last_active":"2024-04-25T00:33:18.167506+0000","last_peered":"2024-04-25T00:33:18.167506+0000","last_clean":"2024-04-25T00:33:18.167506+0000","last_became_active":"2024-04-25T00:32:45.551280+0000","last_became_peered":"2024-04-25T00:32:45.551280+0000","last_unstale":"2024-04-25T00:33:18.167506+0000","last_undegraded":"2024-04-25T00:33:18.167506+0000","last_fullsized":"2024-04-25T00:33:18.167506+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:54:12.307459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"91'9","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055986+0000","last_change":"2024-04-25T00:30:48.532730+0000","last_active":"2024-04-25T00:33:18.055986+0000","last_peered":"2024-04-25T00:33:18.055986+0000","last_clean":"2024-04-25T00:33:18.055986+0000","last_became_active":"2024-04-25T00:30:48.532129+0000","last_became_peered":"2024-04-25T00:30:48.532129+0000","last_unstale":"2024-04-25T00:33:18.055986+0000","last_undegraded":"2024-04-25T00:33:18.055986+0000","last_fullsized":"2024-04-25T00:33:18.055986+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:36:30.115125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"91'9","reported_seq":47,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.304840+0000","last_change":"2024-04-25T00:31:44.712173+0000","last_active":"2024-04-25T00:33:18.304840+0000","last_peered":"2024-04-25T00:33:18.304840+0000","last_clean":"2024-04-25T00:33:18.304840+0000","last_became_active":"2024-04-25T00:31:44.711738+0000","last_became_peered":"2024-04-25T00:31:44.711738+0000","last_unstale":"2024-04-25T00:33:18.304840+0000","last_undegraded":"2024-04-25T00:33:18.304840+0000","last_fullsized":"2024-04-25T00:33:18.304840+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:06:59.466566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,9],"acting":[11,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"91'12","reported_seq":44,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926692+0000","last_change":"2024-04-25T00:30:48.540676+0000","last_active":"2024-04-25T00:33:17.926692+0000","last_peered":"2024-04-25T00:33:17.926692+0000","last_clean":"2024-04-25T00:33:17.926692+0000","last_became_active":"2024-04-25T00:30:48.540258+0000","last_became_peered":"2024-04-25T00:30:48.540258+0000","last_unstale":"2024-04-25T00:33:17.926692+0000","last_undegraded":"2024-04-25T00:33:17.926692+0000","last_fullsized":"2024-04-25T00:33:17.926692+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:10:29.930897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926531+0000","last_change":"2024-04-25T00:30:48.542191+0000","last_active":"2024-04-25T00:33:17.926531+0000","last_peered":"2024-04-25T00:33:17.926531+0000","last_clean":"2024-04-25T00:33:17.926531+0000","last_became_active":"2024-04-25T00:30:48.540433+0000","last_became_peered":"2024-04-25T00:30:48.540433+0000","last_unstale":"2024-04-25T00:33:17.926531+0000","last_undegraded":"2024-04-25T00:33:17.926531+0000","last_fullsized":"2024-04-25T00:33:17.926531+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:21:55.694555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"91'7","reported_seq":45,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305410+0000","last_change":"2024-04-25T00:31:45.028445+0000","last_active":"2024-04-25T00:33:18.305410+0000","last_peered":"2024-04-25T00:33:18.305410+0000","last_clean":"2024-04-25T00:33:18.305410+0000","last_became_active":"2024-04-25T00:31:45.028048+0000","last_became_peered":"2024-04-25T00:31:45.028048+0000","last_unstale":"2024-04-25T00:33:18.305410+0000","last_undegraded":"2024-04-25T00:33:18.305410+0000","last_fullsized":"2024-04-25T00:33:18.305410+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:04:02.714111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,8],"acting":[0,6,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"91'4","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.167135+0000","last_change":"2024-04-25T00:30:48.527892+0000","last_active":"2024-04-25T00:33:18.167135+0000","last_peered":"2024-04-25T00:33:18.167135+0000","last_clean":"2024-04-25T00:33:18.167135+0000","last_became_active":"2024-04-25T00:30:48.527522+0000","last_became_peered":"2024-04-25T00:30:48.527522+0000","last_unstale":"2024-04-25T00:33:18.167135+0000","last_undegraded":"2024-04-25T00:33:18.167135+0000","last_fullsized":"2024-04-25T00:33:18.167135+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:21:22.779470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"91'6","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305133+0000","last_change":"2024-04-25T00:30:48.535066+0000","last_active":"2024-04-25T00:33:18.305133+0000","last_peered":"2024-04-25T00:33:18.305133+0000","last_clean":"2024-04-25T00:33:18.305133+0000","last_became_active":"2024-04-25T00:30:48.534710+0000","last_became_peered":"2024-04-25T00:30:48.534710+0000","last_unstale":"2024-04-25T00:33:18.305133+0000","last_undegraded":"2024-04-25T00:33:18.305133+0000","last_fullsized":"2024-04-25T00:33:18.305133+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:57:18.582569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"91'15","reported_seq":47,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.927199+0000","last_change":"2024-04-25T00:30:48.541921+0000","last_active":"2024-04-25T00:33:17.927199+0000","last_peered":"2024-04-25T00:33:17.927199+0000","last_clean":"2024-04-25T00:33:17.927199+0000","last_became_active":"2024-04-25T00:30:48.541348+0000","last_became_peered":"2024-04-25T00:30:48.541348+0000","last_unstale":"2024-04-25T00:33:17.927199+0000","last_undegraded":"2024-04-25T00:33:17.927199+0000","last_fullsized":"2024-04-25T00:33:17.927199+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:07:00.001504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"91'2","reported_seq":34,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.925569+0000","last_change":"2024-04-25T00:30:48.536902+0000","last_active":"2024-04-25T00:33:17.925569+0000","last_peered":"2024-04-25T00:33:17.925569+0000","last_clean":"2024-04-25T00:33:17.925569+0000","last_became_active":"2024-04-25T00:30:48.536008+0000","last_became_peered":"2024-04-25T00:30:48.536008+0000","last_unstale":"2024-04-25T00:33:17.925569+0000","last_undegraded":"2024-04-25T00:33:17.925569+0000","last_fullsized":"2024-04-25T00:33:17.925569+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:19:15.050142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"91'6","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.036972+0000","last_change":"2024-04-25T00:30:48.533104+0000","last_active":"2024-04-25T00:33:18.036972+0000","last_peered":"2024-04-25T00:33:18.036972+0000","last_clean":"2024-04-25T00:33:18.036972+0000","last_became_active":"2024-04-25T00:30:48.532124+0000","last_became_peered":"2024-04-25T00:30:48.532124+0000","last_unstale":"2024-04-25T00:33:18.036972+0000","last_undegraded":"2024-04-25T00:33:18.036972+0000","last_fullsized":"2024-04-25T00:33:18.036972+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:54:57.391433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.924561+0000","last_change":"2024-04-25T00:30:48.542074+0000","last_active":"2024-04-25T00:33:17.924561+0000","last_peered":"2024-04-25T00:33:17.924561+0000","last_clean":"2024-04-25T00:33:17.924561+0000","last_became_active":"2024-04-25T00:30:48.538752+0000","last_became_peered":"2024-04-25T00:30:48.538752+0000","last_unstale":"2024-04-25T00:33:17.924561+0000","last_undegraded":"2024-04-25T00:33:17.924561+0000","last_fullsized":"2024-04-25T00:33:17.924561+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:55:30.613616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"91'10","reported_seq":42,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.187829+0000","last_change":"2024-04-25T00:30:48.534158+0000","last_active":"2024-04-25T00:33:18.187829+0000","last_peered":"2024-04-25T00:33:18.187829+0000","last_clean":"2024-04-25T00:33:18.187829+0000","last_became_active":"2024-04-25T00:30:48.533821+0000","last_became_peered":"2024-04-25T00:30:48.533821+0000","last_unstale":"2024-04-25T00:33:18.187829+0000","last_undegraded":"2024-04-25T00:33:18.187829+0000","last_fullsized":"2024-04-25T00:33:18.187829+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:05:19.327608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"91'11","reported_seq":43,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.167481+0000","last_change":"2024-04-25T00:30:48.531718+0000","last_active":"2024-04-25T00:33:18.167481+0000","last_peered":"2024-04-25T00:33:18.167481+0000","last_clean":"2024-04-25T00:33:18.167481+0000","last_became_active":"2024-04-25T00:30:48.531291+0000","last_became_peered":"2024-04-25T00:30:48.531291+0000","last_unstale":"2024-04-25T00:33:18.167481+0000","last_undegraded":"2024-04-25T00:33:18.167481+0000","last_fullsized":"2024-04-25T00:33:18.167481+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:56:49.032937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"91'8","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.304987+0000","last_change":"2024-04-25T00:31:44.713036+0000","last_active":"2024-04-25T00:33:18.304987+0000","last_peered":"2024-04-25T00:33:18.304987+0000","last_clean":"2024-04-25T00:33:18.304987+0000","last_became_active":"2024-04-25T00:31:44.712651+0000","last_became_peered":"2024-04-25T00:31:44.712651+0000","last_unstale":"2024-04-25T00:33:18.304987+0000","last_undegraded":"2024-04-25T00:33:18.304987+0000","last_fullsized":"2024-04-25T00:33:18.304987+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:05:04.811510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,9,8],"acting":[11,9,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":234,"num_objects":2,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":117,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":32768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2,"ondisk_log_size":2,"up":128,"acting":128,"num_store_stats":12},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":500,"ondisk_log_size":500,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":597867,"num_objects":62,"num_object_clones":0,"num_object_copies":186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":62,"num_whiteouts":0,"num_read":864,"num_read_kb":540,"num_write":289,"num_write_kb":1441,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":134,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2027520,"data_stored":1793601,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":229,"ondisk_log_size":229,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":80,"seq":343597383723,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32232,"kb_used_data":1052,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712920,"statfs":{"total":95995035648,"available":95962030080,"internally_reserved":0,"allocated":1077248,"data_stored":529717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1596,"internal_metadata":31914436},"hb_peers":[0,1,2,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":10,"up_from":72,"seq":309237645360,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28880,"kb_used_data":1864,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716272,"statfs":{"total":95995035648,"available":95965462528,"internally_reserved":0,"allocated":1908736,"data_stored":1321709,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1600,"internal_metadata":27654592},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":9,"up_from":67,"seq":287762808885,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36788,"kb_used_data":1448,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708364,"statfs":{"total":95995035648,"available":95957364736,"internally_reserved":0,"allocated":1482752,"data_stored":923147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1600,"internal_metadata":36174272},"hb_peers":[0,1,2,3,4,5,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":8,"up_from":59,"seq":253403070522,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32620,"kb_used_data":1444,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712532,"statfs":{"total":95995035648,"available":95961632768,"internally_reserved":0,"allocated":1478656,"data_stored":923045,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":7,"up_from":51,"seq":219043332159,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36984,"kb_used_data":1644,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708168,"statfs":{"total":95995035648,"available":95957164032,"internally_reserved":0,"allocated":1683456,"data_stored":1127382,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":36174282},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495685,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36384,"kb_used_data":1044,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708768,"statfs":{"total":95995035648,"available":95957778432,"internally_reserved":0,"allocated":1069056,"data_stored":525107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":36174278},"hb_peers":[0,1,2,3,4,5,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574940,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28116,"kb_used_data":1100,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717036,"statfs":{"total":95995035648,"available":95966244864,"internally_reserved":0,"allocated":1126400,"data_stored":581739,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738467,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28076,"kb_used_data":1052,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717076,"statfs":{"total":95995035648,"available":95966285824,"internally_reserved":0,"allocated":1077248,"data_stored":516571,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":18,"seq":77309411416,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28152,"kb_used_data":1132,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717000,"statfs":{"total":95995035648,"available":95966208000,"internally_reserved":0,"allocated":1159168,"data_stored":594074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":25,"seq":107374182483,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36592,"kb_used_data":1252,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708560,"statfs":{"total":95995035648,"available":95957565440,"internally_reserved":0,"allocated":1282048,"data_stored":722421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":36174279},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":33,"seq":141733920847,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28084,"kb_used_data":1064,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717068,"statfs":{"total":95995035648,"available":95966277632,"internally_reserved":0,"allocated":1089536,"data_stored":530709,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757322,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28800,"kb_used_data":1784,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716352,"statfs":{"total":95995035648,"available":95965544448,"internally_reserved":0,"allocated":1826816,"data_stored":1252948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":77824,"data_stored":67428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":94208,"data_stored":69057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":544768,"data_stored":525100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":2378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":163840,"data_stored":133467,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":3805,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":409600,"data_stored":393932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":394034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":245760,"data_stored":202228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":8192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-25T00:33:20.945 INFO:tasks.fwd_scrub.ceph_manager:clean! 2024-04-25T00:33:20.945 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-04-25T00:33:21.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.085+0000 7f22bb9a5640 1 -- 172.21.15.32:0/3465854048 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22b4101870 msgr2=0x7f22b4101cf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:21.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.085+0000 7f22bb9a5640 1 --2- 172.21.15.32:0/3465854048 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22b4101870 0x7f22b4101cf0 secure :-1 s=READY pgs=487 cs=0 l=1 rev1=1 crypto rx=0x7f22b0009900 tx=0x7f22b002a410 comp rx=0 tx=0).stop 2024-04-25T00:33:21.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.085+0000 7f22bb9a5640 1 -- 172.21.15.32:0/3465854048 shutdown_connections 2024-04-25T00:33:21.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.085+0000 7f22bb9a5640 1 --2- 172.21.15.32:0/3465854048 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22b4102230 0x7f22b4108ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.085+0000 7f22bb9a5640 1 --2- 172.21.15.32:0/3465854048 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22b4101870 0x7f22b4101cf0 unknown :-1 s=CLOSED pgs=487 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.085+0000 7f22bb9a5640 1 --2- 172.21.15.32:0/3465854048 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f22b4100680 0x7f22b4100aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.085+0000 7f22bb9a5640 1 -- 172.21.15.32:0/3465854048 >> 172.21.15.32:0/3465854048 conn(0x7f22b40fbbf0 msgr2=0x7f22b40fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:21.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.085+0000 7f22bb9a5640 1 -- 172.21.15.32:0/3465854048 shutdown_connections 2024-04-25T00:33:21.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.085+0000 7f22bb9a5640 1 -- 172.21.15.32:0/3465854048 wait complete. 2024-04-25T00:33:21.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.085+0000 7f22bb9a5640 1 Processor -- start 2024-04-25T00:33:21.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.086+0000 7f22bb9a5640 1 -- start start 2024-04-25T00:33:21.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.086+0000 7f22bb9a5640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f22b4100680 0x7f22b41969a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:21.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.086+0000 7f22bb9a5640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22b4101870 0x7f22b4196ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:21.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.086+0000 7f22bb9a5640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22b4102230 0x7f22b419df50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:21.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.086+0000 7f22bb9a5640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f22b410c300 con 0x7f22b4102230 2024-04-25T00:33:21.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.086+0000 7f22bb9a5640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f22b410c180 con 0x7f22b4100680 2024-04-25T00:33:21.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.086+0000 7f22bb9a5640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f22b410c480 con 0x7f22b4101870 2024-04-25T00:33:21.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.086+0000 7f22b9f42640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22b4102230 0x7f22b419df50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:21.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.086+0000 7f22b8f40640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22b4101870 0x7f22b4196ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:21.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.086+0000 7f22b9f42640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22b4102230 0x7f22b419df50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44410/0 (socket says 172.21.15.32:44410) 2024-04-25T00:33:21.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.086+0000 7f22b9741640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f22b4100680 0x7f22b41969a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:21.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.086+0000 7f22b9f42640 1 -- 172.21.15.32:0/3082144087 learned_addr learned my addr 172.21.15.32:0/3082144087 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:21.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.086+0000 7f22b9f42640 1 -- 172.21.15.32:0/3082144087 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22b4101870 msgr2=0x7f22b4196ee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:21.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.086+0000 7f22b9f42640 1 --2- 172.21.15.32:0/3082144087 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22b4101870 0x7f22b4196ee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.086+0000 7f22b9f42640 1 -- 172.21.15.32:0/3082144087 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f22b4100680 msgr2=0x7f22b41969a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:21.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.086+0000 7f22b9f42640 1 --2- 172.21.15.32:0/3082144087 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f22b4100680 0x7f22b41969a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.086+0000 7f22b9f42640 1 -- 172.21.15.32:0/3082144087 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f22b419e5c0 con 0x7f22b4102230 2024-04-25T00:33:21.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.086+0000 7f22b9741640 1 --2- 172.21.15.32:0/3082144087 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f22b4100680 0x7f22b41969a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:21.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.086+0000 7f22b8f40640 1 --2- 172.21.15.32:0/3082144087 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22b4101870 0x7f22b4196ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:21.090 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.086+0000 7f22b9f42640 1 --2- 172.21.15.32:0/3082144087 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22b4102230 0x7f22b419df50 secure :-1 s=READY pgs=488 cs=0 l=1 rev1=1 crypto rx=0x7f22ac00eb10 tx=0x7f22ac00c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:21.090 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.087+0000 7f22aa7fc640 1 -- 172.21.15.32:0/3082144087 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f22ac004b20 con 0x7f22b4102230 2024-04-25T00:33:21.090 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.087+0000 7f22bb9a5640 1 -- 172.21.15.32:0/3082144087 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f22b419e8b0 con 0x7f22b4102230 2024-04-25T00:33:21.090 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.087+0000 7f22aa7fc640 1 -- 172.21.15.32:0/3082144087 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f22ac004cc0 con 0x7f22b4102230 2024-04-25T00:33:21.090 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.087+0000 7f22aa7fc640 1 -- 172.21.15.32:0/3082144087 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f22ac010760 con 0x7f22b4102230 2024-04-25T00:33:21.090 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.087+0000 7f22bb9a5640 1 -- 172.21.15.32:0/3082144087 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f22b419ee20 con 0x7f22b4102230 2024-04-25T00:33:21.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.088+0000 7f22bb9a5640 1 -- 172.21.15.32:0/3082144087 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f22b41079a0 con 0x7f22b4102230 2024-04-25T00:33:21.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.089+0000 7f22aa7fc640 1 -- 172.21.15.32:0/3082144087 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 23) ==== 137953+0+0 (secure 0 0 0) 0x7f22ac005020 con 0x7f22b4102230 2024-04-25T00:33:21.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.089+0000 7f22aa7fc640 1 --2- 172.21.15.32:0/3082144087 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f22a00a2d00 0x7f22a00a51c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:21.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.090+0000 7f22b9741640 1 --2- 172.21.15.32:0/3082144087 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f22a00a2d00 0x7f22a00a51c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:21.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.090+0000 7f22aa7fc640 1 -- 172.21.15.32:0/3082144087 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f22ac0cfea0 con 0x7f22b4102230 2024-04-25T00:33:21.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.090+0000 7f22b9741640 1 --2- 172.21.15.32:0/3082144087 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f22a00a2d00 0x7f22a00a51c0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f22b41016d0 tx=0x7f22a4005f90 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:21.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.094+0000 7f22aa7fc640 1 -- 172.21.15.32:0/3082144087 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f22ac09c710 con 0x7f22b4102230 2024-04-25T00:33:21.282 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.282+0000 7f22bb9a5640 1 -- 172.21.15.32:0/3082144087 --> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f22b4061b80 con 0x7f22a00a2d00 2024-04-25T00:33:21.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.291+0000 7f22aa7fc640 1 -- 172.21.15.32:0/3082144087 <== mgr.15483 v2:172.21.15.32:6800/3307874911 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+339393 (secure 0 0 0) 0x7f22b4061b80 con 0x7f22a00a2d00 2024-04-25T00:33:21.295 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:33:21.297 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-04-25T00:33:21.300 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.295+0000 7f22bb9a5640 1 -- 172.21.15.32:0/3082144087 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f22a00a2d00 msgr2=0x7f22a00a51c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:21.300 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.295+0000 7f22bb9a5640 1 --2- 172.21.15.32:0/3082144087 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f22a00a2d00 0x7f22a00a51c0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f22b41016d0 tx=0x7f22a4005f90 comp rx=0 tx=0).stop 2024-04-25T00:33:21.300 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.295+0000 7f22bb9a5640 1 -- 172.21.15.32:0/3082144087 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22b4102230 msgr2=0x7f22b419df50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:21.300 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.295+0000 7f22bb9a5640 1 --2- 172.21.15.32:0/3082144087 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22b4102230 0x7f22b419df50 secure :-1 s=READY pgs=488 cs=0 l=1 rev1=1 crypto rx=0x7f22ac00eb10 tx=0x7f22ac00c2d0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.300 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.295+0000 7f22bb9a5640 1 -- 172.21.15.32:0/3082144087 shutdown_connections 2024-04-25T00:33:21.300 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.295+0000 7f22bb9a5640 1 --2- 172.21.15.32:0/3082144087 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f22a00a2d00 0x7f22a00a51c0 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.300 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.295+0000 7f22bb9a5640 1 --2- 172.21.15.32:0/3082144087 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22b4102230 0x7f22b419df50 unknown :-1 s=CLOSED pgs=488 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.300 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.295+0000 7f22bb9a5640 1 --2- 172.21.15.32:0/3082144087 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22b4101870 0x7f22b4196ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.300 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.295+0000 7f22bb9a5640 1 --2- 172.21.15.32:0/3082144087 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f22b4100680 0x7f22b41969a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.300 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.295+0000 7f22bb9a5640 1 -- 172.21.15.32:0/3082144087 >> 172.21.15.32:0/3082144087 conn(0x7f22b40fbbf0 msgr2=0x7f22b4104b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:21.300 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.295+0000 7f22bb9a5640 1 -- 172.21.15.32:0/3082144087 shutdown_connections 2024-04-25T00:33:21.300 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.295+0000 7f22bb9a5640 1 -- 172.21.15.32:0/3082144087 wait complete. 2024-04-25T00:33:21.307 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":4,"stamp":"2024-04-25T00:33:19.938647+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1188469,"num_objects":66,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":66,"num_whiteouts":0,"num_read":1000,"num_read_kb":659,"num_write":472,"num_write_kb":4381,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":1180987,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":868,"ondisk_log_size":868,"up":419,"acting":419,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":419,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":381708,"kb_used_data":15880,"kb_used_omap":18,"kb_used_meta":365677,"kb_avail":1124560116,"statfs":{"total":1151940427776,"available":1151549558784,"internally_reserved":0,"allocated":16261120,"data_stored":9548569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19142,"internal_metadata":374453562},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.990637"},"pg_stats":[{"pgid":"3.3f","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305395+0000","last_change":"2024-04-25T00:30:48.538719+0000","last_active":"2024-04-25T00:33:18.305395+0000","last_peered":"2024-04-25T00:33:18.305395+0000","last_clean":"2024-04-25T00:33:18.305395+0000","last_became_active":"2024-04-25T00:30:48.538356+0000","last_became_peered":"2024-04-25T00:30:48.538356+0000","last_unstale":"2024-04-25T00:33:18.305395+0000","last_undegraded":"2024-04-25T00:33:18.305395+0000","last_fullsized":"2024-04-25T00:33:18.305395+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:39:08.974838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"91'4","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305901+0000","last_change":"2024-04-25T00:30:48.532262+0000","last_active":"2024-04-25T00:33:18.305901+0000","last_peered":"2024-04-25T00:33:18.305901+0000","last_clean":"2024-04-25T00:33:18.305901+0000","last_became_active":"2024-04-25T00:30:48.531926+0000","last_became_peered":"2024-04-25T00:30:48.531926+0000","last_unstale":"2024-04-25T00:33:18.305901+0000","last_undegraded":"2024-04-25T00:33:18.305901+0000","last_fullsized":"2024-04-25T00:33:18.305901+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:12:06.527649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"91'7","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305705+0000","last_change":"2024-04-25T00:30:48.536383+0000","last_active":"2024-04-25T00:33:18.305705+0000","last_peered":"2024-04-25T00:33:18.305705+0000","last_clean":"2024-04-25T00:33:18.305705+0000","last_became_active":"2024-04-25T00:30:48.536048+0000","last_became_peered":"2024-04-25T00:30:48.536048+0000","last_unstale":"2024-04-25T00:33:18.305705+0000","last_undegraded":"2024-04-25T00:33:18.305705+0000","last_fullsized":"2024-04-25T00:33:18.305705+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:32:26.728441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926366+0000","last_change":"2024-04-25T00:30:48.532195+0000","last_active":"2024-04-25T00:33:17.926366+0000","last_peered":"2024-04-25T00:33:17.926366+0000","last_clean":"2024-04-25T00:33:17.926366+0000","last_became_active":"2024-04-25T00:30:48.531851+0000","last_became_peered":"2024-04-25T00:30:48.531851+0000","last_unstale":"2024-04-25T00:33:17.926366+0000","last_undegraded":"2024-04-25T00:33:17.926366+0000","last_fullsized":"2024-04-25T00:33:17.926366+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:15:34.704958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"91'4","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.030694+0000","last_change":"2024-04-25T00:30:48.534024+0000","last_active":"2024-04-25T00:33:18.030694+0000","last_peered":"2024-04-25T00:33:18.030694+0000","last_clean":"2024-04-25T00:33:18.030694+0000","last_became_active":"2024-04-25T00:30:48.533681+0000","last_became_peered":"2024-04-25T00:30:48.533681+0000","last_unstale":"2024-04-25T00:33:18.030694+0000","last_undegraded":"2024-04-25T00:33:18.030694+0000","last_fullsized":"2024-04-25T00:33:18.030694+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:07:35.816816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"91'6","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.040282+0000","last_change":"2024-04-25T00:30:48.541263+0000","last_active":"2024-04-25T00:33:18.040282+0000","last_peered":"2024-04-25T00:33:18.040282+0000","last_clean":"2024-04-25T00:33:18.040282+0000","last_became_active":"2024-04-25T00:30:48.540889+0000","last_became_peered":"2024-04-25T00:30:48.540889+0000","last_unstale":"2024-04-25T00:33:18.040282+0000","last_undegraded":"2024-04-25T00:33:18.040282+0000","last_fullsized":"2024-04-25T00:33:18.040282+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:31:42.067004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,4],"acting":[10,9,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"91'5","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055606+0000","last_change":"2024-04-25T00:30:48.533194+0000","last_active":"2024-04-25T00:33:18.055606+0000","last_peered":"2024-04-25T00:33:18.055606+0000","last_clean":"2024-04-25T00:33:18.055606+0000","last_became_active":"2024-04-25T00:30:48.532424+0000","last_became_peered":"2024-04-25T00:30:48.532424+0000","last_unstale":"2024-04-25T00:33:18.055606+0000","last_undegraded":"2024-04-25T00:33:18.055606+0000","last_fullsized":"2024-04-25T00:33:18.055606+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:06:50.187796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"91'9","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.924700+0000","last_change":"2024-04-25T00:30:48.542829+0000","last_active":"2024-04-25T00:33:17.924700+0000","last_peered":"2024-04-25T00:33:17.924700+0000","last_clean":"2024-04-25T00:33:17.924700+0000","last_became_active":"2024-04-25T00:30:48.542204+0000","last_became_peered":"2024-04-25T00:30:48.542204+0000","last_unstale":"2024-04-25T00:33:17.924700+0000","last_undegraded":"2024-04-25T00:33:17.924700+0000","last_fullsized":"2024-04-25T00:33:17.924700+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:56:47.869075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"91'11","reported_seq":43,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.188149+0000","last_change":"2024-04-25T00:30:48.537924+0000","last_active":"2024-04-25T00:33:18.188149+0000","last_peered":"2024-04-25T00:33:18.188149+0000","last_clean":"2024-04-25T00:33:18.188149+0000","last_became_active":"2024-04-25T00:30:48.537547+0000","last_became_peered":"2024-04-25T00:30:48.537547+0000","last_unstale":"2024-04-25T00:33:18.188149+0000","last_undegraded":"2024-04-25T00:33:18.188149+0000","last_fullsized":"2024-04-25T00:33:18.188149+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:26:45.590923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"91'16","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.040641+0000","last_change":"2024-04-25T00:30:48.539717+0000","last_active":"2024-04-25T00:33:18.040641+0000","last_peered":"2024-04-25T00:33:18.040641+0000","last_clean":"2024-04-25T00:33:18.040641+0000","last_became_active":"2024-04-25T00:30:48.539242+0000","last_became_peered":"2024-04-25T00:30:48.539242+0000","last_unstale":"2024-04-25T00:33:18.040641+0000","last_undegraded":"2024-04-25T00:33:18.040641+0000","last_fullsized":"2024-04-25T00:33:18.040641+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":16,"log_dups_size":0,"ondisk_log_size":16,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:58:15.353959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"91'6","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055839+0000","last_change":"2024-04-25T00:30:48.542767+0000","last_active":"2024-04-25T00:33:18.055839+0000","last_peered":"2024-04-25T00:33:18.055839+0000","last_clean":"2024-04-25T00:33:18.055839+0000","last_became_active":"2024-04-25T00:30:48.542252+0000","last_became_peered":"2024-04-25T00:30:48.542252+0000","last_unstale":"2024-04-25T00:33:18.055839+0000","last_undegraded":"2024-04-25T00:33:18.055839+0000","last_fullsized":"2024-04-25T00:33:18.055839+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:07:49.414660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"91'7","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926388+0000","last_change":"2024-04-25T00:30:48.532248+0000","last_active":"2024-04-25T00:33:17.926388+0000","last_peered":"2024-04-25T00:33:17.926388+0000","last_clean":"2024-04-25T00:33:17.926388+0000","last_became_active":"2024-04-25T00:30:48.531876+0000","last_became_peered":"2024-04-25T00:30:48.531876+0000","last_unstale":"2024-04-25T00:33:17.926388+0000","last_undegraded":"2024-04-25T00:33:17.926388+0000","last_fullsized":"2024-04-25T00:33:17.926388+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:00:35.711693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"91'10","reported_seq":42,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.167693+0000","last_change":"2024-04-25T00:30:48.526871+0000","last_active":"2024-04-25T00:33:18.167693+0000","last_peered":"2024-04-25T00:33:18.167693+0000","last_clean":"2024-04-25T00:33:18.167693+0000","last_became_active":"2024-04-25T00:30:48.526477+0000","last_became_peered":"2024-04-25T00:30:48.526477+0000","last_unstale":"2024-04-25T00:33:18.167693+0000","last_undegraded":"2024-04-25T00:33:18.167693+0000","last_fullsized":"2024-04-25T00:33:18.167693+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:17:59.447932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"91'7","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305099+0000","last_change":"2024-04-25T00:30:48.530297+0000","last_active":"2024-04-25T00:33:18.305099+0000","last_peered":"2024-04-25T00:33:18.305099+0000","last_clean":"2024-04-25T00:33:18.305099+0000","last_became_active":"2024-04-25T00:30:48.529091+0000","last_became_peered":"2024-04-25T00:30:48.529091+0000","last_unstale":"2024-04-25T00:33:18.305099+0000","last_undegraded":"2024-04-25T00:33:18.305099+0000","last_fullsized":"2024-04-25T00:33:18.305099+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:54:52.206881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"91'7","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305119+0000","last_change":"2024-04-25T00:30:48.535629+0000","last_active":"2024-04-25T00:33:18.305119+0000","last_peered":"2024-04-25T00:33:18.305119+0000","last_clean":"2024-04-25T00:33:18.305119+0000","last_became_active":"2024-04-25T00:30:48.535190+0000","last_became_peered":"2024-04-25T00:30:48.535190+0000","last_unstale":"2024-04-25T00:33:18.305119+0000","last_undegraded":"2024-04-25T00:33:18.305119+0000","last_fullsized":"2024-04-25T00:33:18.305119+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:11:27.127964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"91'9","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.187509+0000","last_change":"2024-04-25T00:30:48.532988+0000","last_active":"2024-04-25T00:33:18.187509+0000","last_peered":"2024-04-25T00:33:18.187509+0000","last_clean":"2024-04-25T00:33:18.187509+0000","last_became_active":"2024-04-25T00:30:48.532670+0000","last_became_peered":"2024-04-25T00:30:48.532670+0000","last_unstale":"2024-04-25T00:33:18.187509+0000","last_undegraded":"2024-04-25T00:33:18.187509+0000","last_fullsized":"2024-04-25T00:33:18.187509+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:03:35.336175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305254+0000","last_change":"2024-04-25T00:30:48.539499+0000","last_active":"2024-04-25T00:33:18.305254+0000","last_peered":"2024-04-25T00:33:18.305254+0000","last_clean":"2024-04-25T00:33:18.305254+0000","last_became_active":"2024-04-25T00:30:48.538516+0000","last_became_peered":"2024-04-25T00:30:48.538516+0000","last_unstale":"2024-04-25T00:33:18.305254+0000","last_undegraded":"2024-04-25T00:33:18.305254+0000","last_fullsized":"2024-04-25T00:33:18.305254+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:03:16.781091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"91'10","reported_seq":42,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.028510+0000","last_change":"2024-04-25T00:30:48.534204+0000","last_active":"2024-04-25T00:33:18.028510+0000","last_peered":"2024-04-25T00:33:18.028510+0000","last_clean":"2024-04-25T00:33:18.028510+0000","last_became_active":"2024-04-25T00:30:48.533818+0000","last_became_peered":"2024-04-25T00:30:48.533818+0000","last_unstale":"2024-04-25T00:33:18.028510+0000","last_undegraded":"2024-04-25T00:33:18.028510+0000","last_fullsized":"2024-04-25T00:33:18.028510+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:08:42.964056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"91'7","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.927103+0000","last_change":"2024-04-25T00:30:48.542756+0000","last_active":"2024-04-25T00:33:17.927103+0000","last_peered":"2024-04-25T00:33:17.927103+0000","last_clean":"2024-04-25T00:33:17.927103+0000","last_became_active":"2024-04-25T00:30:48.541824+0000","last_became_peered":"2024-04-25T00:30:48.541824+0000","last_unstale":"2024-04-25T00:33:17.927103+0000","last_undegraded":"2024-04-25T00:33:17.927103+0000","last_fullsized":"2024-04-25T00:33:17.927103+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:31:40.996474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926670+0000","last_change":"2024-04-25T00:30:52.788477+0000","last_active":"2024-04-25T00:33:17.926670+0000","last_peered":"2024-04-25T00:33:17.926670+0000","last_clean":"2024-04-25T00:33:17.926670+0000","last_became_active":"2024-04-25T00:30:52.788073+0000","last_became_peered":"2024-04-25T00:30:52.788073+0000","last_unstale":"2024-04-25T00:33:17.926670+0000","last_undegraded":"2024-04-25T00:33:17.926670+0000","last_fullsized":"2024-04-25T00:33:17.926670+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:29:34.781119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,9,0],"acting":[2,3,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"89'9","reported_seq":44,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.028297+0000","last_change":"2024-04-25T00:30:47.524580+0000","last_active":"2024-04-25T00:33:18.028297+0000","last_peered":"2024-04-25T00:33:18.028297+0000","last_clean":"2024-04-25T00:33:18.028297+0000","last_became_active":"2024-04-25T00:30:47.524170+0000","last_became_peered":"2024-04-25T00:30:47.524170+0000","last_unstale":"2024-04-25T00:33:18.028297+0000","last_undegraded":"2024-04-25T00:33:18.028297+0000","last_fullsized":"2024-04-25T00:33:18.028297+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:53:08.734842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"91'5","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.925270+0000","last_change":"2024-04-25T00:30:48.542077+0000","last_active":"2024-04-25T00:33:17.925270+0000","last_peered":"2024-04-25T00:33:17.925270+0000","last_clean":"2024-04-25T00:33:17.925270+0000","last_became_active":"2024-04-25T00:30:48.539043+0000","last_became_peered":"2024-04-25T00:30:48.539043+0000","last_unstale":"2024-04-25T00:33:17.925270+0000","last_undegraded":"2024-04-25T00:33:17.925270+0000","last_fullsized":"2024-04-25T00:33:17.925270+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:48:52.254800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926432+0000","last_change":"2024-04-25T00:30:52.786091+0000","last_active":"2024-04-25T00:33:17.926432+0000","last_peered":"2024-04-25T00:33:17.926432+0000","last_clean":"2024-04-25T00:33:17.926432+0000","last_became_active":"2024-04-25T00:30:52.785511+0000","last_became_peered":"2024-04-25T00:30:52.785511+0000","last_unstale":"2024-04-25T00:33:17.926432+0000","last_undegraded":"2024-04-25T00:33:17.926432+0000","last_fullsized":"2024-04-25T00:33:17.926432+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:19:17.992033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,9,3],"acting":[4,6,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1e","version":"89'2","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.925488+0000","last_change":"2024-04-25T00:30:47.529293+0000","last_active":"2024-04-25T00:33:17.925488+0000","last_peered":"2024-04-25T00:33:17.925488+0000","last_clean":"2024-04-25T00:33:17.925488+0000","last_became_active":"2024-04-25T00:30:47.527975+0000","last_became_peered":"2024-04-25T00:30:47.527975+0000","last_unstale":"2024-04-25T00:33:17.925488+0000","last_undegraded":"2024-04-25T00:33:17.925488+0000","last_fullsized":"2024-04-25T00:33:17.925488+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:17:40.057962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"91'6","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305797+0000","last_change":"2024-04-25T00:30:48.530993+0000","last_active":"2024-04-25T00:33:18.305797+0000","last_peered":"2024-04-25T00:33:18.305797+0000","last_clean":"2024-04-25T00:33:18.305797+0000","last_became_active":"2024-04-25T00:30:48.530550+0000","last_became_peered":"2024-04-25T00:30:48.530550+0000","last_unstale":"2024-04-25T00:33:18.305797+0000","last_undegraded":"2024-04-25T00:33:18.305797+0000","last_fullsized":"2024-04-25T00:33:18.305797+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:11:35.308456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":24,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926274+0000","last_change":"2024-04-25T00:30:52.787774+0000","last_active":"2024-04-25T00:33:17.926274+0000","last_peered":"2024-04-25T00:33:17.926274+0000","last_clean":"2024-04-25T00:33:17.926274+0000","last_became_active":"2024-04-25T00:30:52.786988+0000","last_became_peered":"2024-04-25T00:30:52.786988+0000","last_unstale":"2024-04-25T00:33:17.926274+0000","last_undegraded":"2024-04-25T00:33:17.926274+0000","last_fullsized":"2024-04-25T00:33:17.926274+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:58:28.081767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,9,5],"acting":[4,3,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1d","version":"89'4","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.030776+0000","last_change":"2024-04-25T00:30:47.523477+0000","last_active":"2024-04-25T00:33:18.030776+0000","last_peered":"2024-04-25T00:33:18.030776+0000","last_clean":"2024-04-25T00:33:18.030776+0000","last_became_active":"2024-04-25T00:30:47.522961+0000","last_became_peered":"2024-04-25T00:30:47.522961+0000","last_unstale":"2024-04-25T00:33:18.030776+0000","last_undegraded":"2024-04-25T00:33:18.030776+0000","last_fullsized":"2024-04-25T00:33:18.030776+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:39:39.793477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"91'9","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.187779+0000","last_change":"2024-04-25T00:30:48.527746+0000","last_active":"2024-04-25T00:33:18.187779+0000","last_peered":"2024-04-25T00:33:18.187779+0000","last_clean":"2024-04-25T00:33:18.187779+0000","last_became_active":"2024-04-25T00:30:48.527374+0000","last_became_peered":"2024-04-25T00:30:48.527374+0000","last_unstale":"2024-04-25T00:33:18.187779+0000","last_undegraded":"2024-04-25T00:33:18.187779+0000","last_fullsized":"2024-04-25T00:33:18.187779+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:28:36.822220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926088+0000","last_change":"2024-04-25T00:30:52.782368+0000","last_active":"2024-04-25T00:33:17.926088+0000","last_peered":"2024-04-25T00:33:17.926088+0000","last_clean":"2024-04-25T00:33:17.926088+0000","last_became_active":"2024-04-25T00:30:52.781875+0000","last_became_peered":"2024-04-25T00:30:52.781875+0000","last_unstale":"2024-04-25T00:33:17.926088+0000","last_undegraded":"2024-04-25T00:33:17.926088+0000","last_fullsized":"2024-04-25T00:33:17.926088+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:07:55.509131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5,3],"acting":[4,6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1c","version":"84'2","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.031019+0000","last_change":"2024-04-25T00:30:47.523707+0000","last_active":"2024-04-25T00:33:18.031019+0000","last_peered":"2024-04-25T00:33:18.031019+0000","last_clean":"2024-04-25T00:33:18.031019+0000","last_became_active":"2024-04-25T00:30:47.523216+0000","last_became_peered":"2024-04-25T00:30:47.523216+0000","last_unstale":"2024-04-25T00:33:18.031019+0000","last_undegraded":"2024-04-25T00:33:18.031019+0000","last_fullsized":"2024-04-25T00:33:18.031019+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:21:21.006368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"91'15","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.037238+0000","last_change":"2024-04-25T00:30:48.529235+0000","last_active":"2024-04-25T00:33:18.037238+0000","last_peered":"2024-04-25T00:33:18.037238+0000","last_clean":"2024-04-25T00:33:18.037238+0000","last_became_active":"2024-04-25T00:30:48.528887+0000","last_became_peered":"2024-04-25T00:30:48.528887+0000","last_unstale":"2024-04-25T00:33:18.037238+0000","last_undegraded":"2024-04-25T00:33:18.037238+0000","last_fullsized":"2024-04-25T00:33:18.037238+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:18:05.290578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":31,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926986+0000","last_change":"2024-04-25T00:31:44.714433+0000","last_active":"2024-04-25T00:33:17.926986+0000","last_peered":"2024-04-25T00:33:17.926986+0000","last_clean":"2024-04-25T00:33:17.926986+0000","last_became_active":"2024-04-25T00:31:44.713810+0000","last_became_peered":"2024-04-25T00:31:44.713810+0000","last_unstale":"2024-04-25T00:33:17.926986+0000","last_undegraded":"2024-04-25T00:33:17.926986+0000","last_fullsized":"2024-04-25T00:33:17.926986+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:00:49.815134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,7,11],"acting":[2,8,7,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1b","version":"94'6","reported_seq":166,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.185075+0000","last_change":"2024-04-25T00:30:47.518472+0000","last_active":"2024-04-25T00:33:18.185075+0000","last_peered":"2024-04-25T00:33:18.185075+0000","last_clean":"2024-04-25T00:33:18.185075+0000","last_became_active":"2024-04-25T00:30:47.518140+0000","last_became_peered":"2024-04-25T00:30:47.518140+0000","last_unstale":"2024-04-25T00:33:18.185075+0000","last_undegraded":"2024-04-25T00:33:18.185075+0000","last_fullsized":"2024-04-25T00:33:18.185075+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:19:07.780949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1013,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":2,"num_write":3,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"91'7","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.030957+0000","last_change":"2024-04-25T00:30:48.533757+0000","last_active":"2024-04-25T00:33:18.030957+0000","last_peered":"2024-04-25T00:33:18.030957+0000","last_clean":"2024-04-25T00:33:18.030957+0000","last_became_active":"2024-04-25T00:30:48.533421+0000","last_became_peered":"2024-04-25T00:30:48.533421+0000","last_unstale":"2024-04-25T00:33:18.030957+0000","last_undegraded":"2024-04-25T00:33:18.030957+0000","last_fullsized":"2024-04-25T00:33:18.030957+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:11:38.190783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":31,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926818+0000","last_change":"2024-04-25T00:31:45.031412+0000","last_active":"2024-04-25T00:33:17.926818+0000","last_peered":"2024-04-25T00:33:17.926818+0000","last_clean":"2024-04-25T00:33:17.926818+0000","last_became_active":"2024-04-25T00:31:45.030822+0000","last_became_peered":"2024-04-25T00:31:45.030822+0000","last_unstale":"2024-04-25T00:33:17.926818+0000","last_undegraded":"2024-04-25T00:33:17.926818+0000","last_fullsized":"2024-04-25T00:33:17.926818+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:34:26.758966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,8,5],"acting":[2,6,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1a","version":"89'3","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.040284+0000","last_change":"2024-04-25T00:30:47.527306+0000","last_active":"2024-04-25T00:33:18.040284+0000","last_peered":"2024-04-25T00:33:18.040284+0000","last_clean":"2024-04-25T00:33:18.040284+0000","last_became_active":"2024-04-25T00:30:47.526935+0000","last_became_peered":"2024-04-25T00:30:47.526935+0000","last_unstale":"2024-04-25T00:33:18.040284+0000","last_undegraded":"2024-04-25T00:33:18.040284+0000","last_fullsized":"2024-04-25T00:33:18.040284+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:41:37.951143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"91'8","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.027475+0000","last_change":"2024-04-25T00:31:45.170610+0000","last_active":"2024-04-25T00:33:18.027475+0000","last_peered":"2024-04-25T00:33:18.027475+0000","last_clean":"2024-04-25T00:33:18.027475+0000","last_became_active":"2024-04-25T00:31:45.170183+0000","last_became_peered":"2024-04-25T00:31:45.170183+0000","last_unstale":"2024-04-25T00:33:18.027475+0000","last_undegraded":"2024-04-25T00:33:18.027475+0000","last_fullsized":"2024-04-25T00:33:18.027475+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:39:39.781997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,4,3],"acting":[9,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305164+0000","last_change":"2024-04-25T00:30:52.784986+0000","last_active":"2024-04-25T00:33:18.305164+0000","last_peered":"2024-04-25T00:33:18.305164+0000","last_clean":"2024-04-25T00:33:18.305164+0000","last_became_active":"2024-04-25T00:30:52.784593+0000","last_became_peered":"2024-04-25T00:30:52.784593+0000","last_unstale":"2024-04-25T00:33:18.305164+0000","last_undegraded":"2024-04-25T00:33:18.305164+0000","last_fullsized":"2024-04-25T00:33:18.305164+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:04:05.833730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,9,0],"acting":[11,4,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.19","version":"89'2","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.166738+0000","last_change":"2024-04-25T00:30:47.523663+0000","last_active":"2024-04-25T00:33:18.166738+0000","last_peered":"2024-04-25T00:33:18.166738+0000","last_clean":"2024-04-25T00:33:18.166738+0000","last_became_active":"2024-04-25T00:30:47.523324+0000","last_became_peered":"2024-04-25T00:30:47.523324+0000","last_unstale":"2024-04-25T00:33:18.166738+0000","last_undegraded":"2024-04-25T00:33:18.166738+0000","last_fullsized":"2024-04-25T00:33:18.166738+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:51:23.593579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.924628+0000","last_change":"2024-04-25T00:30:48.529738+0000","last_active":"2024-04-25T00:33:17.924628+0000","last_peered":"2024-04-25T00:33:17.924628+0000","last_clean":"2024-04-25T00:33:17.924628+0000","last_became_active":"2024-04-25T00:30:48.529350+0000","last_became_peered":"2024-04-25T00:30:48.529350+0000","last_unstale":"2024-04-25T00:33:17.924628+0000","last_undegraded":"2024-04-25T00:33:17.924628+0000","last_fullsized":"2024-04-25T00:33:17.924628+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:51:29.238293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305792+0000","last_change":"2024-04-25T00:30:52.788480+0000","last_active":"2024-04-25T00:33:18.305792+0000","last_peered":"2024-04-25T00:33:18.305792+0000","last_clean":"2024-04-25T00:33:18.305792+0000","last_became_active":"2024-04-25T00:30:52.788065+0000","last_became_peered":"2024-04-25T00:30:52.788065+0000","last_unstale":"2024-04-25T00:33:18.305792+0000","last_undegraded":"2024-04-25T00:33:18.305792+0000","last_fullsized":"2024-04-25T00:33:18.305792+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:16:50.179177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6,1],"acting":[0,7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.18","version":"89'1","reported_seq":35,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.036951+0000","last_change":"2024-04-25T00:30:47.524706+0000","last_active":"2024-04-25T00:33:18.036951+0000","last_peered":"2024-04-25T00:33:18.036951+0000","last_clean":"2024-04-25T00:33:18.036951+0000","last_became_active":"2024-04-25T00:30:47.524355+0000","last_became_peered":"2024-04-25T00:30:47.524355+0000","last_unstale":"2024-04-25T00:33:18.036951+0000","last_undegraded":"2024-04-25T00:33:18.036951+0000","last_fullsized":"2024-04-25T00:33:18.036951+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:22:38.093362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"91'11","reported_seq":43,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.027727+0000","last_change":"2024-04-25T00:30:48.533266+0000","last_active":"2024-04-25T00:33:18.027727+0000","last_peered":"2024-04-25T00:33:18.027727+0000","last_clean":"2024-04-25T00:33:18.027727+0000","last_became_active":"2024-04-25T00:30:48.532932+0000","last_became_peered":"2024-04-25T00:30:48.532932+0000","last_unstale":"2024-04-25T00:33:18.027727+0000","last_undegraded":"2024-04-25T00:33:18.027727+0000","last_fullsized":"2024-04-25T00:33:18.027727+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:00:54.147725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.924818+0000","last_change":"2024-04-25T00:30:52.780317+0000","last_active":"2024-04-25T00:33:17.924818+0000","last_peered":"2024-04-25T00:33:17.924818+0000","last_clean":"2024-04-25T00:33:17.924818+0000","last_became_active":"2024-04-25T00:30:52.778539+0000","last_became_peered":"2024-04-25T00:30:52.778539+0000","last_unstale":"2024-04-25T00:33:17.924818+0000","last_undegraded":"2024-04-25T00:33:17.924818+0000","last_fullsized":"2024-04-25T00:33:17.924818+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:34:49.527307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1,5],"acting":[3,7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.17","version":"94'4","reported_seq":162,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.185182+0000","last_change":"2024-04-25T00:30:47.525647+0000","last_active":"2024-04-25T00:33:18.185182+0000","last_peered":"2024-04-25T00:33:18.185182+0000","last_clean":"2024-04-25T00:33:18.185182+0000","last_became_active":"2024-04-25T00:30:47.525280+0000","last_became_peered":"2024-04-25T00:30:47.525280+0000","last_unstale":"2024-04-25T00:33:18.185182+0000","last_undegraded":"2024-04-25T00:33:18.185182+0000","last_fullsized":"2024-04-25T00:33:18.185182+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:56:15.761173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":124,"num_read_kb":124,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"91'10","reported_seq":42,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.188103+0000","last_change":"2024-04-25T00:30:48.527913+0000","last_active":"2024-04-25T00:33:18.188103+0000","last_peered":"2024-04-25T00:33:18.188103+0000","last_clean":"2024-04-25T00:33:18.188103+0000","last_became_active":"2024-04-25T00:30:48.527538+0000","last_became_peered":"2024-04-25T00:30:48.527538+0000","last_unstale":"2024-04-25T00:33:18.188103+0000","last_undegraded":"2024-04-25T00:33:18.188103+0000","last_fullsized":"2024-04-25T00:33:18.188103+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:48:38.646867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.027433+0000","last_change":"2024-04-25T00:30:52.788267+0000","last_active":"2024-04-25T00:33:18.027433+0000","last_peered":"2024-04-25T00:33:18.027433+0000","last_clean":"2024-04-25T00:33:18.027433+0000","last_became_active":"2024-04-25T00:30:52.785188+0000","last_became_peered":"2024-04-25T00:30:52.785188+0000","last_unstale":"2024-04-25T00:33:18.027433+0000","last_undegraded":"2024-04-25T00:33:18.027433+0000","last_fullsized":"2024-04-25T00:33:18.027433+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:38:25.194673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,0,2,4],"acting":[9,0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.16","version":"89'2","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055922+0000","last_change":"2024-04-25T00:30:47.522213+0000","last_active":"2024-04-25T00:33:18.055922+0000","last_peered":"2024-04-25T00:33:18.055922+0000","last_clean":"2024-04-25T00:33:18.055922+0000","last_became_active":"2024-04-25T00:30:47.521519+0000","last_became_peered":"2024-04-25T00:30:47.521519+0000","last_unstale":"2024-04-25T00:33:18.055922+0000","last_undegraded":"2024-04-25T00:33:18.055922+0000","last_fullsized":"2024-04-25T00:33:18.055922+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:16:10.001222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"91'9","reported_seq":47,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.306004+0000","last_change":"2024-04-25T00:31:45.029866+0000","last_active":"2024-04-25T00:33:18.306004+0000","last_peered":"2024-04-25T00:33:18.306004+0000","last_clean":"2024-04-25T00:33:18.306004+0000","last_became_active":"2024-04-25T00:31:45.029587+0000","last_became_peered":"2024-04-25T00:31:45.029587+0000","last_unstale":"2024-04-25T00:33:18.306004+0000","last_undegraded":"2024-04-25T00:33:18.306004+0000","last_fullsized":"2024-04-25T00:33:18.306004+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:09:09.434492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926486+0000","last_change":"2024-04-25T00:30:52.786818+0000","last_active":"2024-04-25T00:33:17.926486+0000","last_peered":"2024-04-25T00:33:17.926486+0000","last_clean":"2024-04-25T00:33:17.926486+0000","last_became_active":"2024-04-25T00:30:52.786293+0000","last_became_peered":"2024-04-25T00:30:52.786293+0000","last_unstale":"2024-04-25T00:33:17.926486+0000","last_undegraded":"2024-04-25T00:33:17.926486+0000","last_fullsized":"2024-04-25T00:33:17.926486+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:27:02.751296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2,9],"acting":[4,1,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.15","version":"89'4","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.037256+0000","last_change":"2024-04-25T00:30:47.520882+0000","last_active":"2024-04-25T00:33:18.037256+0000","last_peered":"2024-04-25T00:33:18.037256+0000","last_clean":"2024-04-25T00:33:18.037256+0000","last_became_active":"2024-04-25T00:30:47.520545+0000","last_became_peered":"2024-04-25T00:30:47.520545+0000","last_unstale":"2024-04-25T00:33:18.037256+0000","last_undegraded":"2024-04-25T00:33:18.037256+0000","last_fullsized":"2024-04-25T00:33:18.037256+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:46:04.715580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"91'11","reported_seq":43,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.187850+0000","last_change":"2024-04-25T00:30:48.536814+0000","last_active":"2024-04-25T00:33:18.187850+0000","last_peered":"2024-04-25T00:33:18.187850+0000","last_clean":"2024-04-25T00:33:18.187850+0000","last_became_active":"2024-04-25T00:30:48.536486+0000","last_became_peered":"2024-04-25T00:30:48.536486+0000","last_unstale":"2024-04-25T00:33:18.187850+0000","last_undegraded":"2024-04-25T00:33:18.187850+0000","last_fullsized":"2024-04-25T00:33:18.187850+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:58:23.861959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":18,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.040101+0000","last_change":"2024-04-25T00:31:45.031208+0000","last_active":"2024-04-25T00:33:18.040101+0000","last_peered":"2024-04-25T00:33:18.040101+0000","last_clean":"2024-04-25T00:33:18.040101+0000","last_became_active":"2024-04-25T00:31:45.030831+0000","last_became_peered":"2024-04-25T00:31:45.030831+0000","last_unstale":"2024-04-25T00:33:18.040101+0000","last_undegraded":"2024-04-25T00:33:18.040101+0000","last_fullsized":"2024-04-25T00:33:18.040101+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:42:01.869146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,11,5],"acting":[10,8,11,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.14","version":"94'39","reported_seq":331,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.187222+0000","last_change":"2024-04-25T00:30:47.526899+0000","last_active":"2024-04-25T00:33:18.187222+0000","last_peered":"2024-04-25T00:33:18.187222+0000","last_clean":"2024-04-25T00:33:18.187222+0000","last_became_active":"2024-04-25T00:30:47.526528+0000","last_became_peered":"2024-04-25T00:30:47.526528+0000","last_unstale":"2024-04-25T00:33:18.187222+0000","last_undegraded":"2024-04-25T00:33:18.187222+0000","last_fullsized":"2024-04-25T00:33:18.187222+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":39,"log_dups_size":0,"ondisk_log_size":39,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:13:34.004584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67428,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":258,"num_read_kb":195,"num_write":40,"num_write_kb":86,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"91'6","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.030904+0000","last_change":"2024-04-25T00:30:48.538225+0000","last_active":"2024-04-25T00:33:18.030904+0000","last_peered":"2024-04-25T00:33:18.030904+0000","last_clean":"2024-04-25T00:33:18.030904+0000","last_became_active":"2024-04-25T00:30:48.537807+0000","last_became_peered":"2024-04-25T00:30:48.537807+0000","last_unstale":"2024-04-25T00:33:18.030904+0000","last_undegraded":"2024-04-25T00:33:18.030904+0000","last_fullsized":"2024-04-25T00:33:18.030904+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:00:46.275584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.037669+0000","last_change":"2024-04-25T00:30:52.782266+0000","last_active":"2024-04-25T00:33:18.037669+0000","last_peered":"2024-04-25T00:33:18.037669+0000","last_clean":"2024-04-25T00:33:18.037669+0000","last_became_active":"2024-04-25T00:30:52.781133+0000","last_became_peered":"2024-04-25T00:30:52.781133+0000","last_unstale":"2024-04-25T00:33:18.037669+0000","last_undegraded":"2024-04-25T00:33:18.037669+0000","last_fullsized":"2024-04-25T00:33:18.037669+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:19:03.818560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,8,3,6],"acting":[5,8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.13","version":"89'6","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926797+0000","last_change":"2024-04-25T00:30:47.520909+0000","last_active":"2024-04-25T00:33:17.926797+0000","last_peered":"2024-04-25T00:33:17.926797+0000","last_clean":"2024-04-25T00:33:17.926797+0000","last_became_active":"2024-04-25T00:30:47.520287+0000","last_became_peered":"2024-04-25T00:30:47.520287+0000","last_unstale":"2024-04-25T00:33:17.926797+0000","last_undegraded":"2024-04-25T00:33:17.926797+0000","last_fullsized":"2024-04-25T00:33:17.926797+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:49:43.795115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"91'9","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.167123+0000","last_change":"2024-04-25T00:30:48.532270+0000","last_active":"2024-04-25T00:33:18.167123+0000","last_peered":"2024-04-25T00:33:18.167123+0000","last_clean":"2024-04-25T00:33:18.167123+0000","last_became_active":"2024-04-25T00:30:48.531882+0000","last_became_peered":"2024-04-25T00:30:48.531882+0000","last_unstale":"2024-04-25T00:33:18.167123+0000","last_undegraded":"2024-04-25T00:33:18.167123+0000","last_fullsized":"2024-04-25T00:33:18.167123+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:58:17.865635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.304924+0000","last_change":"2024-04-25T00:30:52.777708+0000","last_active":"2024-04-25T00:33:18.304924+0000","last_peered":"2024-04-25T00:33:18.304924+0000","last_clean":"2024-04-25T00:33:18.304924+0000","last_became_active":"2024-04-25T00:30:52.777254+0000","last_became_peered":"2024-04-25T00:30:52.777254+0000","last_unstale":"2024-04-25T00:33:18.304924+0000","last_undegraded":"2024-04-25T00:33:18.304924+0000","last_fullsized":"2024-04-25T00:33:18.304924+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:29:47.427190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,9,7,6],"acting":[11,9,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.12","version":"89'3","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.037312+0000","last_change":"2024-04-25T00:30:47.524667+0000","last_active":"2024-04-25T00:33:18.037312+0000","last_peered":"2024-04-25T00:33:18.037312+0000","last_clean":"2024-04-25T00:33:18.037312+0000","last_became_active":"2024-04-25T00:30:47.524361+0000","last_became_peered":"2024-04-25T00:30:47.524361+0000","last_unstale":"2024-04-25T00:33:18.037312+0000","last_undegraded":"2024-04-25T00:33:18.037312+0000","last_fullsized":"2024-04-25T00:33:18.037312+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:16:08.805720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"91'15","reported_seq":47,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055606+0000","last_change":"2024-04-25T00:30:48.538658+0000","last_active":"2024-04-25T00:33:18.055606+0000","last_peered":"2024-04-25T00:33:18.055606+0000","last_clean":"2024-04-25T00:33:18.055606+0000","last_became_active":"2024-04-25T00:30:48.538316+0000","last_became_peered":"2024-04-25T00:30:48.538316+0000","last_unstale":"2024-04-25T00:33:18.055606+0000","last_undegraded":"2024-04-25T00:33:18.055606+0000","last_fullsized":"2024-04-25T00:33:18.055606+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:24:47.303559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":24,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.028189+0000","last_change":"2024-04-25T00:30:52.789512+0000","last_active":"2024-04-25T00:33:18.028189+0000","last_peered":"2024-04-25T00:33:18.028189+0000","last_clean":"2024-04-25T00:33:18.028189+0000","last_became_active":"2024-04-25T00:30:52.788929+0000","last_became_peered":"2024-04-25T00:30:52.788929+0000","last_unstale":"2024-04-25T00:33:18.028189+0000","last_undegraded":"2024-04-25T00:33:18.028189+0000","last_fullsized":"2024-04-25T00:33:18.028189+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:21:05.195296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,0,3],"acting":[9,7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.11","version":"94'8","reported_seq":171,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.185195+0000","last_change":"2024-04-25T00:30:47.513627+0000","last_active":"2024-04-25T00:33:18.185195+0000","last_peered":"2024-04-25T00:33:18.185195+0000","last_clean":"2024-04-25T00:33:18.185195+0000","last_became_active":"2024-04-25T00:30:47.513215+0000","last_became_peered":"2024-04-25T00:30:47.513215+0000","last_unstale":"2024-04-25T00:33:18.185195+0000","last_undegraded":"2024-04-25T00:33:18.185195+0000","last_fullsized":"2024-04-25T00:33:18.185195+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:32:16.111590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2210,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":130,"num_read_kb":6,"num_write":7,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"91'6","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055536+0000","last_change":"2024-04-25T00:30:48.533056+0000","last_active":"2024-04-25T00:33:18.055536+0000","last_peered":"2024-04-25T00:33:18.055536+0000","last_clean":"2024-04-25T00:33:18.055536+0000","last_became_active":"2024-04-25T00:30:48.532698+0000","last_became_peered":"2024-04-25T00:30:48.532698+0000","last_unstale":"2024-04-25T00:33:18.055536+0000","last_undegraded":"2024-04-25T00:33:18.055536+0000","last_fullsized":"2024-04-25T00:33:18.055536+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:36:24.987784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.167620+0000","last_change":"2024-04-25T00:30:52.780669+0000","last_active":"2024-04-25T00:33:18.167620+0000","last_peered":"2024-04-25T00:33:18.167620+0000","last_clean":"2024-04-25T00:33:18.167620+0000","last_became_active":"2024-04-25T00:30:52.780222+0000","last_became_peered":"2024-04-25T00:30:52.780222+0000","last_unstale":"2024-04-25T00:33:18.167620+0000","last_undegraded":"2024-04-25T00:33:18.167620+0000","last_fullsized":"2024-04-25T00:33:18.167620+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:50:27.415077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,9,11],"acting":[6,3,9,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"89'15","reported_seq":74,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.030496+0000","last_change":"2024-04-25T00:32:45.486412+0000","last_active":"2024-04-25T00:33:18.030496+0000","last_peered":"2024-04-25T00:33:18.030496+0000","last_clean":"2024-04-25T00:33:18.030496+0000","last_became_active":"2024-04-25T00:32:45.474176+0000","last_became_peered":"2024-04-25T00:32:45.474176+0000","last_unstale":"2024-04-25T00:33:18.030496+0000","last_undegraded":"2024-04-25T00:33:18.030496+0000","last_fullsized":"2024-04-25T00:33:18.030496+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:59:50.867374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":112,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,4],"acting":[8,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"91'6","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.037133+0000","last_change":"2024-04-25T00:30:48.534363+0000","last_active":"2024-04-25T00:33:18.037133+0000","last_peered":"2024-04-25T00:33:18.037133+0000","last_clean":"2024-04-25T00:33:18.037133+0000","last_became_active":"2024-04-25T00:30:48.533855+0000","last_became_peered":"2024-04-25T00:30:48.533855+0000","last_unstale":"2024-04-25T00:33:18.037133+0000","last_undegraded":"2024-04-25T00:33:18.037133+0000","last_fullsized":"2024-04-25T00:33:18.037133+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:28:29.374501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.037011+0000","last_change":"2024-04-25T00:30:52.790323+0000","last_active":"2024-04-25T00:33:18.037011+0000","last_peered":"2024-04-25T00:33:18.037011+0000","last_clean":"2024-04-25T00:33:18.037011+0000","last_became_active":"2024-04-25T00:30:52.789905+0000","last_became_peered":"2024-04-25T00:30:52.789905+0000","last_unstale":"2024-04-25T00:33:18.037011+0000","last_undegraded":"2024-04-25T00:33:18.037011+0000","last_fullsized":"2024-04-25T00:33:18.037011+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:19:03.687318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3,1],"acting":[5,0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"93'3","reported_seq":20,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305099+0000","last_change":"2024-04-25T00:32:45.490186+0000","last_active":"2024-04-25T00:33:18.305099+0000","last_peered":"2024-04-25T00:33:18.305099+0000","last_clean":"2024-04-25T00:33:18.305099+0000","last_became_active":"2024-04-25T00:32:45.474069+0000","last_became_peered":"2024-04-25T00:32:45.474069+0000","last_unstale":"2024-04-25T00:33:18.305099+0000","last_undegraded":"2024-04-25T00:33:18.305099+0000","last_fullsized":"2024-04-25T00:33:18.305099+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:58:34.104225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.e","version":"91'5","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305056+0000","last_change":"2024-04-25T00:30:48.539679+0000","last_active":"2024-04-25T00:33:18.305056+0000","last_peered":"2024-04-25T00:33:18.305056+0000","last_clean":"2024-04-25T00:33:18.305056+0000","last_became_active":"2024-04-25T00:30:48.538065+0000","last_became_peered":"2024-04-25T00:30:48.538065+0000","last_unstale":"2024-04-25T00:33:18.305056+0000","last_undegraded":"2024-04-25T00:33:18.305056+0000","last_fullsized":"2024-04-25T00:33:18.305056+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:05:16.435010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.030891+0000","last_change":"2024-04-25T00:30:52.804824+0000","last_active":"2024-04-25T00:33:18.030891+0000","last_peered":"2024-04-25T00:33:18.030891+0000","last_clean":"2024-04-25T00:33:18.030891+0000","last_became_active":"2024-04-25T00:30:52.804322+0000","last_became_peered":"2024-04-25T00:30:52.804322+0000","last_unstale":"2024-04-25T00:33:18.030891+0000","last_undegraded":"2024-04-25T00:33:18.030891+0000","last_fullsized":"2024-04-25T00:33:18.030891+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:38:08.385170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,10,11,9],"acting":[8,10,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":34,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.027849+0000","last_change":"2024-04-25T00:30:47.516916+0000","last_active":"2024-04-25T00:33:18.027849+0000","last_peered":"2024-04-25T00:33:18.027849+0000","last_clean":"2024-04-25T00:33:18.027849+0000","last_became_active":"2024-04-25T00:30:47.516520+0000","last_became_peered":"2024-04-25T00:30:47.516520+0000","last_unstale":"2024-04-25T00:33:18.027849+0000","last_undegraded":"2024-04-25T00:33:18.027849+0000","last_fullsized":"2024-04-25T00:33:18.027849+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:29:19.099764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"91'4","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.040501+0000","last_change":"2024-04-25T00:30:48.537584+0000","last_active":"2024-04-25T00:33:18.040501+0000","last_peered":"2024-04-25T00:33:18.040501+0000","last_clean":"2024-04-25T00:33:18.040501+0000","last_became_active":"2024-04-25T00:30:48.537193+0000","last_became_peered":"2024-04-25T00:30:48.537193+0000","last_unstale":"2024-04-25T00:33:18.040501+0000","last_undegraded":"2024-04-25T00:33:18.040501+0000","last_fullsized":"2024-04-25T00:33:18.040501+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:15:22.075323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.166777+0000","last_change":"2024-04-25T00:30:52.784802+0000","last_active":"2024-04-25T00:33:18.166777+0000","last_peered":"2024-04-25T00:33:18.166777+0000","last_clean":"2024-04-25T00:33:18.166777+0000","last_became_active":"2024-04-25T00:30:52.784359+0000","last_became_peered":"2024-04-25T00:30:52.784359+0000","last_unstale":"2024-04-25T00:33:18.166777+0000","last_undegraded":"2024-04-25T00:33:18.166777+0000","last_fullsized":"2024-04-25T00:33:18.166777+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:19:57.723542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1,8],"acting":[6,0,1,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"89'1","reported_seq":35,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055523+0000","last_change":"2024-04-25T00:30:47.525494+0000","last_active":"2024-04-25T00:33:18.055523+0000","last_peered":"2024-04-25T00:33:18.055523+0000","last_clean":"2024-04-25T00:33:18.055523+0000","last_became_active":"2024-04-25T00:30:47.525158+0000","last_became_peered":"2024-04-25T00:30:47.525158+0000","last_unstale":"2024-04-25T00:33:18.055523+0000","last_undegraded":"2024-04-25T00:33:18.055523+0000","last_fullsized":"2024-04-25T00:33:18.055523+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:51:18.543352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"91'7","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.040217+0000","last_change":"2024-04-25T00:30:48.540926+0000","last_active":"2024-04-25T00:33:18.040217+0000","last_peered":"2024-04-25T00:33:18.040217+0000","last_clean":"2024-04-25T00:33:18.040217+0000","last_became_active":"2024-04-25T00:30:48.540204+0000","last_became_peered":"2024-04-25T00:30:48.540204+0000","last_unstale":"2024-04-25T00:33:18.040217+0000","last_undegraded":"2024-04-25T00:33:18.040217+0000","last_fullsized":"2024-04-25T00:33:18.040217+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:02:45.530266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.167067+0000","last_change":"2024-04-25T00:30:52.805780+0000","last_active":"2024-04-25T00:33:18.167067+0000","last_peered":"2024-04-25T00:33:18.167067+0000","last_clean":"2024-04-25T00:33:18.167067+0000","last_became_active":"2024-04-25T00:30:52.805222+0000","last_became_peered":"2024-04-25T00:30:52.805222+0000","last_unstale":"2024-04-25T00:33:18.167067+0000","last_undegraded":"2024-04-25T00:33:18.167067+0000","last_fullsized":"2024-04-25T00:33:18.167067+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:32:17.034630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,10,0],"acting":[6,5,10,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.0","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.037047+0000","last_change":"2024-04-25T00:30:48.534335+0000","last_active":"2024-04-25T00:33:18.037047+0000","last_peered":"2024-04-25T00:33:18.037047+0000","last_clean":"2024-04-25T00:33:18.037047+0000","last_became_active":"2024-04-25T00:30:48.532670+0000","last_became_peered":"2024-04-25T00:30:48.532670+0000","last_unstale":"2024-04-25T00:33:18.037047+0000","last_undegraded":"2024-04-25T00:33:18.037047+0000","last_fullsized":"2024-04-25T00:33:18.037047+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:26:23.145918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"93'6","reported_seq":164,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.166569+0000","last_change":"2024-04-25T00:30:47.527695+0000","last_active":"2024-04-25T00:33:18.166569+0000","last_peered":"2024-04-25T00:33:18.166569+0000","last_clean":"2024-04-25T00:33:18.166569+0000","last_became_active":"2024-04-25T00:30:47.527239+0000","last_became_peered":"2024-04-25T00:30:47.527239+0000","last_unstale":"2024-04-25T00:33:18.166569+0000","last_undegraded":"2024-04-25T00:33:18.166569+0000","last_fullsized":"2024-04-25T00:33:18.166569+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:54:30.911284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":124,"num_read_kb":124,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"89'1","reported_seq":26,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055558+0000","last_change":"2024-04-25T00:30:52.781091+0000","last_active":"2024-04-25T00:33:18.055558+0000","last_peered":"2024-04-25T00:33:18.055558+0000","last_clean":"2024-04-25T00:33:18.055558+0000","last_became_active":"2024-04-25T00:30:52.780538+0000","last_became_peered":"2024-04-25T00:30:52.780538+0000","last_unstale":"2024-04-25T00:33:18.055558+0000","last_undegraded":"2024-04-25T00:33:18.055558+0000","last_fullsized":"2024-04-25T00:33:18.055558+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:49:18.141802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,11,9],"acting":[7,5,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.924882+0000","last_change":"2024-04-25T00:30:48.538730+0000","last_active":"2024-04-25T00:33:17.924882+0000","last_peered":"2024-04-25T00:33:17.924882+0000","last_clean":"2024-04-25T00:33:17.924882+0000","last_became_active":"2024-04-25T00:30:48.538336+0000","last_became_peered":"2024-04-25T00:30:48.538336+0000","last_unstale":"2024-04-25T00:33:17.924882+0000","last_undegraded":"2024-04-25T00:33:17.924882+0000","last_fullsized":"2024-04-25T00:33:17.924882+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T00:44:22.370448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"89'3","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926327+0000","last_change":"2024-04-25T00:30:47.520450+0000","last_active":"2024-04-25T00:33:17.926327+0000","last_peered":"2024-04-25T00:33:17.926327+0000","last_clean":"2024-04-25T00:33:17.926327+0000","last_became_active":"2024-04-25T00:30:47.520073+0000","last_became_peered":"2024-04-25T00:30:47.520073+0000","last_unstale":"2024-04-25T00:33:17.926327+0000","last_undegraded":"2024-04-25T00:33:17.926327+0000","last_fullsized":"2024-04-25T00:33:17.926327+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:49:53.570421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":4,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"22'137","reported_seq":367,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055359+0000","last_change":"2024-04-25T00:29:34.720552+0000","last_active":"2024-04-25T00:33:18.055359+0000","last_peered":"2024-04-25T00:33:18.055359+0000","last_clean":"2024-04-25T00:33:18.055359+0000","last_became_active":"2024-04-25T00:29:34.720041+0000","last_became_peered":"2024-04-25T00:29:34.720041+0000","last_unstale":"2024-04-25T00:33:18.055359+0000","last_undegraded":"2024-04-25T00:33:18.055359+0000","last_fullsized":"2024-04-25T00:33:18.055359+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:26:07.691582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:26:07.691582+0000","last_clean_scrub_stamp":"2024-04-25T00:26:07.691582+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:15:27.995162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.037382+0000","last_change":"2024-04-25T00:30:52.805489+0000","last_active":"2024-04-25T00:33:18.037382+0000","last_peered":"2024-04-25T00:33:18.037382+0000","last_clean":"2024-04-25T00:33:18.037382+0000","last_became_active":"2024-04-25T00:30:52.804758+0000","last_became_peered":"2024-04-25T00:30:52.804758+0000","last_unstale":"2024-04-25T00:33:18.037382+0000","last_undegraded":"2024-04-25T00:33:18.037382+0000","last_fullsized":"2024-04-25T00:33:18.037382+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:17:31.325548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,6,10],"acting":[5,9,6,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1","version":"91'5","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.166495+0000","last_change":"2024-04-25T00:30:48.537699+0000","last_active":"2024-04-25T00:33:18.166495+0000","last_peered":"2024-04-25T00:33:18.166495+0000","last_clean":"2024-04-25T00:33:18.166495+0000","last_became_active":"2024-04-25T00:30:48.537291+0000","last_became_peered":"2024-04-25T00:30:48.537291+0000","last_unstale":"2024-04-25T00:33:18.166495+0000","last_undegraded":"2024-04-25T00:33:18.166495+0000","last_fullsized":"2024-04-25T00:33:18.166495+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:25:36.816499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"89'3","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055470+0000","last_change":"2024-04-25T00:30:47.524694+0000","last_active":"2024-04-25T00:33:18.055470+0000","last_peered":"2024-04-25T00:33:18.055470+0000","last_clean":"2024-04-25T00:33:18.055470+0000","last_became_active":"2024-04-25T00:30:47.524359+0000","last_became_peered":"2024-04-25T00:30:47.524359+0000","last_unstale":"2024-04-25T00:33:18.055470+0000","last_undegraded":"2024-04-25T00:33:18.055470+0000","last_fullsized":"2024-04-25T00:33:18.055470+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:55:16.677234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.304812+0000","last_change":"2024-04-25T00:30:52.780364+0000","last_active":"2024-04-25T00:33:18.304812+0000","last_peered":"2024-04-25T00:33:18.304812+0000","last_clean":"2024-04-25T00:33:18.304812+0000","last_became_active":"2024-04-25T00:30:52.779904+0000","last_became_peered":"2024-04-25T00:30:52.779904+0000","last_unstale":"2024-04-25T00:33:18.304812+0000","last_undegraded":"2024-04-25T00:33:18.304812+0000","last_fullsized":"2024-04-25T00:33:18.304812+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:11:26.738526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,7,6],"acting":[11,5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.5","version":"89'1","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.030818+0000","last_change":"2024-04-25T00:30:47.525205+0000","last_active":"2024-04-25T00:33:18.030818+0000","last_peered":"2024-04-25T00:33:18.030818+0000","last_clean":"2024-04-25T00:33:18.030818+0000","last_became_active":"2024-04-25T00:30:47.517729+0000","last_became_peered":"2024-04-25T00:30:47.517729+0000","last_unstale":"2024-04-25T00:33:18.030818+0000","last_undegraded":"2024-04-25T00:33:18.030818+0000","last_fullsized":"2024-04-25T00:33:18.030818+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:21:14.810119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"91'9","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.166967+0000","last_change":"2024-04-25T00:30:48.533822+0000","last_active":"2024-04-25T00:33:18.166967+0000","last_peered":"2024-04-25T00:33:18.166967+0000","last_clean":"2024-04-25T00:33:18.166967+0000","last_became_active":"2024-04-25T00:30:48.533479+0000","last_became_peered":"2024-04-25T00:30:48.533479+0000","last_unstale":"2024-04-25T00:33:18.166967+0000","last_undegraded":"2024-04-25T00:33:18.166967+0000","last_fullsized":"2024-04-25T00:33:18.166967+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:49:01.028124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.924543+0000","last_change":"2024-04-25T00:30:52.804770+0000","last_active":"2024-04-25T00:33:17.924543+0000","last_peered":"2024-04-25T00:33:17.924543+0000","last_clean":"2024-04-25T00:33:17.924543+0000","last_became_active":"2024-04-25T00:30:52.804242+0000","last_became_peered":"2024-04-25T00:30:52.804242+0000","last_unstale":"2024-04-25T00:33:17.924543+0000","last_undegraded":"2024-04-25T00:33:17.924543+0000","last_fullsized":"2024-04-25T00:33:17.924543+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:34:11.761478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,10,0,7],"acting":[3,10,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"94'44","reported_seq":101,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.144934+0000","last_change":"2024-04-25T00:30:47.521364+0000","last_active":"2024-04-25T00:33:18.144934+0000","last_peered":"2024-04-25T00:33:18.144934+0000","last_clean":"2024-04-25T00:33:18.144934+0000","last_became_active":"2024-04-25T00:30:47.521032+0000","last_became_peered":"2024-04-25T00:30:47.521032+0000","last_unstale":"2024-04-25T00:33:18.144934+0000","last_undegraded":"2024-04-25T00:33:18.144934+0000","last_fullsized":"2024-04-25T00:33:18.144934+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":44,"log_dups_size":0,"ondisk_log_size":44,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:33:26.156568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393216,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":82,"num_read_kb":70,"num_write":78,"num_write_kb":836,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,8,3],"acting":[9,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"91'5","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.925363+0000","last_change":"2024-04-25T00:30:48.532942+0000","last_active":"2024-04-25T00:33:17.925363+0000","last_peered":"2024-04-25T00:33:17.925363+0000","last_clean":"2024-04-25T00:33:17.925363+0000","last_became_active":"2024-04-25T00:30:48.532580+0000","last_became_peered":"2024-04-25T00:30:48.532580+0000","last_unstale":"2024-04-25T00:33:17.925363+0000","last_undegraded":"2024-04-25T00:33:17.925363+0000","last_fullsized":"2024-04-25T00:33:17.925363+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:42:22.887920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926476+0000","last_change":"2024-04-25T00:30:52.779757+0000","last_active":"2024-04-25T00:33:17.926476+0000","last_peered":"2024-04-25T00:33:17.926476+0000","last_clean":"2024-04-25T00:33:17.926476+0000","last_became_active":"2024-04-25T00:30:52.779324+0000","last_became_peered":"2024-04-25T00:30:52.779324+0000","last_unstale":"2024-04-25T00:33:17.926476+0000","last_undegraded":"2024-04-25T00:33:17.926476+0000","last_fullsized":"2024-04-25T00:33:17.926476+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:22:39.062000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4,7],"acting":[2,1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"89'4","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926614+0000","last_change":"2024-04-25T00:30:47.518212+0000","last_active":"2024-04-25T00:33:17.926614+0000","last_peered":"2024-04-25T00:33:17.926614+0000","last_clean":"2024-04-25T00:33:17.926614+0000","last_became_active":"2024-04-25T00:30:47.517610+0000","last_became_peered":"2024-04-25T00:30:47.517610+0000","last_unstale":"2024-04-25T00:33:17.926614+0000","last_undegraded":"2024-04-25T00:33:17.926614+0000","last_fullsized":"2024-04-25T00:33:17.926614+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:13:58.904020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"91'7","reported_seq":45,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305983+0000","last_change":"2024-04-25T00:31:45.028411+0000","last_active":"2024-04-25T00:33:18.305983+0000","last_peered":"2024-04-25T00:33:18.305983+0000","last_clean":"2024-04-25T00:33:18.305983+0000","last_became_active":"2024-04-25T00:31:45.028066+0000","last_became_peered":"2024-04-25T00:31:45.028066+0000","last_unstale":"2024-04-25T00:33:18.305983+0000","last_undegraded":"2024-04-25T00:33:18.305983+0000","last_fullsized":"2024-04-25T00:33:18.305983+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:33:46.556862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926345+0000","last_change":"2024-04-25T00:30:52.804915+0000","last_active":"2024-04-25T00:33:17.926345+0000","last_peered":"2024-04-25T00:33:17.926345+0000","last_clean":"2024-04-25T00:33:17.926345+0000","last_became_active":"2024-04-25T00:30:52.804491+0000","last_became_peered":"2024-04-25T00:30:52.804491+0000","last_unstale":"2024-04-25T00:33:17.926345+0000","last_undegraded":"2024-04-25T00:33:17.926345+0000","last_fullsized":"2024-04-25T00:33:17.926345+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:51:50.219556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,10,9],"acting":[4,5,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.8","version":"89'2","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.187520+0000","last_change":"2024-04-25T00:30:47.521909+0000","last_active":"2024-04-25T00:33:18.187520+0000","last_peered":"2024-04-25T00:33:18.187520+0000","last_clean":"2024-04-25T00:33:18.187520+0000","last_became_active":"2024-04-25T00:30:47.521551+0000","last_became_peered":"2024-04-25T00:30:47.521551+0000","last_unstale":"2024-04-25T00:33:18.187520+0000","last_undegraded":"2024-04-25T00:33:18.187520+0000","last_fullsized":"2024-04-25T00:33:18.187520+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:51:16.368609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"91'2","reported_seq":34,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.030469+0000","last_change":"2024-04-25T00:30:48.530514+0000","last_active":"2024-04-25T00:33:18.030469+0000","last_peered":"2024-04-25T00:33:18.030469+0000","last_clean":"2024-04-25T00:33:18.030469+0000","last_became_active":"2024-04-25T00:30:48.530129+0000","last_became_peered":"2024-04-25T00:30:48.530129+0000","last_unstale":"2024-04-25T00:33:18.030469+0000","last_undegraded":"2024-04-25T00:33:18.030469+0000","last_fullsized":"2024-04-25T00:33:18.030469+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:51:25.623186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.028315+0000","last_change":"2024-04-25T00:30:52.785767+0000","last_active":"2024-04-25T00:33:18.028315+0000","last_peered":"2024-04-25T00:33:18.028315+0000","last_clean":"2024-04-25T00:33:18.028315+0000","last_became_active":"2024-04-25T00:30:52.785301+0000","last_became_peered":"2024-04-25T00:30:52.785301+0000","last_unstale":"2024-04-25T00:33:18.028315+0000","last_undegraded":"2024-04-25T00:33:18.028315+0000","last_fullsized":"2024-04-25T00:33:18.028315+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T06:10:14.071733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,2,5],"acting":[9,3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.9","version":"89'3","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.924598+0000","last_change":"2024-04-25T00:30:47.526364+0000","last_active":"2024-04-25T00:33:17.924598+0000","last_peered":"2024-04-25T00:33:17.924598+0000","last_clean":"2024-04-25T00:33:17.924598+0000","last_became_active":"2024-04-25T00:30:47.526009+0000","last_became_peered":"2024-04-25T00:30:47.526009+0000","last_unstale":"2024-04-25T00:33:17.924598+0000","last_undegraded":"2024-04-25T00:33:17.924598+0000","last_fullsized":"2024-04-25T00:33:17.924598+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:17:38.471878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"91'9","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.304758+0000","last_change":"2024-04-25T00:30:48.532283+0000","last_active":"2024-04-25T00:33:18.304758+0000","last_peered":"2024-04-25T00:33:18.304758+0000","last_clean":"2024-04-25T00:33:18.304758+0000","last_became_active":"2024-04-25T00:30:48.531935+0000","last_became_peered":"2024-04-25T00:30:48.531935+0000","last_unstale":"2024-04-25T00:33:18.304758+0000","last_undegraded":"2024-04-25T00:33:18.304758+0000","last_fullsized":"2024-04-25T00:33:18.304758+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:01:15.965611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.c","version":"89'1","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926179+0000","last_change":"2024-04-25T00:31:44.717218+0000","last_active":"2024-04-25T00:33:17.926179+0000","last_peered":"2024-04-25T00:33:17.926179+0000","last_clean":"2024-04-25T00:33:17.926179+0000","last_became_active":"2024-04-25T00:31:44.712770+0000","last_became_peered":"2024-04-25T00:31:44.712770+0000","last_unstale":"2024-04-25T00:33:17.926179+0000","last_undegraded":"2024-04-25T00:33:17.926179+0000","last_fullsized":"2024-04-25T00:33:17.926179+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:06:58.496133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":117,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,7,8],"acting":[4,10,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.a","version":"89'1","reported_seq":35,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926611+0000","last_change":"2024-04-25T00:30:47.520608+0000","last_active":"2024-04-25T00:33:17.926611+0000","last_peered":"2024-04-25T00:33:17.926611+0000","last_clean":"2024-04-25T00:33:17.926611+0000","last_became_active":"2024-04-25T00:30:47.520063+0000","last_became_peered":"2024-04-25T00:30:47.520063+0000","last_unstale":"2024-04-25T00:33:17.926611+0000","last_undegraded":"2024-04-25T00:33:17.926611+0000","last_fullsized":"2024-04-25T00:33:17.926611+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:58:47.823712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.925247+0000","last_change":"2024-04-25T00:30:48.535073+0000","last_active":"2024-04-25T00:33:17.925247+0000","last_peered":"2024-04-25T00:33:17.925247+0000","last_clean":"2024-04-25T00:33:17.925247+0000","last_became_active":"2024-04-25T00:30:48.534247+0000","last_became_peered":"2024-04-25T00:30:48.534247+0000","last_unstale":"2024-04-25T00:33:17.925247+0000","last_undegraded":"2024-04-25T00:33:17.925247+0000","last_fullsized":"2024-04-25T00:33:17.925247+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:46:18.613705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926144+0000","last_change":"2024-04-25T00:30:52.804519+0000","last_active":"2024-04-25T00:33:17.926144+0000","last_peered":"2024-04-25T00:33:17.926144+0000","last_clean":"2024-04-25T00:33:17.926144+0000","last_became_active":"2024-04-25T00:30:52.804074+0000","last_became_peered":"2024-04-25T00:30:52.804074+0000","last_unstale":"2024-04-25T00:33:17.926144+0000","last_undegraded":"2024-04-25T00:33:17.926144+0000","last_fullsized":"2024-04-25T00:33:17.926144+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:48:45.058873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,10,3],"acting":[4,2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.b","version":"89'4","reported_seq":19,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.304778+0000","last_change":"2024-04-25T00:32:45.477574+0000","last_active":"2024-04-25T00:33:18.304778+0000","last_peered":"2024-04-25T00:33:18.304778+0000","last_clean":"2024-04-25T00:33:18.304778+0000","last_became_active":"2024-04-25T00:32:45.472622+0000","last_became_peered":"2024-04-25T00:32:45.472622+0000","last_unstale":"2024-04-25T00:33:18.304778+0000","last_undegraded":"2024-04-25T00:33:18.304778+0000","last_fullsized":"2024-04-25T00:33:18.304778+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:18:56.072904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,7],"acting":[11,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.a","version":"91'5","reported_seq":37,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055542+0000","last_change":"2024-04-25T00:30:48.537152+0000","last_active":"2024-04-25T00:33:18.055542+0000","last_peered":"2024-04-25T00:33:18.055542+0000","last_clean":"2024-04-25T00:33:18.055542+0000","last_became_active":"2024-04-25T00:30:48.536256+0000","last_became_peered":"2024-04-25T00:30:48.536256+0000","last_unstale":"2024-04-25T00:33:18.055542+0000","last_undegraded":"2024-04-25T00:33:18.055542+0000","last_fullsized":"2024-04-25T00:33:18.055542+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:57:53.870212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.187504+0000","last_change":"2024-04-25T00:30:52.781022+0000","last_active":"2024-04-25T00:33:18.187504+0000","last_peered":"2024-04-25T00:33:18.187504+0000","last_clean":"2024-04-25T00:33:18.187504+0000","last_became_active":"2024-04-25T00:30:52.780595+0000","last_became_peered":"2024-04-25T00:30:52.780595+0000","last_unstale":"2024-04-25T00:33:18.187504+0000","last_undegraded":"2024-04-25T00:33:18.187504+0000","last_fullsized":"2024-04-25T00:33:18.187504+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:27:02.580870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6,7],"acting":[1,5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.c","version":"89'3","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926971+0000","last_change":"2024-04-25T00:30:47.519118+0000","last_active":"2024-04-25T00:33:17.926971+0000","last_peered":"2024-04-25T00:33:17.926971+0000","last_clean":"2024-04-25T00:33:17.926971+0000","last_became_active":"2024-04-25T00:30:47.518639+0000","last_became_peered":"2024-04-25T00:30:47.518639+0000","last_unstale":"2024-04-25T00:33:17.926971+0000","last_undegraded":"2024-04-25T00:33:17.926971+0000","last_fullsized":"2024-04-25T00:33:17.926971+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:32:29.747903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"91'7","reported_seq":39,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055812+0000","last_change":"2024-04-25T00:30:48.542073+0000","last_active":"2024-04-25T00:33:18.055812+0000","last_peered":"2024-04-25T00:33:18.055812+0000","last_clean":"2024-04-25T00:33:18.055812+0000","last_became_active":"2024-04-25T00:30:48.541722+0000","last_became_peered":"2024-04-25T00:30:48.541722+0000","last_unstale":"2024-04-25T00:33:18.055812+0000","last_undegraded":"2024-04-25T00:33:18.055812+0000","last_fullsized":"2024-04-25T00:33:18.055812+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:34:03.397539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305021+0000","last_change":"2024-04-25T00:30:52.790234+0000","last_active":"2024-04-25T00:33:18.305021+0000","last_peered":"2024-04-25T00:33:18.305021+0000","last_clean":"2024-04-25T00:33:18.305021+0000","last_became_active":"2024-04-25T00:30:52.789788+0000","last_became_peered":"2024-04-25T00:30:52.789788+0000","last_unstale":"2024-04-25T00:33:18.305021+0000","last_undegraded":"2024-04-25T00:33:18.305021+0000","last_fullsized":"2024-04-25T00:33:18.305021+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:44:43.399771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,9,1,4],"acting":[0,9,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.d","version":"94'39","reported_seq":81,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.142633+0000","last_change":"2024-04-25T00:30:47.523597+0000","last_active":"2024-04-25T00:33:18.142633+0000","last_peered":"2024-04-25T00:33:18.142633+0000","last_clean":"2024-04-25T00:33:18.142633+0000","last_became_active":"2024-04-25T00:30:47.523225+0000","last_became_peered":"2024-04-25T00:30:47.523225+0000","last_unstale":"2024-04-25T00:33:18.142633+0000","last_undegraded":"2024-04-25T00:33:18.142633+0000","last_fullsized":"2024-04-25T00:33:18.142633+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":39,"log_dups_size":0,"ondisk_log_size":39,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:51:12.341015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131670,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":11,"num_read_kb":10,"num_write":49,"num_write_kb":475,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"91'6","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055778+0000","last_change":"2024-04-25T00:30:48.541417+0000","last_active":"2024-04-25T00:33:18.055778+0000","last_peered":"2024-04-25T00:33:18.055778+0000","last_clean":"2024-04-25T00:33:18.055778+0000","last_became_active":"2024-04-25T00:30:48.541015+0000","last_became_peered":"2024-04-25T00:30:48.541015+0000","last_unstale":"2024-04-25T00:33:18.055778+0000","last_undegraded":"2024-04-25T00:33:18.055778+0000","last_fullsized":"2024-04-25T00:33:18.055778+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:21:35.299823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":25,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.036816+0000","last_change":"2024-04-25T00:30:52.782230+0000","last_active":"2024-04-25T00:33:18.036816+0000","last_peered":"2024-04-25T00:33:18.036816+0000","last_clean":"2024-04-25T00:33:18.036816+0000","last_became_active":"2024-04-25T00:30:52.781708+0000","last_became_peered":"2024-04-25T00:30:52.781708+0000","last_unstale":"2024-04-25T00:33:18.036816+0000","last_undegraded":"2024-04-25T00:33:18.036816+0000","last_fullsized":"2024-04-25T00:33:18.036816+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:51.749239+0000","last_clean_scrub_stamp":"2024-04-25T00:30:51.749239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:58:53.260985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7,1],"acting":[5,2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.e","version":"84'2","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.167506+0000","last_change":"2024-04-25T00:32:45.553977+0000","last_active":"2024-04-25T00:33:18.167506+0000","last_peered":"2024-04-25T00:33:18.167506+0000","last_clean":"2024-04-25T00:33:18.167506+0000","last_became_active":"2024-04-25T00:32:45.551280+0000","last_became_peered":"2024-04-25T00:32:45.551280+0000","last_unstale":"2024-04-25T00:33:18.167506+0000","last_undegraded":"2024-04-25T00:33:18.167506+0000","last_fullsized":"2024-04-25T00:33:18.167506+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:46.491510+0000","last_clean_scrub_stamp":"2024-04-25T00:30:46.491510+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:54:12.307459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"91'9","reported_seq":41,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.055986+0000","last_change":"2024-04-25T00:30:48.532730+0000","last_active":"2024-04-25T00:33:18.055986+0000","last_peered":"2024-04-25T00:33:18.055986+0000","last_clean":"2024-04-25T00:33:18.055986+0000","last_became_active":"2024-04-25T00:30:48.532129+0000","last_became_peered":"2024-04-25T00:30:48.532129+0000","last_unstale":"2024-04-25T00:33:18.055986+0000","last_undegraded":"2024-04-25T00:33:18.055986+0000","last_fullsized":"2024-04-25T00:33:18.055986+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:36:30.115125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"91'9","reported_seq":47,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.304840+0000","last_change":"2024-04-25T00:31:44.712173+0000","last_active":"2024-04-25T00:33:18.304840+0000","last_peered":"2024-04-25T00:33:18.304840+0000","last_clean":"2024-04-25T00:33:18.304840+0000","last_became_active":"2024-04-25T00:31:44.711738+0000","last_became_peered":"2024-04-25T00:31:44.711738+0000","last_unstale":"2024-04-25T00:33:18.304840+0000","last_undegraded":"2024-04-25T00:33:18.304840+0000","last_fullsized":"2024-04-25T00:33:18.304840+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:06:59.466566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,9],"acting":[11,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"91'12","reported_seq":44,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926692+0000","last_change":"2024-04-25T00:30:48.540676+0000","last_active":"2024-04-25T00:33:17.926692+0000","last_peered":"2024-04-25T00:33:17.926692+0000","last_clean":"2024-04-25T00:33:17.926692+0000","last_became_active":"2024-04-25T00:30:48.540258+0000","last_became_peered":"2024-04-25T00:30:48.540258+0000","last_unstale":"2024-04-25T00:33:17.926692+0000","last_undegraded":"2024-04-25T00:33:17.926692+0000","last_fullsized":"2024-04-25T00:33:17.926692+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T12:10:29.930897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.926531+0000","last_change":"2024-04-25T00:30:48.542191+0000","last_active":"2024-04-25T00:33:17.926531+0000","last_peered":"2024-04-25T00:33:17.926531+0000","last_clean":"2024-04-25T00:33:17.926531+0000","last_became_active":"2024-04-25T00:30:48.540433+0000","last_became_peered":"2024-04-25T00:30:48.540433+0000","last_unstale":"2024-04-25T00:33:17.926531+0000","last_undegraded":"2024-04-25T00:33:17.926531+0000","last_fullsized":"2024-04-25T00:33:17.926531+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T08:21:55.694555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"91'7","reported_seq":45,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305410+0000","last_change":"2024-04-25T00:31:45.028445+0000","last_active":"2024-04-25T00:33:18.305410+0000","last_peered":"2024-04-25T00:33:18.305410+0000","last_clean":"2024-04-25T00:33:18.305410+0000","last_became_active":"2024-04-25T00:31:45.028048+0000","last_became_peered":"2024-04-25T00:31:45.028048+0000","last_unstale":"2024-04-25T00:33:18.305410+0000","last_undegraded":"2024-04-25T00:33:18.305410+0000","last_fullsized":"2024-04-25T00:33:18.305410+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T10:04:02.714111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,8],"acting":[0,6,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"91'4","reported_seq":36,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.167135+0000","last_change":"2024-04-25T00:30:48.527892+0000","last_active":"2024-04-25T00:33:18.167135+0000","last_peered":"2024-04-25T00:33:18.167135+0000","last_clean":"2024-04-25T00:33:18.167135+0000","last_became_active":"2024-04-25T00:30:48.527522+0000","last_became_peered":"2024-04-25T00:30:48.527522+0000","last_unstale":"2024-04-25T00:33:18.167135+0000","last_undegraded":"2024-04-25T00:33:18.167135+0000","last_fullsized":"2024-04-25T00:33:18.167135+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:21:22.779470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"91'6","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.305133+0000","last_change":"2024-04-25T00:30:48.535066+0000","last_active":"2024-04-25T00:33:18.305133+0000","last_peered":"2024-04-25T00:33:18.305133+0000","last_clean":"2024-04-25T00:33:18.305133+0000","last_became_active":"2024-04-25T00:30:48.534710+0000","last_became_peered":"2024-04-25T00:30:48.534710+0000","last_unstale":"2024-04-25T00:33:18.305133+0000","last_undegraded":"2024-04-25T00:33:18.305133+0000","last_fullsized":"2024-04-25T00:33:18.305133+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T02:57:18.582569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"91'15","reported_seq":47,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.927199+0000","last_change":"2024-04-25T00:30:48.541921+0000","last_active":"2024-04-25T00:33:17.927199+0000","last_peered":"2024-04-25T00:33:17.927199+0000","last_clean":"2024-04-25T00:33:17.927199+0000","last_became_active":"2024-04-25T00:30:48.541348+0000","last_became_peered":"2024-04-25T00:30:48.541348+0000","last_unstale":"2024-04-25T00:33:17.927199+0000","last_undegraded":"2024-04-25T00:33:17.927199+0000","last_fullsized":"2024-04-25T00:33:17.927199+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:07:00.001504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"91'2","reported_seq":34,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.925569+0000","last_change":"2024-04-25T00:30:48.536902+0000","last_active":"2024-04-25T00:33:17.925569+0000","last_peered":"2024-04-25T00:33:17.925569+0000","last_clean":"2024-04-25T00:33:17.925569+0000","last_became_active":"2024-04-25T00:30:48.536008+0000","last_became_peered":"2024-04-25T00:30:48.536008+0000","last_unstale":"2024-04-25T00:33:17.925569+0000","last_undegraded":"2024-04-25T00:33:17.925569+0000","last_fullsized":"2024-04-25T00:33:17.925569+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T03:19:15.050142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"91'6","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.036972+0000","last_change":"2024-04-25T00:30:48.533104+0000","last_active":"2024-04-25T00:33:18.036972+0000","last_peered":"2024-04-25T00:33:18.036972+0000","last_clean":"2024-04-25T00:33:18.036972+0000","last_became_active":"2024-04-25T00:30:48.532124+0000","last_became_peered":"2024-04-25T00:30:48.532124+0000","last_unstale":"2024-04-25T00:33:18.036972+0000","last_undegraded":"2024-04-25T00:33:18.036972+0000","last_fullsized":"2024-04-25T00:33:18.036972+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T01:54:57.391433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"91'8","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:17.924561+0000","last_change":"2024-04-25T00:30:48.542074+0000","last_active":"2024-04-25T00:33:17.924561+0000","last_peered":"2024-04-25T00:33:17.924561+0000","last_clean":"2024-04-25T00:33:17.924561+0000","last_became_active":"2024-04-25T00:30:48.538752+0000","last_became_peered":"2024-04-25T00:30:48.538752+0000","last_unstale":"2024-04-25T00:33:17.924561+0000","last_undegraded":"2024-04-25T00:33:17.924561+0000","last_fullsized":"2024-04-25T00:33:17.924561+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T05:55:30.613616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"91'10","reported_seq":42,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.187829+0000","last_change":"2024-04-25T00:30:48.534158+0000","last_active":"2024-04-25T00:33:18.187829+0000","last_peered":"2024-04-25T00:33:18.187829+0000","last_clean":"2024-04-25T00:33:18.187829+0000","last_became_active":"2024-04-25T00:30:48.533821+0000","last_became_peered":"2024-04-25T00:30:48.533821+0000","last_unstale":"2024-04-25T00:33:18.187829+0000","last_undegraded":"2024-04-25T00:33:18.187829+0000","last_fullsized":"2024-04-25T00:33:18.187829+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T09:05:19.327608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"91'11","reported_seq":43,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.167481+0000","last_change":"2024-04-25T00:30:48.531718+0000","last_active":"2024-04-25T00:33:18.167481+0000","last_peered":"2024-04-25T00:33:18.167481+0000","last_clean":"2024-04-25T00:33:18.167481+0000","last_became_active":"2024-04-25T00:30:48.531291+0000","last_became_peered":"2024-04-25T00:30:48.531291+0000","last_unstale":"2024-04-25T00:33:18.167481+0000","last_undegraded":"2024-04-25T00:33:18.167481+0000","last_fullsized":"2024-04-25T00:33:18.167481+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T04:56:49.032937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"91'8","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2024-04-25T00:33:18.304987+0000","last_change":"2024-04-25T00:31:44.713036+0000","last_active":"2024-04-25T00:33:18.304987+0000","last_peered":"2024-04-25T00:33:18.304987+0000","last_clean":"2024-04-25T00:33:18.304987+0000","last_became_active":"2024-04-25T00:31:44.712651+0000","last_became_peered":"2024-04-25T00:31:44.712651+0000","last_unstale":"2024-04-25T00:33:18.304987+0000","last_undegraded":"2024-04-25T00:33:18.304987+0000","last_fullsized":"2024-04-25T00:33:18.304987+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":83,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T00:30:47.497921+0000","last_clean_scrub_stamp":"2024-04-25T00:30:47.497921+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T11:05:04.811510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,9,8],"acting":[11,9,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":234,"num_objects":2,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":117,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":32768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2,"ondisk_log_size":2,"up":128,"acting":128,"num_store_stats":12},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":500,"ondisk_log_size":500,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":597867,"num_objects":62,"num_object_clones":0,"num_object_copies":186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":62,"num_whiteouts":0,"num_read":864,"num_read_kb":540,"num_write":289,"num_write_kb":1441,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":134,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2027520,"data_stored":1793601,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":229,"ondisk_log_size":229,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":80,"seq":343597383723,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32232,"kb_used_data":1052,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712920,"statfs":{"total":95995035648,"available":95962030080,"internally_reserved":0,"allocated":1077248,"data_stored":529717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1596,"internal_metadata":31914436},"hb_peers":[0,1,2,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":10,"up_from":72,"seq":309237645360,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28880,"kb_used_data":1864,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716272,"statfs":{"total":95995035648,"available":95965462528,"internally_reserved":0,"allocated":1908736,"data_stored":1321709,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1600,"internal_metadata":27654592},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":9,"up_from":67,"seq":287762808885,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36788,"kb_used_data":1448,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708364,"statfs":{"total":95995035648,"available":95957364736,"internally_reserved":0,"allocated":1482752,"data_stored":923147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1600,"internal_metadata":36174272},"hb_peers":[0,1,2,3,4,5,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":8,"up_from":59,"seq":253403070522,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32620,"kb_used_data":1444,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712532,"statfs":{"total":95995035648,"available":95961632768,"internally_reserved":0,"allocated":1478656,"data_stored":923045,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":7,"up_from":51,"seq":219043332159,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36984,"kb_used_data":1644,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708168,"statfs":{"total":95995035648,"available":95957164032,"internally_reserved":0,"allocated":1683456,"data_stored":1127382,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":36174282},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495685,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36384,"kb_used_data":1044,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708768,"statfs":{"total":95995035648,"available":95957778432,"internally_reserved":0,"allocated":1069056,"data_stored":525107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":36174278},"hb_peers":[0,1,2,3,4,5,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574940,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28116,"kb_used_data":1100,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717036,"statfs":{"total":95995035648,"available":95966244864,"internally_reserved":0,"allocated":1126400,"data_stored":581739,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738467,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28076,"kb_used_data":1052,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717076,"statfs":{"total":95995035648,"available":95966285824,"internally_reserved":0,"allocated":1077248,"data_stored":516571,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":18,"seq":77309411416,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28152,"kb_used_data":1132,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717000,"statfs":{"total":95995035648,"available":95966208000,"internally_reserved":0,"allocated":1159168,"data_stored":594074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":25,"seq":107374182483,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36592,"kb_used_data":1252,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708560,"statfs":{"total":95995035648,"available":95957565440,"internally_reserved":0,"allocated":1282048,"data_stored":722421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":36174279},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":33,"seq":141733920847,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28084,"kb_used_data":1064,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717068,"statfs":{"total":95995035648,"available":95966277632,"internally_reserved":0,"allocated":1089536,"data_stored":530709,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757322,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28800,"kb_used_data":1784,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716352,"statfs":{"total":95995035648,"available":95965544448,"internally_reserved":0,"allocated":1826816,"data_stored":1252948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":77824,"data_stored":67428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":94208,"data_stored":69057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":544768,"data_stored":525100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":2378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":163840,"data_stored":133467,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":3805,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":409600,"data_stored":393932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":394034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":245760,"data_stored":202228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":8192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":4096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-25T00:33:21.312 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-25T00:33:21.460 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.459+0000 7fce78aa3640 1 -- 172.21.15.32:0/1108209192 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fce74103350 msgr2=0x7fce74109be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:21.460 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.459+0000 7fce78aa3640 1 --2- 172.21.15.32:0/1108209192 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fce74103350 0x7fce74109be0 secure :-1 s=READY pgs=489 cs=0 l=1 rev1=1 crypto rx=0x7fce6c009ca0 tx=0x7fce6c02a8c0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.460 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.459+0000 7fce78aa3640 1 -- 172.21.15.32:0/1108209192 shutdown_connections 2024-04-25T00:33:21.460 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.459+0000 7fce78aa3640 1 --2- 172.21.15.32:0/1108209192 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fce74103350 0x7fce74109be0 unknown :-1 s=CLOSED pgs=489 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.460 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.459+0000 7fce78aa3640 1 --2- 172.21.15.32:0/1108209192 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fce74102990 0x7fce74102e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.460 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.459+0000 7fce78aa3640 1 --2- 172.21.15.32:0/1108209192 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fce741017a0 0x7fce74101bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.460 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.459+0000 7fce78aa3640 1 -- 172.21.15.32:0/1108209192 >> 172.21.15.32:0/1108209192 conn(0x7fce740fccf0 msgr2=0x7fce740ff150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:21.460 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.459+0000 7fce78aa3640 1 -- 172.21.15.32:0/1108209192 shutdown_connections 2024-04-25T00:33:21.460 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.459+0000 7fce78aa3640 1 -- 172.21.15.32:0/1108209192 wait complete. 2024-04-25T00:33:21.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.460+0000 7fce78aa3640 1 Processor -- start 2024-04-25T00:33:21.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.460+0000 7fce78aa3640 1 -- start start 2024-04-25T00:33:21.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.460+0000 7fce78aa3640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fce741017a0 0x7fce740754f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:21.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.460+0000 7fce78aa3640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fce74102990 0x7fce74073b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:21.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.460+0000 7fce78aa3640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fce74103350 0x7fce74074070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:21.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.460+0000 7fce78aa3640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fce7410d420 con 0x7fce741017a0 2024-04-25T00:33:21.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.460+0000 7fce78aa3640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fce7410d2a0 con 0x7fce74103350 2024-04-25T00:33:21.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.460+0000 7fce78aa3640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fce7410d5a0 con 0x7fce74102990 2024-04-25T00:33:21.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.460+0000 7fce7259c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fce741017a0 0x7fce740754f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:21.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.460+0000 7fce72d9d640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fce74103350 0x7fce74074070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:21.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.460+0000 7fce7259c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fce741017a0 0x7fce740754f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44434/0 (socket says 172.21.15.32:44434) 2024-04-25T00:33:21.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.460+0000 7fce7259c640 1 -- 172.21.15.32:0/3871120236 learned_addr learned my addr 172.21.15.32:0/3871120236 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:21.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.460+0000 7fce71d9b640 1 --2- 172.21.15.32:0/3871120236 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fce74102990 0x7fce74073b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:21.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.460+0000 7fce72d9d640 1 -- 172.21.15.32:0/3871120236 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fce74102990 msgr2=0x7fce74073b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:21.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.460+0000 7fce72d9d640 1 --2- 172.21.15.32:0/3871120236 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fce74102990 0x7fce74073b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.460+0000 7fce72d9d640 1 -- 172.21.15.32:0/3871120236 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fce741017a0 msgr2=0x7fce740754f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:21.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.460+0000 7fce72d9d640 1 --2- 172.21.15.32:0/3871120236 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fce741017a0 0x7fce740754f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.460+0000 7fce72d9d640 1 -- 172.21.15.32:0/3871120236 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fce74074900 con 0x7fce74103350 2024-04-25T00:33:21.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.460+0000 7fce7259c640 1 --2- 172.21.15.32:0/3871120236 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fce741017a0 0x7fce740754f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:21.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.461+0000 7fce72d9d640 1 --2- 172.21.15.32:0/3871120236 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fce74103350 0x7fce74074070 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7fce6c031ae0 tx=0x7fce6c031ee0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:21.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.461+0000 7fce637fe640 1 -- 172.21.15.32:0/3871120236 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fce6c003c30 con 0x7fce74103350 2024-04-25T00:33:21.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.461+0000 7fce637fe640 1 -- 172.21.15.32:0/3871120236 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fce6c003dd0 con 0x7fce74103350 2024-04-25T00:33:21.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.461+0000 7fce78aa3640 1 -- 172.21.15.32:0/3871120236 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fce741a1b50 con 0x7fce74103350 2024-04-25T00:33:21.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.461+0000 7fce637fe640 1 -- 172.21.15.32:0/3871120236 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fce6c02fcc0 con 0x7fce74103350 2024-04-25T00:33:21.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.461+0000 7fce78aa3640 1 -- 172.21.15.32:0/3871120236 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fce741a20c0 con 0x7fce74103350 2024-04-25T00:33:21.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.462+0000 7fce78aa3640 1 -- 172.21.15.32:0/3871120236 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fce74108ac0 con 0x7fce74103350 2024-04-25T00:33:21.465 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.464+0000 7fce637fe640 1 -- 172.21.15.32:0/3871120236 <== mon.2 v2:172.21.15.121:3300/0 4 ==== mgrmap(e 23) ==== 137953+0+0 (secure 0 0 0) 0x7fce6c004050 con 0x7fce74103350 2024-04-25T00:33:21.465 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.465+0000 7fce637fe640 1 --2- 172.21.15.32:0/3871120236 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fce5c0a2d80 0x7fce5c0a5240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:21.465 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.465+0000 7fce7259c640 1 --2- 172.21.15.32:0/3871120236 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fce5c0a2d80 0x7fce5c0a5240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:21.465 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.465+0000 7fce637fe640 1 -- 172.21.15.32:0/3871120236 <== mon.2 v2:172.21.15.121:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fce6c0f0940 con 0x7fce74103350 2024-04-25T00:33:21.466 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.465+0000 7fce7259c640 1 --2- 172.21.15.32:0/3871120236 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fce5c0a2d80 0x7fce5c0a5240 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7fce64005fd0 tx=0x7fce6400b410 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:21.468 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.468+0000 7fce637fe640 1 -- 172.21.15.32:0/3871120236 <== mon.2 v2:172.21.15.121:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fce6c0bd2c0 con 0x7fce74103350 2024-04-25T00:33:21.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.657+0000 7fce78aa3640 1 -- 172.21.15.32:0/3871120236 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fce7402d2f0 con 0x7fce74103350 2024-04-25T00:33:21.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.659+0000 7fce637fe640 1 -- 172.21.15.32:0/3871120236 <== mon.2 v2:172.21.15.121:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v94) ==== 74+0+25219 (secure 0 0 0) 0x7fce6c0bcbc0 con 0x7fce74103350 2024-04-25T00:33:21.660 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:33:21.660 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":94,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:33:17.919932+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-25T00:30:46.239556+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-25T00:30:47.257182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2024-04-25T00:30:51.193975+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.b","mappings":[{"from":8,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":1}]},{"pgid":"2.f","mappings":[{"from":8,"to":11}]},{"pgid":"2.10","mappings":[{"from":2,"to":4}]},{"pgid":"3.6","mappings":[{"from":3,"to":6}]},{"pgid":"3.17","mappings":[{"from":4,"to":7}]},{"pgid":"3.1b","mappings":[{"from":5,"to":9}]},{"pgid":"3.20","mappings":[{"from":5,"to":9}]},{"pgid":"3.23","mappings":[{"from":4,"to":8}]},{"pgid":"3.2c","mappings":[{"from":4,"to":9}]},{"pgid":"4.c","mappings":[{"from":9,"to":10}]},{"pgid":"4.12","mappings":[{"from":9,"to":10}]},{"pgid":"4.1c","mappings":[{"from":9,"to":8}]},{"pgid":"4.1d","mappings":[{"from":5,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:6801/585810931":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/1096527590":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/1767429535":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/3915353181":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6800/585810931":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/1694990206":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-num-failure-domains":"0","crush-osds-per-failure-domain":"0","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:33:21.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.661+0000 7fce78aa3640 1 -- 172.21.15.32:0/3871120236 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fce5c0a2d80 msgr2=0x7fce5c0a5240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:21.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.661+0000 7fce78aa3640 1 --2- 172.21.15.32:0/3871120236 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fce5c0a2d80 0x7fce5c0a5240 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7fce64005fd0 tx=0x7fce6400b410 comp rx=0 tx=0).stop 2024-04-25T00:33:21.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.661+0000 7fce78aa3640 1 -- 172.21.15.32:0/3871120236 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fce74103350 msgr2=0x7fce74074070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:21.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.661+0000 7fce78aa3640 1 --2- 172.21.15.32:0/3871120236 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fce74103350 0x7fce74074070 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7fce6c031ae0 tx=0x7fce6c031ee0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.661+0000 7fce78aa3640 1 -- 172.21.15.32:0/3871120236 shutdown_connections 2024-04-25T00:33:21.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.661+0000 7fce78aa3640 1 --2- 172.21.15.32:0/3871120236 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fce5c0a2d80 0x7fce5c0a5240 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.661+0000 7fce78aa3640 1 --2- 172.21.15.32:0/3871120236 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fce74103350 0x7fce74074070 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.661+0000 7fce78aa3640 1 --2- 172.21.15.32:0/3871120236 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fce74102990 0x7fce74073b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.661+0000 7fce78aa3640 1 --2- 172.21.15.32:0/3871120236 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fce741017a0 0x7fce740754f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.661+0000 7fce78aa3640 1 -- 172.21.15.32:0/3871120236 >> 172.21.15.32:0/3871120236 conn(0x7fce740fccf0 msgr2=0x7fce74105c90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:21.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.662+0000 7fce78aa3640 1 -- 172.21.15.32:0/3871120236 shutdown_connections 2024-04-25T00:33:21.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.662+0000 7fce78aa3640 1 -- 172.21.15.32:0/3871120236 wait complete. 2024-04-25T00:33:21.673 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:21 smithi032 ceph-mon[29357]: pgmap v4: 129 pgs: 129 active+clean; 1.1 MiB data, 373 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:33:21.674 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:21 smithi032 ceph-mon[29357]: mgrmap e23: x(active, since 2s), standbys: z, y 2024-04-25T00:33:21.674 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:21 smithi032 ceph-mon[29357]: from='client.25232 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:33:21.674 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:21 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:21.674 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:21 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:21.674 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:21 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:21.674 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:21 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.674 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:21 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:21.674 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:21 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.674 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:21 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.674 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:21 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.674 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:21 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.674 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:21 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.675 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:21 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.675 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:21 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.675 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:21 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:21.675 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:21 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:21.675 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:21 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.675 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:21 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.675 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:21 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.675 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:21 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.675 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:21 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:33:21.675 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:21 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:33:21.676 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-25T00:26:07.306839+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-04-25T00:30:46.239556+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '84', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.8799999952316284, 'score_stable': 1.8799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.8799999952316284, 'raw_score_stable': 1.8799999952316284, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-04-25T00:30:47.257182+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '85', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.05000000074505806}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_data_ec', 'create_time': '2024-04-25T00:30:51.193975+0000', 'flags': 5, 'flags_names': 'hashpspool,ec_overwrites', 'type': 3, 'size': 4, 'min_size': 3, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '89', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'cephfs_data_ec', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 8192, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}] 2024-04-25T00:33:21.677 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-25T00:33:21.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.785+0000 7fdac3e66640 1 -- 172.21.15.32:0/3271024307 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdabc100680 msgr2=0x7fdabc100aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:21.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.785+0000 7fdac3e66640 1 --2- 172.21.15.32:0/3271024307 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdabc100680 0x7fdabc100aa0 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7fdab8009860 tx=0x7fdab802a340 comp rx=0 tx=0).stop 2024-04-25T00:33:21.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.785+0000 7fdac3e66640 1 -- 172.21.15.32:0/3271024307 shutdown_connections 2024-04-25T00:33:21.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.785+0000 7fdac3e66640 1 --2- 172.21.15.32:0/3271024307 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdabc102230 0x7fdabc108ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.785+0000 7fdac3e66640 1 --2- 172.21.15.32:0/3271024307 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdabc101870 0x7fdabc101cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.785+0000 7fdac3e66640 1 --2- 172.21.15.32:0/3271024307 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdabc100680 0x7fdabc100aa0 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.785+0000 7fdac3e66640 1 -- 172.21.15.32:0/3271024307 >> 172.21.15.32:0/3271024307 conn(0x7fdabc0fbbf0 msgr2=0x7fdabc0fe030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:21.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.785+0000 7fdac3e66640 1 -- 172.21.15.32:0/3271024307 shutdown_connections 2024-04-25T00:33:21.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.785+0000 7fdac3e66640 1 -- 172.21.15.32:0/3271024307 wait complete. 2024-04-25T00:33:21.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.786+0000 7fdac3e66640 1 Processor -- start 2024-04-25T00:33:21.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.786+0000 7fdac3e66640 1 -- start start 2024-04-25T00:33:21.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.786+0000 7fdac3e66640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdabc100680 0x7fdabc1969e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:21.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.786+0000 7fdac3e66640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdabc101870 0x7fdabc196f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:21.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.786+0000 7fdac3e66640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdabc102230 0x7fdabc19df90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:21.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.786+0000 7fdac3e66640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fdabc10c2d0 con 0x7fdabc100680 2024-04-25T00:33:21.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.786+0000 7fdac3e66640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fdabc10c150 con 0x7fdabc102230 2024-04-25T00:33:21.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.786+0000 7fdac3e66640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fdabc10c450 con 0x7fdabc101870 2024-04-25T00:33:21.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.786+0000 7fdac1c02640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdabc100680 0x7fdabc1969e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:21.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.787+0000 7fdac1c02640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdabc100680 0x7fdabc1969e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:57842/0 (socket says 172.21.15.32:57842) 2024-04-25T00:33:21.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.787+0000 7fdac1c02640 1 -- 172.21.15.32:0/2382167167 learned_addr learned my addr 172.21.15.32:0/2382167167 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:21.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.787+0000 7fdac2403640 1 --2- 172.21.15.32:0/2382167167 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdabc102230 0x7fdabc19df90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:21.790 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.787+0000 7fdac1401640 1 --2- 172.21.15.32:0/2382167167 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdabc101870 0x7fdabc196f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:21.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.787+0000 7fdac1c02640 1 -- 172.21.15.32:0/2382167167 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdabc101870 msgr2=0x7fdabc196f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:21.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.787+0000 7fdac1c02640 1 --2- 172.21.15.32:0/2382167167 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdabc101870 0x7fdabc196f20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.787+0000 7fdac1c02640 1 -- 172.21.15.32:0/2382167167 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdabc102230 msgr2=0x7fdabc19df90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:21.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.787+0000 7fdac1c02640 1 --2- 172.21.15.32:0/2382167167 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdabc102230 0x7fdabc19df90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:21.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.787+0000 7fdac1c02640 1 -- 172.21.15.32:0/2382167167 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fdabc19e600 con 0x7fdabc100680 2024-04-25T00:33:21.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.787+0000 7fdac2403640 1 --2- 172.21.15.32:0/2382167167 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdabc102230 0x7fdabc19df90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:21.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.787+0000 7fdac1c02640 1 --2- 172.21.15.32:0/2382167167 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdabc100680 0x7fdabc1969e0 secure :-1 s=READY pgs=490 cs=0 l=1 rev1=1 crypto rx=0x7fdab8035040 tx=0x7fdab8033d90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:21.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.787+0000 7fdaaeffd640 1 -- 172.21.15.32:0/2382167167 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdab8037040 con 0x7fdabc100680 2024-04-25T00:33:21.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.787+0000 7fdaaeffd640 1 -- 172.21.15.32:0/2382167167 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fdab8030850 con 0x7fdabc100680 2024-04-25T00:33:21.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.787+0000 7fdac3e66640 1 -- 172.21.15.32:0/2382167167 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fdabc19e890 con 0x7fdabc100680 2024-04-25T00:33:21.792 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.787+0000 7fdaaeffd640 1 -- 172.21.15.32:0/2382167167 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdab8004920 con 0x7fdabc100680 2024-04-25T00:33:21.792 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.787+0000 7fdac3e66640 1 -- 172.21.15.32:0/2382167167 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fdabc19ed80 con 0x7fdabc100680 2024-04-25T00:33:21.792 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.788+0000 7fdac3e66640 1 -- 172.21.15.32:0/2382167167 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fdabc107b60 con 0x7fdabc100680 2024-04-25T00:33:21.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.789+0000 7fdaaeffd640 1 -- 172.21.15.32:0/2382167167 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 23) ==== 137953+0+0 (secure 0 0 0) 0x7fdab8004050 con 0x7fdabc100680 2024-04-25T00:33:21.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.790+0000 7fdaaeffd640 1 --2- 172.21.15.32:0/2382167167 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fdaa80a2d00 0x7fdaa80a51c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:21.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.790+0000 7fdac1401640 1 --2- 172.21.15.32:0/2382167167 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fdaa80a2d00 0x7fdaa80a51c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:21.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.791+0000 7fdaaeffd640 1 -- 172.21.15.32:0/2382167167 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fdab80ef4a0 con 0x7fdabc100680 2024-04-25T00:33:21.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.791+0000 7fdac1401640 1 --2- 172.21.15.32:0/2382167167 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fdaa80a2d00 0x7fdaa80a51c0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7fdabc197ef0 tx=0x7fdab0005f90 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:21.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:21.795+0000 7fdaaeffd640 1 -- 172.21.15.32:0/2382167167 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fdab80bbec0 con 0x7fdabc100680 2024-04-25T00:33:21.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:21 smithi121 ceph-mon[32231]: pgmap v4: 129 pgs: 129 active+clean; 1.1 MiB data, 373 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:33:21.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:21 smithi121 ceph-mon[32231]: mgrmap e23: x(active, since 2s), standbys: z, y 2024-04-25T00:33:21.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:21 smithi121 ceph-mon[32231]: from='client.25232 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:33:21.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:21 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:21.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:21 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:21.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:21 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:21.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:21 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:21 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:21.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:21 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:21 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:21 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:21 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:21 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:21 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:21 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:21 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:21.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:21 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:21.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:21 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:21 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:21 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:21 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:21 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:33:21.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:21 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:33:21.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:21 smithi130 ceph-mon[32525]: pgmap v4: 129 pgs: 129 active+clean; 1.1 MiB data, 373 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:33:21.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:21 smithi130 ceph-mon[32525]: mgrmap e23: x(active, since 2s), standbys: z, y 2024-04-25T00:33:21.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:21 smithi130 ceph-mon[32525]: from='client.25232 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:33:21.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:21 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:21.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:21 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:21.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:21 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:21.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:21 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:21 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:21.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:21 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:21 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:21 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:21 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:21 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:21 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:21 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:21 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:21.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:21 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:21.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:21 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.895 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:21 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.895 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:21 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.895 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:21 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:33:21.895 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:21 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:33:21.895 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:21 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:33:22.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.002+0000 7fdac3e66640 1 -- 172.21.15.32:0/2382167167 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7fdabc02d3c0 con 0x7fdabc100680 2024-04-25T00:33:22.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.002+0000 7fdaaeffd640 1 -- 172.21.15.32:0/2382167167 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v94) ==== 93+0+10 (secure 0 0 0) 0x7fdab80bb7c0 con 0x7fdabc100680 2024-04-25T00:33:22.008 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 1 2024-04-25T00:33:22.009 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.005+0000 7fdac3e66640 1 -- 172.21.15.32:0/2382167167 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fdaa80a2d00 msgr2=0x7fdaa80a51c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:22.009 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.005+0000 7fdac3e66640 1 --2- 172.21.15.32:0/2382167167 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fdaa80a2d00 0x7fdaa80a51c0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7fdabc197ef0 tx=0x7fdab0005f90 comp rx=0 tx=0).stop 2024-04-25T00:33:22.009 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.005+0000 7fdac3e66640 1 -- 172.21.15.32:0/2382167167 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdabc100680 msgr2=0x7fdabc1969e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:22.010 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.005+0000 7fdac3e66640 1 --2- 172.21.15.32:0/2382167167 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdabc100680 0x7fdabc1969e0 secure :-1 s=READY pgs=490 cs=0 l=1 rev1=1 crypto rx=0x7fdab8035040 tx=0x7fdab8033d90 comp rx=0 tx=0).stop 2024-04-25T00:33:22.010 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.005+0000 7fdac3e66640 1 -- 172.21.15.32:0/2382167167 shutdown_connections 2024-04-25T00:33:22.010 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.005+0000 7fdac3e66640 1 --2- 172.21.15.32:0/2382167167 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fdaa80a2d00 0x7fdaa80a51c0 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.010 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.005+0000 7fdac3e66640 1 --2- 172.21.15.32:0/2382167167 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fdabc102230 0x7fdabc19df90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.010 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.005+0000 7fdac3e66640 1 --2- 172.21.15.32:0/2382167167 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdabc101870 0x7fdabc196f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.011 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.005+0000 7fdac3e66640 1 --2- 172.21.15.32:0/2382167167 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fdabc100680 0x7fdabc1969e0 unknown :-1 s=CLOSED pgs=490 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.011 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.005+0000 7fdac3e66640 1 -- 172.21.15.32:0/2382167167 >> 172.21.15.32:0/2382167167 conn(0x7fdabc0fbbf0 msgr2=0x7fdabc104d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:22.011 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.005+0000 7fdac3e66640 1 -- 172.21.15.32:0/2382167167 shutdown_connections 2024-04-25T00:33:22.011 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.005+0000 7fdac3e66640 1 -- 172.21.15.32:0/2382167167 wait complete. 2024-04-25T00:33:22.018 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-04-25T00:33:22.138 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.134+0000 7ffbb7664640 1 -- 172.21.15.32:0/1983069420 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffbb0100970 msgr2=0x7ffbb0100dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:22.139 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.134+0000 7ffbb7664640 1 --2- 172.21.15.32:0/1983069420 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffbb0100970 0x7ffbb0100dd0 secure :-1 s=READY pgs=491 cs=0 l=1 rev1=1 crypto rx=0x7ffba0009900 tx=0x7ffba002a410 comp rx=0 tx=0).stop 2024-04-25T00:33:22.139 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.134+0000 7ffbb7664640 1 -- 172.21.15.32:0/1983069420 shutdown_connections 2024-04-25T00:33:22.139 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.134+0000 7ffbb7664640 1 --2- 172.21.15.32:0/1983069420 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffbb0101310 0x7ffbb0101790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.139 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.134+0000 7ffbb7664640 1 --2- 172.21.15.32:0/1983069420 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffbb0100970 0x7ffbb0100dd0 unknown :-1 s=CLOSED pgs=491 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.139 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.134+0000 7ffbb7664640 1 --2- 172.21.15.32:0/1983069420 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ffbb0100010 0x7ffbb0100430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.139 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.134+0000 7ffbb7664640 1 -- 172.21.15.32:0/1983069420 >> 172.21.15.32:0/1983069420 conn(0x7ffbb00fbc30 msgr2=0x7ffbb00fe050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:22.139 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.134+0000 7ffbb7664640 1 -- 172.21.15.32:0/1983069420 shutdown_connections 2024-04-25T00:33:22.139 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.134+0000 7ffbb7664640 1 -- 172.21.15.32:0/1983069420 wait complete. 2024-04-25T00:33:22.139 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.134+0000 7ffbb7664640 1 Processor -- start 2024-04-25T00:33:22.139 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.134+0000 7ffbb7664640 1 -- start start 2024-04-25T00:33:22.139 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.135+0000 7ffbb7664640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ffbb0100010 0x7ffbb0199c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:22.139 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.135+0000 7ffbb7664640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffbb0100970 0x7ffbb019a190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:22.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.135+0000 7ffbb7664640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffbb0101310 0x7ffbb01a11c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:22.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.135+0000 7ffbb7664640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7ffbb010f5e0 con 0x7ffbb0100970 2024-04-25T00:33:22.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.135+0000 7ffbb7664640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7ffbb010f460 con 0x7ffbb0100010 2024-04-25T00:33:22.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.135+0000 7ffbb7664640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7ffbb010f760 con 0x7ffbb0101310 2024-04-25T00:33:22.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.135+0000 7ffbb4bff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffbb0100970 0x7ffbb019a190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:22.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.135+0000 7ffbb4bff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffbb0100970 0x7ffbb019a190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:57858/0 (socket says 172.21.15.32:57858) 2024-04-25T00:33:22.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.135+0000 7ffbb4bff640 1 -- 172.21.15.32:0/1074999696 learned_addr learned my addr 172.21.15.32:0/1074999696 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:22.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.135+0000 7ffbb5c01640 1 --2- 172.21.15.32:0/1074999696 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffbb0101310 0x7ffbb01a11c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:22.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.135+0000 7ffbb5400640 1 --2- 172.21.15.32:0/1074999696 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ffbb0100010 0x7ffbb0199c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:22.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.135+0000 7ffbb4bff640 1 -- 172.21.15.32:0/1074999696 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffbb0101310 msgr2=0x7ffbb01a11c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:22.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.135+0000 7ffbb4bff640 1 --2- 172.21.15.32:0/1074999696 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffbb0101310 0x7ffbb01a11c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.135+0000 7ffbb4bff640 1 -- 172.21.15.32:0/1074999696 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ffbb0100010 msgr2=0x7ffbb0199c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:22.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.135+0000 7ffbb4bff640 1 --2- 172.21.15.32:0/1074999696 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ffbb0100010 0x7ffbb0199c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.141 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.135+0000 7ffbb4bff640 1 -- 172.21.15.32:0/1074999696 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ffbb01a1860 con 0x7ffbb0100970 2024-04-25T00:33:22.141 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.135+0000 7ffbb5400640 1 --2- 172.21.15.32:0/1074999696 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ffbb0100010 0x7ffbb0199c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:22.141 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.135+0000 7ffbb4bff640 1 --2- 172.21.15.32:0/1074999696 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffbb0100970 0x7ffbb019a190 secure :-1 s=READY pgs=492 cs=0 l=1 rev1=1 crypto rx=0x7ffba0009230 tx=0x7ffba0004360 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:22.141 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.135+0000 7ffba67fc640 1 -- 172.21.15.32:0/1074999696 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffba003f030 con 0x7ffbb0100970 2024-04-25T00:33:22.141 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.136+0000 7ffbb7664640 1 -- 172.21.15.32:0/1074999696 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ffbb01a1af0 con 0x7ffbb0100970 2024-04-25T00:33:22.141 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.136+0000 7ffba67fc640 1 -- 172.21.15.32:0/1074999696 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ffba0003ce0 con 0x7ffbb0100970 2024-04-25T00:33:22.141 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.136+0000 7ffba67fc640 1 -- 172.21.15.32:0/1074999696 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffba0035070 con 0x7ffbb0100970 2024-04-25T00:33:22.141 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.136+0000 7ffbb7664640 1 -- 172.21.15.32:0/1074999696 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ffbb01a2030 con 0x7ffbb0100970 2024-04-25T00:33:22.141 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.137+0000 7ffbb7664640 1 -- 172.21.15.32:0/1074999696 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ffbb010ad60 con 0x7ffbb0100970 2024-04-25T00:33:22.142 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.138+0000 7ffba67fc640 1 -- 172.21.15.32:0/1074999696 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 23) ==== 137953+0+0 (secure 0 0 0) 0x7ffba0042030 con 0x7ffbb0100970 2024-04-25T00:33:22.142 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.138+0000 7ffba67fc640 1 --2- 172.21.15.32:0/1074999696 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7ffb9c0a2d80 0x7ffb9c0a5240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:22.142 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.139+0000 7ffbb5400640 1 --2- 172.21.15.32:0/1074999696 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7ffb9c0a2d80 0x7ffb9c0a5240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:22.142 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.139+0000 7ffba67fc640 1 -- 172.21.15.32:0/1074999696 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7ffba00f0330 con 0x7ffbb0100970 2024-04-25T00:33:22.142 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.139+0000 7ffbb5400640 1 --2- 172.21.15.32:0/1074999696 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7ffb9c0a2d80 0x7ffb9c0a5240 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7ffbac005d00 tx=0x7ffbac005c90 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:22.144 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.143+0000 7ffba67fc640 1 -- 172.21.15.32:0/1074999696 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7ffba00bcb40 con 0x7ffbb0100970 2024-04-25T00:33:22.349 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.348+0000 7ffbb7664640 1 -- 172.21.15.32:0/1074999696 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) -- 0x7ffbb002d370 con 0x7ffbb0100970 2024-04-25T00:33:22.349 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.348+0000 7ffba67fc640 1 -- 172.21.15.32:0/1074999696 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v94) ==== 104+0+11 (secure 0 0 0) 0x7ffba00bc440 con 0x7ffbb0100970 2024-04-25T00:33:22.350 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 32 2024-04-25T00:33:22.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.351+0000 7ffbb7664640 1 -- 172.21.15.32:0/1074999696 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7ffb9c0a2d80 msgr2=0x7ffb9c0a5240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:22.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.352+0000 7ffbb7664640 1 --2- 172.21.15.32:0/1074999696 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7ffb9c0a2d80 0x7ffb9c0a5240 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7ffbac005d00 tx=0x7ffbac005c90 comp rx=0 tx=0).stop 2024-04-25T00:33:22.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.352+0000 7ffbb7664640 1 -- 172.21.15.32:0/1074999696 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffbb0100970 msgr2=0x7ffbb019a190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:22.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.352+0000 7ffbb7664640 1 --2- 172.21.15.32:0/1074999696 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffbb0100970 0x7ffbb019a190 secure :-1 s=READY pgs=492 cs=0 l=1 rev1=1 crypto rx=0x7ffba0009230 tx=0x7ffba0004360 comp rx=0 tx=0).stop 2024-04-25T00:33:22.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.352+0000 7ffbb7664640 1 -- 172.21.15.32:0/1074999696 shutdown_connections 2024-04-25T00:33:22.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.352+0000 7ffbb7664640 1 --2- 172.21.15.32:0/1074999696 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7ffb9c0a2d80 0x7ffb9c0a5240 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.352+0000 7ffbb7664640 1 --2- 172.21.15.32:0/1074999696 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffbb0101310 0x7ffbb01a11c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.352+0000 7ffbb7664640 1 --2- 172.21.15.32:0/1074999696 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffbb0100970 0x7ffbb019a190 unknown :-1 s=CLOSED pgs=492 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.352+0000 7ffbb7664640 1 --2- 172.21.15.32:0/1074999696 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ffbb0100010 0x7ffbb0199c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.352+0000 7ffbb7664640 1 -- 172.21.15.32:0/1074999696 >> 172.21.15.32:0/1074999696 conn(0x7ffbb00fbc30 msgr2=0x7ffbb00fd730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:22.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.352+0000 7ffbb7664640 1 -- 172.21.15.32:0/1074999696 shutdown_connections 2024-04-25T00:33:22.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.352+0000 7ffbb7664640 1 -- 172.21.15.32:0/1074999696 wait complete. 2024-04-25T00:33:22.367 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-04-25T00:33:22.480 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.477+0000 7fea2d729640 1 -- 172.21.15.32:0/3145165296 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea28106650 msgr2=0x7fea28106a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:22.480 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.477+0000 7fea2d729640 1 --2- 172.21.15.32:0/3145165296 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea28106650 0x7fea28106a30 secure :-1 s=READY pgs=493 cs=0 l=1 rev1=1 crypto rx=0x7fea18009900 tx=0x7fea1802a480 comp rx=0 tx=0).stop 2024-04-25T00:33:22.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.477+0000 7fea2d729640 1 -- 172.21.15.32:0/3145165296 shutdown_connections 2024-04-25T00:33:22.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.477+0000 7fea2d729640 1 --2- 172.21.15.32:0/3145165296 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fea28101720 0x7fea2810d4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.477+0000 7fea2d729640 1 --2- 172.21.15.32:0/3145165296 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea28100d60 0x7fea281011e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.477+0000 7fea2d729640 1 --2- 172.21.15.32:0/3145165296 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea28106650 0x7fea28106a30 unknown :-1 s=CLOSED pgs=493 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.477+0000 7fea2d729640 1 -- 172.21.15.32:0/3145165296 >> 172.21.15.32:0/3145165296 conn(0x7fea28076be0 msgr2=0x7fea280fdf60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:22.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.477+0000 7fea2d729640 1 -- 172.21.15.32:0/3145165296 shutdown_connections 2024-04-25T00:33:22.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.477+0000 7fea2d729640 1 -- 172.21.15.32:0/3145165296 wait complete. 2024-04-25T00:33:22.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.477+0000 7fea2d729640 1 Processor -- start 2024-04-25T00:33:22.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.477+0000 7fea2d729640 1 -- start start 2024-04-25T00:33:22.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.478+0000 7fea2d729640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea28100d60 0x7fea28198080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:22.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.478+0000 7fea2d729640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fea28101720 0x7fea281985c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:22.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.478+0000 7fea2d729640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea28106650 0x7fea2819c8f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:22.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.478+0000 7fea2d729640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fea28110a70 con 0x7fea28100d60 2024-04-25T00:33:22.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.478+0000 7fea2d729640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fea281108f0 con 0x7fea28101720 2024-04-25T00:33:22.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.478+0000 7fea2d729640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fea28110bf0 con 0x7fea28106650 2024-04-25T00:33:22.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.478+0000 7fea26ffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea28100d60 0x7fea28198080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:22.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.478+0000 7fea26ffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea28100d60 0x7fea28198080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:57866/0 (socket says 172.21.15.32:57866) 2024-04-25T00:33:22.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.478+0000 7fea26ffd640 1 -- 172.21.15.32:0/3856611077 learned_addr learned my addr 172.21.15.32:0/3856611077 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:22.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.478+0000 7fea1ffff640 1 --2- 172.21.15.32:0/3856611077 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fea28101720 0x7fea281985c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:22.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.478+0000 7fea277fe640 1 --2- 172.21.15.32:0/3856611077 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea28106650 0x7fea2819c8f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:22.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.478+0000 7fea1ffff640 1 -- 172.21.15.32:0/3856611077 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea28106650 msgr2=0x7fea2819c8f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:22.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.478+0000 7fea1ffff640 1 --2- 172.21.15.32:0/3856611077 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea28106650 0x7fea2819c8f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.478+0000 7fea1ffff640 1 -- 172.21.15.32:0/3856611077 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea28100d60 msgr2=0x7fea28198080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:22.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.478+0000 7fea1ffff640 1 --2- 172.21.15.32:0/3856611077 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea28100d60 0x7fea28198080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.478+0000 7fea1ffff640 1 -- 172.21.15.32:0/3856611077 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fea2819d020 con 0x7fea28101720 2024-04-25T00:33:22.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.478+0000 7fea26ffd640 1 --2- 172.21.15.32:0/3856611077 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea28100d60 0x7fea28198080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:22.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.478+0000 7fea1ffff640 1 --2- 172.21.15.32:0/3856611077 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fea28101720 0x7fea281985c0 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7fea1000d720 tx=0x7fea1000dbe0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:22.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.479+0000 7fea24ff9640 1 -- 172.21.15.32:0/3856611077 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fea10017070 con 0x7fea28101720 2024-04-25T00:33:22.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.479+0000 7fea24ff9640 1 -- 172.21.15.32:0/3856611077 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fea1000de30 con 0x7fea28101720 2024-04-25T00:33:22.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.479+0000 7fea2d729640 1 -- 172.21.15.32:0/3856611077 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fea2819d310 con 0x7fea28101720 2024-04-25T00:33:22.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.479+0000 7fea24ff9640 1 -- 172.21.15.32:0/3856611077 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fea100047a0 con 0x7fea28101720 2024-04-25T00:33:22.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.479+0000 7fea2d729640 1 -- 172.21.15.32:0/3856611077 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fea281a4ba0 con 0x7fea28101720 2024-04-25T00:33:22.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.480+0000 7fea2d729640 1 -- 172.21.15.32:0/3856611077 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fea28067cd0 con 0x7fea28101720 2024-04-25T00:33:22.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.481+0000 7fea24ff9640 1 -- 172.21.15.32:0/3856611077 <== mon.2 v2:172.21.15.121:3300/0 4 ==== mgrmap(e 23) ==== 137953+0+0 (secure 0 0 0) 0x7fea100099d0 con 0x7fea28101720 2024-04-25T00:33:22.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.481+0000 7fea24ff9640 1 --2- 172.21.15.32:0/3856611077 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fea140a2d00 0x7fea140a51c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:22.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.481+0000 7fea26ffd640 1 --2- 172.21.15.32:0/3856611077 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fea140a2d00 0x7fea140a51c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:22.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.482+0000 7fea24ff9640 1 -- 172.21.15.32:0/3856611077 <== mon.2 v2:172.21.15.121:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fea100cf3b0 con 0x7fea28101720 2024-04-25T00:33:22.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.482+0000 7fea26ffd640 1 --2- 172.21.15.32:0/3856611077 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fea140a2d00 0x7fea140a51c0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7fea18000c00 tx=0x7fea180046c0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:22.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.487+0000 7fea24ff9640 1 -- 172.21.15.32:0/3856611077 <== mon.2 v2:172.21.15.121:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fea10020020 con 0x7fea28101720 2024-04-25T00:33:22.509 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.508+0000 7fea24ff9640 1 -- 172.21.15.32:0/3856611077 <== mon.2 v2:172.21.15.121:3300/0 7 ==== mgrmap(e 24) ==== 138009+0+0 (secure 0 0 0) 0x7fea100cc9d0 con 0x7fea28101720 2024-04-25T00:33:22.693 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.692+0000 7fea2d729640 1 -- 172.21.15.32:0/3856611077 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) -- 0x7fea281a4dd0 con 0x7fea28101720 2024-04-25T00:33:22.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.692+0000 7fea24ff9640 1 -- 172.21.15.32:0/3856611077 <== mon.2 v2:172.21.15.121:3300/0 8 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v94) ==== 100+0+11 (secure 0 0 0) 0x7fea1009bc20 con 0x7fea28101720 2024-04-25T00:33:22.694 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 64 2024-04-25T00:33:22.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.694+0000 7fea2d729640 1 -- 172.21.15.32:0/3856611077 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fea140a2d00 msgr2=0x7fea140a51c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:22.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.694+0000 7fea2d729640 1 --2- 172.21.15.32:0/3856611077 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fea140a2d00 0x7fea140a51c0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7fea18000c00 tx=0x7fea180046c0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.694+0000 7fea2d729640 1 -- 172.21.15.32:0/3856611077 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fea28101720 msgr2=0x7fea281985c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:22.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.694+0000 7fea2d729640 1 --2- 172.21.15.32:0/3856611077 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fea28101720 0x7fea281985c0 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7fea1000d720 tx=0x7fea1000dbe0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.694+0000 7fea2d729640 1 -- 172.21.15.32:0/3856611077 shutdown_connections 2024-04-25T00:33:22.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.694+0000 7fea2d729640 1 --2- 172.21.15.32:0/3856611077 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fea140a2d00 0x7fea140a51c0 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.694+0000 7fea2d729640 1 --2- 172.21.15.32:0/3856611077 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea28106650 0x7fea2819c8f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.694+0000 7fea2d729640 1 --2- 172.21.15.32:0/3856611077 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fea28101720 0x7fea281985c0 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.694+0000 7fea2d729640 1 --2- 172.21.15.32:0/3856611077 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea28100d60 0x7fea28198080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.694+0000 7fea2d729640 1 -- 172.21.15.32:0/3856611077 >> 172.21.15.32:0/3856611077 conn(0x7fea28076be0 msgr2=0x7fea280fe9d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:22.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.694+0000 7fea2d729640 1 -- 172.21.15.32:0/3856611077 shutdown_connections 2024-04-25T00:33:22.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.694+0000 7fea2d729640 1 -- 172.21.15.32:0/3856611077 wait complete. 2024-04-25T00:33:22.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:22 smithi032 ceph-mon[29357]: from='client.15627 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:33:22.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:22 smithi032 ceph-mon[29357]: Adjusting osd_memory_target on smithi121 to 176.1M 2024-04-25T00:33:22.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:22 smithi032 ceph-mon[29357]: Unable to set osd_memory_target on smithi121 to 184727961: error parsing value: Value '184727961' is below minimum 939524096 2024-04-25T00:33:22.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:22 smithi032 ceph-mon[29357]: Adjusting osd_memory_target on smithi130 to 176.2M 2024-04-25T00:33:22.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:22 smithi032 ceph-mon[29357]: Unable to set osd_memory_target on smithi130 to 184770252: error parsing value: Value '184770252' is below minimum 939524096 2024-04-25T00:33:22.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:22 smithi032 ceph-mon[29357]: from='client.15633 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:33:22.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:22 smithi032 ceph-mon[29357]: Adjusting osd_memory_target on smithi032 to 177.2M 2024-04-25T00:33:22.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:22 smithi032 ceph-mon[29357]: Unable to set osd_memory_target on smithi032 to 185897062: error parsing value: Value '185897062' is below minimum 939524096 2024-04-25T00:33:22.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:22 smithi032 ceph-mon[29357]: Updating smithi032:/etc/ceph/ceph.conf 2024-04-25T00:33:22.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:22 smithi032 ceph-mon[29357]: Updating smithi121:/etc/ceph/ceph.conf 2024-04-25T00:33:22.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:22 smithi032 ceph-mon[29357]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-25T00:33:22.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:22 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3871120236' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:33:22.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:22 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2382167167' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:33:22.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:22 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1074999696' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:33:22.708 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data_ec pg_num 2024-04-25T00:33:22.819 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.815+0000 7f15c823f640 1 -- 172.21.15.32:0/1646295291 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f15c0107720 msgr2=0x7f15c0107b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:22.819 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.815+0000 7f15c823f640 1 --2- 172.21.15.32:0/1646295291 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f15c0107720 0x7f15c0107b00 secure :-1 s=READY pgs=494 cs=0 l=1 rev1=1 crypto rx=0x7f15b0009930 tx=0x7f15b002a420 comp rx=0 tx=0).stop 2024-04-25T00:33:22.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.815+0000 7f15c823f640 1 -- 172.21.15.32:0/1646295291 shutdown_connections 2024-04-25T00:33:22.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.815+0000 7f15c823f640 1 --2- 172.21.15.32:0/1646295291 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f15c01020d0 0x7f15c010e5e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.815+0000 7f15c823f640 1 --2- 172.21.15.32:0/1646295291 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f15c0101710 0x7f15c0101b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.815+0000 7f15c823f640 1 --2- 172.21.15.32:0/1646295291 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f15c0107720 0x7f15c0107b00 unknown :-1 s=CLOSED pgs=494 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.815+0000 7f15c823f640 1 -- 172.21.15.32:0/1646295291 >> 172.21.15.32:0/1646295291 conn(0x7f15c00fd180 msgr2=0x7f15c00ff5a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:22.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.815+0000 7f15c823f640 1 -- 172.21.15.32:0/1646295291 shutdown_connections 2024-04-25T00:33:22.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.815+0000 7f15c823f640 1 -- 172.21.15.32:0/1646295291 wait complete. 2024-04-25T00:33:22.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.816+0000 7f15c823f640 1 Processor -- start 2024-04-25T00:33:22.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.816+0000 7f15c823f640 1 -- start start 2024-04-25T00:33:22.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.816+0000 7f15c823f640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f15c0101710 0x7f15c0073a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:22.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.816+0000 7f15c823f640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f15c01020d0 0x7f15c0073f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:22.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.816+0000 7f15c823f640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f15c0107720 0x7f15c00744c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:22.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.816+0000 7f15c823f640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f15c0111af0 con 0x7f15c0107720 2024-04-25T00:33:22.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.816+0000 7f15c823f640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f15c0111970 con 0x7f15c01020d0 2024-04-25T00:33:22.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.816+0000 7f15c823f640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f15c0111c70 con 0x7f15c0101710 2024-04-25T00:33:22.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.816+0000 7f15c67dc640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f15c0107720 0x7f15c00744c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:22.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.816+0000 7f15c67dc640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f15c0107720 0x7f15c00744c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:57876/0 (socket says 172.21.15.32:57876) 2024-04-25T00:33:22.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.816+0000 7f15c67dc640 1 -- 172.21.15.32:0/538135899 learned_addr learned my addr 172.21.15.32:0/538135899 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:22.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.816+0000 7f15c5fdb640 1 --2- 172.21.15.32:0/538135899 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f15c0101710 0x7f15c0073a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:22.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.816+0000 7f15c57da640 1 --2- 172.21.15.32:0/538135899 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f15c01020d0 0x7f15c0073f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:22.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.816+0000 7f15c67dc640 1 -- 172.21.15.32:0/538135899 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f15c0101710 msgr2=0x7f15c0073a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:22.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.816+0000 7f15c67dc640 1 --2- 172.21.15.32:0/538135899 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f15c0101710 0x7f15c0073a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.816+0000 7f15c67dc640 1 -- 172.21.15.32:0/538135899 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f15c01020d0 msgr2=0x7f15c0073f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:22.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.816+0000 7f15c67dc640 1 --2- 172.21.15.32:0/538135899 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f15c01020d0 0x7f15c0073f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:22.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.816+0000 7f15c67dc640 1 -- 172.21.15.32:0/538135899 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f15c01a64f0 con 0x7f15c0107720 2024-04-25T00:33:22.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.816+0000 7f15c57da640 1 --2- 172.21.15.32:0/538135899 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f15c01020d0 0x7f15c0073f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:33:22.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.817+0000 7f15c67dc640 1 --2- 172.21.15.32:0/538135899 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f15c0107720 0x7f15c00744c0 secure :-1 s=READY pgs=495 cs=0 l=1 rev1=1 crypto rx=0x7f15b800eaa0 tx=0x7f15b800ef60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:22.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.817+0000 7f15b6ffd640 1 -- 172.21.15.32:0/538135899 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f15b800cdb0 con 0x7f15c0107720 2024-04-25T00:33:22.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.817+0000 7f15b6ffd640 1 -- 172.21.15.32:0/538135899 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f15b8005020 con 0x7f15c0107720 2024-04-25T00:33:22.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.817+0000 7f15c823f640 1 -- 172.21.15.32:0/538135899 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f15c01a67e0 con 0x7f15c0107720 2024-04-25T00:33:22.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.817+0000 7f15b6ffd640 1 -- 172.21.15.32:0/538135899 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f15b80116b0 con 0x7f15c0107720 2024-04-25T00:33:22.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.817+0000 7f15c823f640 1 -- 172.21.15.32:0/538135899 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f15c01a6d30 con 0x7f15c0107720 2024-04-25T00:33:22.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.818+0000 7f15c823f640 1 -- 172.21.15.32:0/538135899 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f15c00689b0 con 0x7f15c0107720 2024-04-25T00:33:22.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.819+0000 7f15b6ffd640 1 -- 172.21.15.32:0/538135899 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 24) ==== 138009+0+0 (secure 0 0 0) 0x7f15b8005560 con 0x7f15c0107720 2024-04-25T00:33:22.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.820+0000 7f15b6ffd640 1 --2- 172.21.15.32:0/538135899 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f15ac0a2e10 0x7f15ac0a52d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:22.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.820+0000 7f15c5fdb640 1 --2- 172.21.15.32:0/538135899 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f15ac0a2e10 0x7f15ac0a52d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:22.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.820+0000 7f15b6ffd640 1 -- 172.21.15.32:0/538135899 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f15b80cfbd0 con 0x7f15c0107720 2024-04-25T00:33:22.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.820+0000 7f15c5fdb640 1 --2- 172.21.15.32:0/538135899 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f15ac0a2e10 0x7f15ac0a52d0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f15b0000c00 tx=0x7f15b0005a20 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:22.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:22.825+0000 7f15b6ffd640 1 -- 172.21.15.32:0/538135899 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f15b8016030 con 0x7f15c0107720 2024-04-25T00:33:22.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:22 smithi121 ceph-mon[32231]: from='client.15627 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:33:22.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:22 smithi121 ceph-mon[32231]: Adjusting osd_memory_target on smithi121 to 176.1M 2024-04-25T00:33:22.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:22 smithi121 ceph-mon[32231]: Unable to set osd_memory_target on smithi121 to 184727961: error parsing value: Value '184727961' is below minimum 939524096 2024-04-25T00:33:22.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:22 smithi121 ceph-mon[32231]: Adjusting osd_memory_target on smithi130 to 176.2M 2024-04-25T00:33:22.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:22 smithi121 ceph-mon[32231]: Unable to set osd_memory_target on smithi130 to 184770252: error parsing value: Value '184770252' is below minimum 939524096 2024-04-25T00:33:22.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:22 smithi121 ceph-mon[32231]: from='client.15633 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:33:22.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:22 smithi121 ceph-mon[32231]: Adjusting osd_memory_target on smithi032 to 177.2M 2024-04-25T00:33:22.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:22 smithi121 ceph-mon[32231]: Unable to set osd_memory_target on smithi032 to 185897062: error parsing value: Value '185897062' is below minimum 939524096 2024-04-25T00:33:22.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:22 smithi121 ceph-mon[32231]: Updating smithi032:/etc/ceph/ceph.conf 2024-04-25T00:33:22.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:22 smithi121 ceph-mon[32231]: Updating smithi121:/etc/ceph/ceph.conf 2024-04-25T00:33:22.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:22 smithi121 ceph-mon[32231]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-25T00:33:22.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:22 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3871120236' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:33:22.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:22 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2382167167' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:33:22.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:22 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1074999696' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:33:22.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:22 smithi130 ceph-mon[32525]: from='client.15627 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:33:22.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:22 smithi130 ceph-mon[32525]: Adjusting osd_memory_target on smithi121 to 176.1M 2024-04-25T00:33:22.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:22 smithi130 ceph-mon[32525]: Unable to set osd_memory_target on smithi121 to 184727961: error parsing value: Value '184727961' is below minimum 939524096 2024-04-25T00:33:22.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:22 smithi130 ceph-mon[32525]: Adjusting osd_memory_target on smithi130 to 176.2M 2024-04-25T00:33:22.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:22 smithi130 ceph-mon[32525]: Unable to set osd_memory_target on smithi130 to 184770252: error parsing value: Value '184770252' is below minimum 939524096 2024-04-25T00:33:22.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:22 smithi130 ceph-mon[32525]: from='client.15633 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T00:33:22.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:22 smithi130 ceph-mon[32525]: Adjusting osd_memory_target on smithi032 to 177.2M 2024-04-25T00:33:22.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:22 smithi130 ceph-mon[32525]: Unable to set osd_memory_target on smithi032 to 185897062: error parsing value: Value '185897062' is below minimum 939524096 2024-04-25T00:33:22.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:22 smithi130 ceph-mon[32525]: Updating smithi032:/etc/ceph/ceph.conf 2024-04-25T00:33:22.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:22 smithi130 ceph-mon[32525]: Updating smithi121:/etc/ceph/ceph.conf 2024-04-25T00:33:22.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:22 smithi130 ceph-mon[32525]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-25T00:33:22.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:22 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3871120236' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:33:22.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:22 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2382167167' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:33:22.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:22 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1074999696' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:33:23.014 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.012+0000 7f15c823f640 1 -- 172.21.15.32:0/538135899 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"} v 0) -- 0x7f15c002d170 con 0x7f15c0107720 2024-04-25T00:33:23.014 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.013+0000 7f15b6ffd640 1 -- 172.21.15.32:0/538135899 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]=0 v94) ==== 103+0+11 (secure 0 0 0) 0x7f15b809c2d0 con 0x7f15c0107720 2024-04-25T00:33:23.014 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 32 2024-04-25T00:33:23.016 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.014+0000 7f15c823f640 1 -- 172.21.15.32:0/538135899 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f15ac0a2e10 msgr2=0x7f15ac0a52d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.016 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.014+0000 7f15c823f640 1 --2- 172.21.15.32:0/538135899 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f15ac0a2e10 0x7f15ac0a52d0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f15b0000c00 tx=0x7f15b0005a20 comp rx=0 tx=0).stop 2024-04-25T00:33:23.017 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.014+0000 7f15c823f640 1 -- 172.21.15.32:0/538135899 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f15c0107720 msgr2=0x7f15c00744c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.017 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.014+0000 7f15c823f640 1 --2- 172.21.15.32:0/538135899 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f15c0107720 0x7f15c00744c0 secure :-1 s=READY pgs=495 cs=0 l=1 rev1=1 crypto rx=0x7f15b800eaa0 tx=0x7f15b800ef60 comp rx=0 tx=0).stop 2024-04-25T00:33:23.017 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.015+0000 7f15c823f640 1 -- 172.21.15.32:0/538135899 shutdown_connections 2024-04-25T00:33:23.017 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.015+0000 7f15c823f640 1 --2- 172.21.15.32:0/538135899 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f15ac0a2e10 0x7f15ac0a52d0 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.017 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.015+0000 7f15c823f640 1 --2- 172.21.15.32:0/538135899 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f15c0107720 0x7f15c00744c0 unknown :-1 s=CLOSED pgs=495 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.017 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.015+0000 7f15c823f640 1 --2- 172.21.15.32:0/538135899 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f15c01020d0 0x7f15c0073f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.017 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.015+0000 7f15c823f640 1 --2- 172.21.15.32:0/538135899 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f15c0101710 0x7f15c0073a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.017 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.015+0000 7f15c823f640 1 -- 172.21.15.32:0/538135899 >> 172.21.15.32:0/538135899 conn(0x7f15c00fd180 msgr2=0x7f15c00fd950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:23.017 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.015+0000 7f15c823f640 1 -- 172.21.15.32:0/538135899 shutdown_connections 2024-04-25T00:33:23.017 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.015+0000 7f15c823f640 1 -- 172.21.15.32:0/538135899 wait complete. 2024-04-25T00:33:23.027 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:33:23.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.128+0000 7eff7b51d640 1 -- 172.21.15.32:0/589547327 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff74106630 msgr2=0x7eff74106a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.128+0000 7eff7b51d640 1 --2- 172.21.15.32:0/589547327 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff74106630 0x7eff74106a10 secure :-1 s=READY pgs=496 cs=0 l=1 rev1=1 crypto rx=0x7eff6c009930 tx=0x7eff6c02a420 comp rx=0 tx=0).stop 2024-04-25T00:33:23.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.129+0000 7eff7b51d640 1 -- 172.21.15.32:0/589547327 shutdown_connections 2024-04-25T00:33:23.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.129+0000 7eff7b51d640 1 --2- 172.21.15.32:0/589547327 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7eff74100fe0 0x7eff7410d4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.129+0000 7eff7b51d640 1 --2- 172.21.15.32:0/589547327 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7eff74100620 0x7eff74100aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.129+0000 7eff7b51d640 1 --2- 172.21.15.32:0/589547327 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff74106630 0x7eff74106a10 unknown :-1 s=CLOSED pgs=496 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.129+0000 7eff7b51d640 1 -- 172.21.15.32:0/589547327 >> 172.21.15.32:0/589547327 conn(0x7eff74077740 msgr2=0x7eff74077b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:23.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.129+0000 7eff7b51d640 1 -- 172.21.15.32:0/589547327 shutdown_connections 2024-04-25T00:33:23.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.129+0000 7eff7b51d640 1 -- 172.21.15.32:0/589547327 wait complete. 2024-04-25T00:33:23.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.129+0000 7eff7b51d640 1 Processor -- start 2024-04-25T00:33:23.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.129+0000 7eff7b51d640 1 -- start start 2024-04-25T00:33:23.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.129+0000 7eff7b51d640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7eff74100620 0x7eff74198080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.129+0000 7eff7b51d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff74100fe0 0x7eff741985c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.129+0000 7eff7b51d640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7eff74106630 0x7eff7419c940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.129+0000 7eff7b51d640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7eff74111b40 con 0x7eff74100fe0 2024-04-25T00:33:23.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.129+0000 7eff7b51d640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7eff741119c0 con 0x7eff74100620 2024-04-25T00:33:23.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.129+0000 7eff7b51d640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7eff74111cc0 con 0x7eff74106630 2024-04-25T00:33:23.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.130+0000 7eff78ab8640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff74100fe0 0x7eff741985c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.130+0000 7eff78ab8640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff74100fe0 0x7eff741985c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:57884/0 (socket says 172.21.15.32:57884) 2024-04-25T00:33:23.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.130+0000 7eff78ab8640 1 -- 172.21.15.32:0/3554938134 learned_addr learned my addr 172.21.15.32:0/3554938134 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:23.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.130+0000 7eff792b9640 1 --2- 172.21.15.32:0/3554938134 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7eff74100620 0x7eff74198080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.130+0000 7eff79aba640 1 --2- 172.21.15.32:0/3554938134 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7eff74106630 0x7eff7419c940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.130+0000 7eff78ab8640 1 -- 172.21.15.32:0/3554938134 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7eff74106630 msgr2=0x7eff7419c940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.130+0000 7eff78ab8640 1 --2- 172.21.15.32:0/3554938134 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7eff74106630 0x7eff7419c940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.130+0000 7eff78ab8640 1 -- 172.21.15.32:0/3554938134 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7eff74100620 msgr2=0x7eff74198080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.130+0000 7eff78ab8640 1 --2- 172.21.15.32:0/3554938134 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7eff74100620 0x7eff74198080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.130+0000 7eff78ab8640 1 -- 172.21.15.32:0/3554938134 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7eff7419d070 con 0x7eff74100fe0 2024-04-25T00:33:23.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.130+0000 7eff78ab8640 1 --2- 172.21.15.32:0/3554938134 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff74100fe0 0x7eff741985c0 secure :-1 s=READY pgs=497 cs=0 l=1 rev1=1 crypto rx=0x7eff6400c970 tx=0x7eff6400ce30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:23.135 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.130+0000 7eff6a7fc640 1 -- 172.21.15.32:0/3554938134 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7eff64007bb0 con 0x7eff74100fe0 2024-04-25T00:33:23.135 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.130+0000 7eff7b51d640 1 -- 172.21.15.32:0/3554938134 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7eff7419d360 con 0x7eff74100fe0 2024-04-25T00:33:23.135 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.130+0000 7eff6a7fc640 1 -- 172.21.15.32:0/3554938134 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7eff64007d50 con 0x7eff74100fe0 2024-04-25T00:33:23.135 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.130+0000 7eff6a7fc640 1 -- 172.21.15.32:0/3554938134 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7eff6400f450 con 0x7eff74100fe0 2024-04-25T00:33:23.135 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.130+0000 7eff7b51d640 1 -- 172.21.15.32:0/3554938134 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7eff741a4bf0 con 0x7eff74100fe0 2024-04-25T00:33:23.136 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.131+0000 7eff7b51d640 1 -- 172.21.15.32:0/3554938134 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7eff74068560 con 0x7eff74100fe0 2024-04-25T00:33:23.136 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.132+0000 7eff6a7fc640 1 -- 172.21.15.32:0/3554938134 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 24) ==== 138009+0+0 (secure 0 0 0) 0x7eff64005020 con 0x7eff74100fe0 2024-04-25T00:33:23.136 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.133+0000 7eff6a7fc640 1 --2- 172.21.15.32:0/3554938134 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7eff600a2d90 0x7eff600a5250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.136 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.133+0000 7eff792b9640 1 --2- 172.21.15.32:0/3554938134 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7eff600a2d90 0x7eff600a5250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.136 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.133+0000 7eff6a7fc640 1 -- 172.21.15.32:0/3554938134 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7eff640cf6e0 con 0x7eff74100fe0 2024-04-25T00:33:23.136 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.133+0000 7eff792b9640 1 --2- 172.21.15.32:0/3554938134 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7eff600a2d90 0x7eff600a5250 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7eff6c009900 tx=0x7eff6c0058d0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:23.139 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.137+0000 7eff6a7fc640 1 -- 172.21.15.32:0/3554938134 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7eff64014030 con 0x7eff74100fe0 2024-04-25T00:33:23.362 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.360+0000 7eff7b51d640 1 -- 172.21.15.32:0/3554938134 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7eff7402d170 con 0x7eff74100fe0 2024-04-25T00:33:23.362 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.361+0000 7eff6a7fc640 1 -- 172.21.15.32:0/3554938134 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 25 v25) ==== 94+0+11876 (secure 0 0 0) 0x7eff6409b790 con 0x7eff74100fe0 2024-04-25T00:33:23.363 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:33:23.363 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":25,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14643,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6839/187492674","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6838","nonce":187492674},{"type":"v1","addr":"172.21.15.32:6839","nonce":187492674}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14655,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6841/1948785545","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6840","nonce":1948785545},{"type":"v1","addr":"172.21.15.32:6841","nonce":1948785545}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":25,"flags":306,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":true},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-25T00:30:47.866875+0000","modified":"2024-04-25T00:33:16.420908+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24506,"mds_1":24481,"mds_2":24476},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24443":{"gid":24443,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24445":{"gid":24445,"name":"f","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24457":{"gid":24457,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24476":{"gid":24476,"name":"h","rank":2,"incarnation":22,"state":"up:active","state_seq":12,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24481":{"gid":24481,"name":"i","rank":1,"incarnation":20,"state":"up:active","state_seq":11,"addr":"172.21.15.130:6839/4267125809","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":4267125809},{"type":"v1","addr":"172.21.15.130:6839","nonce":4267125809}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24506":{"gid":24506,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.121:6839/258444853","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6838","nonce":258444853},{"type":"v1","addr":"172.21.15.121:6839","nonce":258444853}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24506,"qdb_cluster":[24506,24476,24481]},"id":1}]} 2024-04-25T00:33:23.363 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 25 2024-04-25T00:33:23.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.363+0000 7eff7b51d640 1 -- 172.21.15.32:0/3554938134 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7eff600a2d90 msgr2=0x7eff600a5250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.363+0000 7eff7b51d640 1 --2- 172.21.15.32:0/3554938134 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7eff600a2d90 0x7eff600a5250 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7eff6c009900 tx=0x7eff6c0058d0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.363+0000 7eff7b51d640 1 -- 172.21.15.32:0/3554938134 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff74100fe0 msgr2=0x7eff741985c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.363+0000 7eff7b51d640 1 --2- 172.21.15.32:0/3554938134 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff74100fe0 0x7eff741985c0 secure :-1 s=READY pgs=497 cs=0 l=1 rev1=1 crypto rx=0x7eff6400c970 tx=0x7eff6400ce30 comp rx=0 tx=0).stop 2024-04-25T00:33:23.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.363+0000 7eff7b51d640 1 -- 172.21.15.32:0/3554938134 shutdown_connections 2024-04-25T00:33:23.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.363+0000 7eff7b51d640 1 --2- 172.21.15.32:0/3554938134 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7eff600a2d90 0x7eff600a5250 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.363+0000 7eff7b51d640 1 --2- 172.21.15.32:0/3554938134 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7eff74106630 0x7eff7419c940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.363+0000 7eff7b51d640 1 --2- 172.21.15.32:0/3554938134 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7eff74100fe0 0x7eff741985c0 unknown :-1 s=CLOSED pgs=497 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.363+0000 7eff7b51d640 1 --2- 172.21.15.32:0/3554938134 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7eff74100620 0x7eff74198080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.363+0000 7eff7b51d640 1 -- 172.21.15.32:0/3554938134 >> 172.21.15.32:0/3554938134 conn(0x7eff74077740 msgr2=0x7eff74102ef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:23.366 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.363+0000 7eff7b51d640 1 -- 172.21.15.32:0/3554938134 shutdown_connections 2024-04-25T00:33:23.366 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.363+0000 7eff7b51d640 1 -- 172.21.15.32:0/3554938134 wait complete. 2024-04-25T00:33:23.376 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-25T00:33:23.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.477+0000 7fea255e7640 1 -- 172.21.15.32:0/2505155591 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fea201032f0 msgr2=0x7fea2010b7d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.477+0000 7fea255e7640 1 --2- 172.21.15.32:0/2505155591 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fea201032f0 0x7fea2010b7d0 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7fea14009900 tx=0x7fea1402a510 comp rx=0 tx=0).stop 2024-04-25T00:33:23.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.477+0000 7fea255e7640 1 -- 172.21.15.32:0/2505155591 shutdown_connections 2024-04-25T00:33:23.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.477+0000 7fea255e7640 1 --2- 172.21.15.32:0/2505155591 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea2010bd10 0x7fea2010e1b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.477+0000 7fea255e7640 1 --2- 172.21.15.32:0/2505155591 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fea201032f0 0x7fea2010b7d0 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.477+0000 7fea255e7640 1 --2- 172.21.15.32:0/2505155591 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea201029d0 0x7fea20102db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.477+0000 7fea255e7640 1 -- 172.21.15.32:0/2505155591 >> 172.21.15.32:0/2505155591 conn(0x7fea20077740 msgr2=0x7fea20077b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:23.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.477+0000 7fea255e7640 1 -- 172.21.15.32:0/2505155591 shutdown_connections 2024-04-25T00:33:23.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.477+0000 7fea255e7640 1 -- 172.21.15.32:0/2505155591 wait complete. 2024-04-25T00:33:23.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.477+0000 7fea255e7640 1 Processor -- start 2024-04-25T00:33:23.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.478+0000 7fea255e7640 1 -- start start 2024-04-25T00:33:23.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.478+0000 7fea255e7640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fea201029d0 0x7fea20198090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.478+0000 7fea255e7640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea201032f0 0x7fea201985d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.478+0000 7fea255e7640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea2010bd10 0x7fea2019c900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.478+0000 7fea255e7640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fea20111b10 con 0x7fea2010bd10 2024-04-25T00:33:23.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.478+0000 7fea255e7640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fea20111990 con 0x7fea201029d0 2024-04-25T00:33:23.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.478+0000 7fea255e7640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fea20111c90 con 0x7fea201032f0 2024-04-25T00:33:23.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.478+0000 7fea1f7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea2010bd10 0x7fea2019c900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.478+0000 7fea1f7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea2010bd10 0x7fea2019c900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:57890/0 (socket says 172.21.15.32:57890) 2024-04-25T00:33:23.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.478+0000 7fea1f7fe640 1 -- 172.21.15.32:0/1102113382 learned_addr learned my addr 172.21.15.32:0/1102113382 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:23.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.478+0000 7fea1effd640 1 --2- 172.21.15.32:0/1102113382 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fea201029d0 0x7fea20198090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.478+0000 7fea1e7fc640 1 --2- 172.21.15.32:0/1102113382 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea201032f0 0x7fea201985d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.478+0000 7fea1f7fe640 1 -- 172.21.15.32:0/1102113382 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea201032f0 msgr2=0x7fea201985d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.478+0000 7fea1f7fe640 1 --2- 172.21.15.32:0/1102113382 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea201032f0 0x7fea201985d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.478+0000 7fea1f7fe640 1 -- 172.21.15.32:0/1102113382 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fea201029d0 msgr2=0x7fea20198090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.478+0000 7fea1f7fe640 1 --2- 172.21.15.32:0/1102113382 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fea201029d0 0x7fea20198090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.478+0000 7fea1f7fe640 1 -- 172.21.15.32:0/1102113382 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fea2019d030 con 0x7fea2010bd10 2024-04-25T00:33:23.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.478+0000 7fea1e7fc640 1 --2- 172.21.15.32:0/1102113382 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea201032f0 0x7fea201985d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:23.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.479+0000 7fea1f7fe640 1 --2- 172.21.15.32:0/1102113382 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea2010bd10 0x7fea2019c900 secure :-1 s=READY pgs=498 cs=0 l=1 rev1=1 crypto rx=0x7fea1800e9b0 tx=0x7fea1800ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:23.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.479+0000 7fea07fff640 1 -- 172.21.15.32:0/1102113382 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fea1800cd80 con 0x7fea2010bd10 2024-04-25T00:33:23.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.479+0000 7fea255e7640 1 -- 172.21.15.32:0/1102113382 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fea2019d320 con 0x7fea2010bd10 2024-04-25T00:33:23.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.479+0000 7fea07fff640 1 -- 172.21.15.32:0/1102113382 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fea180035f0 con 0x7fea2010bd10 2024-04-25T00:33:23.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.479+0000 7fea07fff640 1 -- 172.21.15.32:0/1102113382 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fea18010670 con 0x7fea2010bd10 2024-04-25T00:33:23.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.479+0000 7fea255e7640 1 -- 172.21.15.32:0/1102113382 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fea201a4bb0 con 0x7fea2010bd10 2024-04-25T00:33:23.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.480+0000 7fea255e7640 1 -- 172.21.15.32:0/1102113382 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fea20068560 con 0x7fea2010bd10 2024-04-25T00:33:23.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.481+0000 7fea07fff640 1 -- 172.21.15.32:0/1102113382 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 24) ==== 138009+0+0 (secure 0 0 0) 0x7fea180037a0 con 0x7fea2010bd10 2024-04-25T00:33:23.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.482+0000 7fea07fff640 1 --2- 172.21.15.32:0/1102113382 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fea080a2d90 0x7fea080a5250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.482+0000 7fea1effd640 1 --2- 172.21.15.32:0/1102113382 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fea080a2d90 0x7fea080a5250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.482+0000 7fea07fff640 1 -- 172.21.15.32:0/1102113382 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fea180cfd30 con 0x7fea2010bd10 2024-04-25T00:33:23.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.482+0000 7fea1effd640 1 --2- 172.21.15.32:0/1102113382 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fea080a2d90 0x7fea080a5250 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fea10005fd0 tx=0x7fea10009500 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:23.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.486+0000 7fea07fff640 1 -- 172.21.15.32:0/1102113382 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fea1809c3d0 con 0x7fea2010bd10 2024-04-25T00:33:23.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.674+0000 7fea255e7640 1 -- 172.21.15.32:0/1102113382 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fea20106900 con 0x7fea2010bd10 2024-04-25T00:33:23.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.675+0000 7fea07fff640 1 -- 172.21.15.32:0/1102113382 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v94) ==== 74+0+25219 (secure 0 0 0) 0x7fea1809bcd0 con 0x7fea2010bd10 2024-04-25T00:33:23.678 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:33:23.678 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":94,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:33:17.919932+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-25T00:30:46.239556+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-25T00:30:47.257182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2024-04-25T00:30:51.193975+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.b","mappings":[{"from":8,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":1}]},{"pgid":"2.f","mappings":[{"from":8,"to":11}]},{"pgid":"2.10","mappings":[{"from":2,"to":4}]},{"pgid":"3.6","mappings":[{"from":3,"to":6}]},{"pgid":"3.17","mappings":[{"from":4,"to":7}]},{"pgid":"3.1b","mappings":[{"from":5,"to":9}]},{"pgid":"3.20","mappings":[{"from":5,"to":9}]},{"pgid":"3.23","mappings":[{"from":4,"to":8}]},{"pgid":"3.2c","mappings":[{"from":4,"to":9}]},{"pgid":"4.c","mappings":[{"from":9,"to":10}]},{"pgid":"4.12","mappings":[{"from":9,"to":10}]},{"pgid":"4.1c","mappings":[{"from":9,"to":8}]},{"pgid":"4.1d","mappings":[{"from":5,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:6801/585810931":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/1096527590":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/1767429535":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/3915353181":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6800/585810931":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/1694990206":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-num-failure-domains":"0","crush-osds-per-failure-domain":"0","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:33:23.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.677+0000 7fea255e7640 1 -- 172.21.15.32:0/1102113382 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fea080a2d90 msgr2=0x7fea080a5250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.677+0000 7fea255e7640 1 --2- 172.21.15.32:0/1102113382 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fea080a2d90 0x7fea080a5250 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fea10005fd0 tx=0x7fea10009500 comp rx=0 tx=0).stop 2024-04-25T00:33:23.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.677+0000 7fea255e7640 1 -- 172.21.15.32:0/1102113382 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea2010bd10 msgr2=0x7fea2019c900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.677+0000 7fea255e7640 1 --2- 172.21.15.32:0/1102113382 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea2010bd10 0x7fea2019c900 secure :-1 s=READY pgs=498 cs=0 l=1 rev1=1 crypto rx=0x7fea1800e9b0 tx=0x7fea1800ee70 comp rx=0 tx=0).stop 2024-04-25T00:33:23.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.677+0000 7fea255e7640 1 -- 172.21.15.32:0/1102113382 shutdown_connections 2024-04-25T00:33:23.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.677+0000 7fea255e7640 1 --2- 172.21.15.32:0/1102113382 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fea080a2d90 0x7fea080a5250 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.677+0000 7fea255e7640 1 --2- 172.21.15.32:0/1102113382 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fea2010bd10 0x7fea2019c900 unknown :-1 s=CLOSED pgs=498 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.677+0000 7fea255e7640 1 --2- 172.21.15.32:0/1102113382 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea201032f0 0x7fea201985d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.677+0000 7fea255e7640 1 --2- 172.21.15.32:0/1102113382 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fea201029d0 0x7fea20198090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.677+0000 7fea255e7640 1 -- 172.21.15.32:0/1102113382 >> 172.21.15.32:0/1102113382 conn(0x7fea20077740 msgr2=0x7fea200fd7b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:23.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.678+0000 7fea255e7640 1 -- 172.21.15.32:0/1102113382 shutdown_connections 2024-04-25T00:33:23.682 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.678+0000 7fea255e7640 1 -- 172.21.15.32:0/1102113382 wait complete. 2024-04-25T00:33:23.689 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: Updating smithi032:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:33:23.690 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: Updating smithi121:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:33:23.690 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: Updating smithi130:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:33:23.690 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: pgmap v5: 129 pgs: 129 active+clean; 1.1 MiB data, 373 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:33:23.690 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-25T00:33:23.690 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-04-25T00:33:23.691 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: Updating smithi032:/etc/ceph/ceph.client.admin.keyring 2024-04-25T00:33:23.691 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: Updating smithi130:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T00:33:23.691 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: Updating smithi121:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T00:33:23.691 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: Updating smithi032:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T00:33:23.691 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: mgrmap e24: x(active, since 4s), standbys: z, y 2024-04-25T00:33:23.691 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:23.692 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:23.692 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:23.692 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:23.692 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:23.692 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:23.692 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:23.692 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3856611077' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:33:23.693 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:33:23.693 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:33:23.693 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:33:23.693 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:23.693 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/538135899' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:33:23.693 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:23 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3554938134' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:33:23.696 DEBUG:tasks.fwd_scrub:Yielding 2024-04-25T00:33:23.696 INFO:teuthology.run_tasks:Running task exec... 2024-04-25T00:33:23.709 INFO:teuthology.task.exec:Executing custom commands... 2024-04-25T00:33:23.709 INFO:teuthology.task.exec:Running commands on role mon.a host ubuntu@smithi032.front.sepia.ceph.com 2024-04-25T00:33:23.709 DEBUG:teuthology.orchestra.run.smithi032:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule add --fs=cephfs --path=/ --snap_schedule=1m' 2024-04-25T00:33:23.710 INFO:tasks.fwd_scrub.fs.[cephfs]:start scrubbing fs: cephfs 2024-04-25T00:33:23.710 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-25T00:33:23.710 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:33:23.710 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-25T00:33:23.817 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.815+0000 7f413ddee640 1 -- 172.21.15.32:0/3558421331 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4138107720 msgr2=0x7f4138107b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.817 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.815+0000 7f413ddee640 1 --2- 172.21.15.32:0/3558421331 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4138107720 0x7f4138107b00 secure :-1 s=READY pgs=499 cs=0 l=1 rev1=1 crypto rx=0x7f4128009930 tx=0x7f412802a420 comp rx=0 tx=0).stop 2024-04-25T00:33:23.817 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.815+0000 7f413ddee640 1 -- 172.21.15.32:0/3558421331 shutdown_connections 2024-04-25T00:33:23.817 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.815+0000 7f413ddee640 1 --2- 172.21.15.32:0/3558421331 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f41381020d0 0x7f413810e5e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.817 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.815+0000 7f413ddee640 1 --2- 172.21.15.32:0/3558421331 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4138101710 0x7f4138101b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.817 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.815+0000 7f413ddee640 1 --2- 172.21.15.32:0/3558421331 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4138107720 0x7f4138107b00 unknown :-1 s=CLOSED pgs=499 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.817 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.815+0000 7f90a06e1640 1 -- 172.21.15.32:0/656991577 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9098101030 msgr2=0x7f909810d4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.818 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.815+0000 7f413ddee640 1 -- 172.21.15.32:0/3558421331 >> 172.21.15.32:0/3558421331 conn(0x7f4138078b90 msgr2=0x7f4138078fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:23.818 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.815+0000 7f90a06e1640 1 --2- 172.21.15.32:0/656991577 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9098101030 0x7f909810d4f0 secure :-1 s=READY pgs=500 cs=0 l=1 rev1=1 crypto rx=0x7f9090009cd0 tx=0x7f909002a8e0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.819 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.815+0000 7f413ddee640 1 -- 172.21.15.32:0/3558421331 shutdown_connections 2024-04-25T00:33:23.819 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.815+0000 7f413ddee640 1 -- 172.21.15.32:0/3558421331 wait complete. 2024-04-25T00:33:23.819 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.815+0000 7f90a06e1640 1 -- 172.21.15.32:0/656991577 shutdown_connections 2024-04-25T00:33:23.819 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.815+0000 7f90a06e1640 1 --2- 172.21.15.32:0/656991577 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9098101030 0x7f909810d4f0 unknown :-1 s=CLOSED pgs=500 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.819 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.815+0000 7f90a06e1640 1 --2- 172.21.15.32:0/656991577 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9098100670 0x7f9098100af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.819 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.815+0000 7f90a06e1640 1 --2- 172.21.15.32:0/656991577 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9098106630 0x7f9098106a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.820 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.815+0000 7f90a06e1640 1 -- 172.21.15.32:0/656991577 >> 172.21.15.32:0/656991577 conn(0x7f90980777c0 msgr2=0x7f9098077bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:23.820 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.815+0000 7f90a06e1640 1 -- 172.21.15.32:0/656991577 shutdown_connections 2024-04-25T00:33:23.820 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.815+0000 7f90a06e1640 1 -- 172.21.15.32:0/656991577 wait complete. 2024-04-25T00:33:23.820 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f413ddee640 1 Processor -- start 2024-04-25T00:33:23.820 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f90a06e1640 1 Processor -- start 2024-04-25T00:33:23.820 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f90a06e1640 1 -- start start 2024-04-25T00:33:23.821 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f413ddee640 1 -- start start 2024-04-25T00:33:23.821 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f90a06e1640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9098100670 0x7f9098198010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f90a06e1640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9098101030 0x7f9098198550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f90a06e1640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9098106630 0x7f909819c8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f90a06e1640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f9098110a40 con 0x7f9098106630 2024-04-25T00:33:23.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f90a06e1640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f90981108c0 con 0x7f9098101030 2024-04-25T00:33:23.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f90a06e1640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f9098110bc0 con 0x7f9098100670 2024-04-25T00:33:23.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f413ddee640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4138101710 0x7f4138073a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f413ddee640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f41381020d0 0x7f4138073f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f413ddee640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4138107720 0x7f41380744c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f413ddee640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f4138111af0 con 0x7f4138101710 2024-04-25T00:33:23.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f413ddee640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f4138111970 con 0x7f4138107720 2024-04-25T00:33:23.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f413ddee640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f4138111c70 con 0x7f41381020d0 2024-04-25T00:33:23.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f909ec7e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9098106630 0x7f909819c8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f41377fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4138101710 0x7f4138073a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f909ec7e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9098106630 0x7f909819c8d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:57920/0 (socket says 172.21.15.32:57920) 2024-04-25T00:33:23.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f909ec7e640 1 -- 172.21.15.32:0/2477513410 learned_addr learned my addr 172.21.15.32:0/2477513410 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:23.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f41377fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4138101710 0x7f4138073a40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:57914/0 (socket says 172.21.15.32:57914) 2024-04-25T00:33:23.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f41377fe640 1 -- 172.21.15.32:0/582164350 learned_addr learned my addr 172.21.15.32:0/582164350 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:23.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f909dc7c640 1 --2- 172.21.15.32:0/2477513410 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9098101030 0x7f9098198550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f4137fff640 1 --2- 172.21.15.32:0/582164350 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4138107720 0x7f41380744c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.825 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f909e47d640 1 --2- 172.21.15.32:0/2477513410 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9098100670 0x7f9098198010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.816+0000 7f4136ffd640 1 --2- 172.21.15.32:0/582164350 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f41381020d0 0x7f4138073f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.817+0000 7f41377fe640 1 -- 172.21.15.32:0/582164350 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f41381020d0 msgr2=0x7f4138073f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.817+0000 7f41377fe640 1 --2- 172.21.15.32:0/582164350 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f41381020d0 0x7f4138073f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.817+0000 7f41377fe640 1 -- 172.21.15.32:0/582164350 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4138107720 msgr2=0x7f41380744c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.817+0000 7f41377fe640 1 --2- 172.21.15.32:0/582164350 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4138107720 0x7f41380744c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.817+0000 7f41377fe640 1 -- 172.21.15.32:0/582164350 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f41381a6510 con 0x7f4138101710 2024-04-25T00:33:23.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.817+0000 7f909ec7e640 1 -- 172.21.15.32:0/2477513410 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9098100670 msgr2=0x7f9098198010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.817+0000 7f909ec7e640 1 --2- 172.21.15.32:0/2477513410 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9098100670 0x7f9098198010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.817+0000 7f909ec7e640 1 -- 172.21.15.32:0/2477513410 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9098101030 msgr2=0x7f9098198550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.817+0000 7f909ec7e640 1 --2- 172.21.15.32:0/2477513410 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9098101030 0x7f9098198550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.817+0000 7f909ec7e640 1 -- 172.21.15.32:0/2477513410 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f909819d000 con 0x7f9098106630 2024-04-25T00:33:23.827 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.817+0000 7f909dc7c640 1 --2- 172.21.15.32:0/2477513410 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9098101030 0x7f9098198550 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:23.828 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.817+0000 7f41377fe640 1 --2- 172.21.15.32:0/582164350 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4138101710 0x7f4138073a40 secure :-1 s=READY pgs=501 cs=0 l=1 rev1=1 crypto rx=0x7f4128009230 tx=0x7f4128004040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:23.828 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.817+0000 7f909ec7e640 1 --2- 172.21.15.32:0/2477513410 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9098106630 0x7f909819c8d0 secure :-1 s=READY pgs=502 cs=0 l=1 rev1=1 crypto rx=0x7f9090005cc0 tx=0x7f90900093d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:23.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.817+0000 7f4134ff9640 1 -- 172.21.15.32:0/582164350 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f412803f030 con 0x7f4138101710 2024-04-25T00:33:23.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.817+0000 7f413ddee640 1 -- 172.21.15.32:0/582164350 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f41381a67a0 con 0x7f4138101710 2024-04-25T00:33:23.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.817+0000 7f413ddee640 1 -- 172.21.15.32:0/582164350 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f41381a6ce0 con 0x7f4138101710 2024-04-25T00:33:23.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.817+0000 7f4134ff9640 1 -- 172.21.15.32:0/582164350 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4128004440 con 0x7f4138101710 2024-04-25T00:33:23.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.817+0000 7f908f7fe640 1 -- 172.21.15.32:0/2477513410 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9090003e00 con 0x7f9098106630 2024-04-25T00:33:23.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.817+0000 7f908f7fe640 1 -- 172.21.15.32:0/2477513410 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9090004520 con 0x7f9098106630 2024-04-25T00:33:23.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.817+0000 7f908f7fe640 1 -- 172.21.15.32:0/2477513410 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9090030db0 con 0x7f9098106630 2024-04-25T00:33:23.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.817+0000 7f90a06e1640 1 -- 172.21.15.32:0/2477513410 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f909819d290 con 0x7f9098106630 2024-04-25T00:33:23.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.817+0000 7f4134ff9640 1 -- 172.21.15.32:0/582164350 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4128035070 con 0x7f4138101710 2024-04-25T00:33:23.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.817+0000 7f90a06e1640 1 -- 172.21.15.32:0/2477513410 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f90981a4b80 con 0x7f9098106630 2024-04-25T00:33:23.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.819+0000 7f90a06e1640 1 -- 172.21.15.32:0/2477513410 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f909804f500 con 0x7f9098106630 2024-04-25T00:33:23.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.820+0000 7f4134ff9640 1 -- 172.21.15.32:0/582164350 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 24) ==== 138009+0+0 (secure 0 0 0) 0x7f4128042030 con 0x7f4138101710 2024-04-25T00:33:23.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.820+0000 7f4140bda740 1 Processor -- start 2024-04-25T00:33:23.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.820+0000 7f4140bda740 1 -- start start 2024-04-25T00:33:23.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.820+0000 7f908f7fe640 1 -- 172.21.15.32:0/2477513410 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 24) ==== 138009+0+0 (secure 0 0 0) 0x7f90900046d0 con 0x7f9098106630 2024-04-25T00:33:23.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.820+0000 7f4140bda740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557db2d97980 0x557db2d95cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.820+0000 7f4140bda740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557db2d96290 0x557db2d96710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.820+0000 7f4140bda740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2da1010 0x557db2da3400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.820+0000 7f4140bda740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x557db2cfa2c0 con 0x557db2da1010 2024-04-25T00:33:23.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.820+0000 7f4140bda740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x557db2ca2910 con 0x557db2d96290 2024-04-25T00:33:23.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.820+0000 7f4140bda740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x557db2cf6280 con 0x557db2d97980 2024-04-25T00:33:23.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.821+0000 7f4134ff9640 1 --2- 172.21.15.32:0/582164350 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f41200a2e10 0x7f41200a52d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.821+0000 7f4136ffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2da1010 0x557db2da3400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.821+0000 7f4136ffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2da1010 0x557db2da3400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:57928/0 (socket says 172.21.15.32:57928) 2024-04-25T00:33:23.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.821+0000 7f4136ffd640 1 -- 172.21.15.32:0/1019259512 learned_addr learned my addr 172.21.15.32:0/1019259512 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:23.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.821+0000 7f4137fff640 1 --2- 172.21.15.32:0/1019259512 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557db2d97980 0x557db2d95cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.821+0000 7f4137fff640 1 --2- 172.21.15.32:0/582164350 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f41200a2e10 0x7f41200a52d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.821+0000 7f4134ff9640 1 -- 172.21.15.32:0/582164350 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f41280efaf0 con 0x7f4138101710 2024-04-25T00:33:23.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.821+0000 7f41377fe640 1 --2- 172.21.15.32:0/1019259512 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557db2d96290 0x557db2d96710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.821+0000 7f4136ffd640 1 -- 172.21.15.32:0/1019259512 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557db2d97980 msgr2=0x557db2d95cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:23.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.821+0000 7f4136ffd640 1 --2- 172.21.15.32:0/1019259512 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557db2d97980 0x557db2d95cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.821+0000 7f4136ffd640 1 -- 172.21.15.32:0/1019259512 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557db2d96290 msgr2=0x557db2d96710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:23.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.821+0000 7f4136ffd640 1 --2- 172.21.15.32:0/1019259512 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557db2d96290 0x557db2d96710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.821+0000 7f908f7fe640 1 --2- 172.21.15.32:0/2477513410 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f90840a2ec0 0x7f90840a5380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.821+0000 7f4136ffd640 1 -- 172.21.15.32:0/1019259512 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557db2cb3fb0 con 0x557db2da1010 2024-04-25T00:33:23.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.821+0000 7f4137fff640 1 --2- 172.21.15.32:0/1019259512 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557db2d97980 0x557db2d95cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:23.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.821+0000 7f908f7fe640 1 -- 172.21.15.32:0/2477513410 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f909002e030 con 0x7f9098106630 2024-04-25T00:33:23.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.821+0000 7f4136ffd640 1 --2- 172.21.15.32:0/1019259512 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2da1010 0x557db2da3400 secure :-1 s=READY pgs=503 cs=0 l=1 rev1=1 crypto rx=0x7f412c009c90 tx=0x7f412c011810 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d960a92f4c87a5b2 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:23.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.821+0000 7f909e47d640 1 --2- 172.21.15.32:0/2477513410 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f90840a2ec0 0x7f90840a5380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.822+0000 7f4137fff640 1 --2- 172.21.15.32:0/582164350 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f41200a2e10 0x7f41200a52d0 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f4130005fd0 tx=0x7f4130009500 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:23.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.822+0000 7f909e47d640 1 --2- 172.21.15.32:0/2477513410 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f90840a2ec0 0x7f90840a5380 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f9094009b50 tx=0x7f9094005f90 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:23.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.825+0000 7f411dffb640 1 -- 172.21.15.32:0/1019259512 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f412c01e070 con 0x557db2da1010 2024-04-25T00:33:23.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.825+0000 7f411dffb640 1 -- 172.21.15.32:0/1019259512 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f412c019b10 con 0x557db2da1010 2024-04-25T00:33:23.837 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.825+0000 7f411dffb640 1 -- 172.21.15.32:0/1019259512 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f412c00eac0 con 0x557db2da1010 2024-04-25T00:33:23.837 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.825+0000 7f908f7fe640 1 -- 172.21.15.32:0/2477513410 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f909004b060 con 0x7f9098106630 2024-04-25T00:33:23.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: Updating smithi032:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:33:23.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: Updating smithi121:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:33:23.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: Updating smithi130:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:33:23.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: pgmap v5: 129 pgs: 129 active+clean; 1.1 MiB data, 373 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:33:23.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-25T00:33:23.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-04-25T00:33:23.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: Updating smithi032:/etc/ceph/ceph.client.admin.keyring 2024-04-25T00:33:23.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: Updating smithi130:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T00:33:23.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: Updating smithi121:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T00:33:23.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: Updating smithi032:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T00:33:23.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: mgrmap e24: x(active, since 4s), standbys: z, y 2024-04-25T00:33:23.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:23.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:23.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:23.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:23.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:23.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:23.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:23.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3856611077' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:33:23.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:33:23.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:33:23.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:33:23.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:23.840 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/538135899' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:33:23.840 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:23 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3554938134' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:33:23.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.826+0000 7f4140bda740 1 -- 172.21.15.32:0/1019259512 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2da1010 msgr2=0x557db2da3400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.826+0000 7f4140bda740 1 --2- 172.21.15.32:0/1019259512 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2da1010 0x557db2da3400 secure :-1 s=READY pgs=503 cs=0 l=1 rev1=1 crypto rx=0x7f412c009c90 tx=0x7f412c011810 comp rx=0 tx=0).stop 2024-04-25T00:33:23.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.826+0000 7f4140bda740 1 -- 172.21.15.32:0/1019259512 shutdown_connections 2024-04-25T00:33:23.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.826+0000 7f4140bda740 1 --2- 172.21.15.32:0/1019259512 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2da1010 0x557db2da3400 unknown :-1 s=CLOSED pgs=503 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.826+0000 7f4140bda740 1 --2- 172.21.15.32:0/1019259512 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557db2d96290 0x557db2d96710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.826+0000 7f4140bda740 1 --2- 172.21.15.32:0/1019259512 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557db2d97980 0x557db2d95cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.847 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.826+0000 7f4140bda740 1 -- 172.21.15.32:0/1019259512 >> 172.21.15.32:0/1019259512 conn(0x557db2d5e950 msgr2=0x557db2d927f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:23.847 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.826+0000 7f4140bda740 1 -- 172.21.15.32:0/1019259512 shutdown_connections 2024-04-25T00:33:23.847 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.826+0000 7f4140bda740 1 -- 172.21.15.32:0/1019259512 wait complete. 2024-04-25T00:33:23.847 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.827+0000 7f4140bda740 1 -- start start 2024-04-25T00:33:23.847 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.827+0000 7f4140bda740 1 Processor -- start 2024-04-25T00:33:23.847 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.827+0000 7f4140bda740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557db2d96290 0x557db2d9dee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.847 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.827+0000 7f4140bda740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2d97980 0x557db2d9e420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.847 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.827+0000 7f4140bda740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557db2da1010 0x557db2d9e960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.847 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.827+0000 7f4140bda740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x557db2da3ea0 con 0x557db2d97980 2024-04-25T00:33:23.847 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.827+0000 7f4140bda740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x557db2cb5e90 con 0x557db2da1010 2024-04-25T00:33:23.847 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.827+0000 7f4140bda740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x557db2da4020 con 0x557db2d96290 2024-04-25T00:33:23.847 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.827+0000 7f4140bda740 10 client.0 _ll_register_callbacks cb 0x557db2cf1970 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:33:23.847 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.827+0000 7f4136ffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2d97980 0x557db2d9e420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.827+0000 7f4136ffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2d97980 0x557db2d9e420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:57936/0 (socket says 172.21.15.32:57936) 2024-04-25T00:33:23.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.827+0000 7f4136ffd640 1 -- 172.21.15.32:0/3122230113 learned_addr learned my addr 172.21.15.32:0/3122230113 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:23.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.827+0000 7f41377fe640 1 --2- 172.21.15.32:0/3122230113 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557db2d96290 0x557db2d9dee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.827+0000 7f4137fff640 1 --2- 172.21.15.32:0/3122230113 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557db2da1010 0x557db2d9e960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.828+0000 7f4136ffd640 1 -- 172.21.15.32:0/3122230113 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557db2d96290 msgr2=0x557db2d9dee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:23.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.828+0000 7f4136ffd640 1 --2- 172.21.15.32:0/3122230113 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557db2d96290 0x557db2d9dee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.828+0000 7f4136ffd640 1 -- 172.21.15.32:0/3122230113 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557db2da1010 msgr2=0x557db2d9e960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:23.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.828+0000 7f4136ffd640 1 --2- 172.21.15.32:0/3122230113 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557db2da1010 0x557db2d9e960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.828+0000 7f4136ffd640 1 -- 172.21.15.32:0/3122230113 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557db2ad67e0 con 0x557db2d97980 2024-04-25T00:33:23.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.828+0000 7f4137fff640 1 --2- 172.21.15.32:0/3122230113 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557db2da1010 0x557db2d9e960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:33:23.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.828+0000 7f41377fe640 1 --2- 172.21.15.32:0/3122230113 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557db2d96290 0x557db2d9dee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:23.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.828+0000 7f4136ffd640 1 --2- 172.21.15.32:0/3122230113 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2d97980 0x557db2d9e420 secure :-1 s=READY pgs=504 cs=0 l=1 rev1=1 crypto rx=0x7f412c019a60 tx=0x7f412c019a90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=86eaf1b9399fac0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:23.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.828+0000 7f4116ffd640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:33:23.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.828+0000 7f4116ffd640 1 -- 172.21.15.32:0/3122230113 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f412c01e070 con 0x557db2d97980 2024-04-25T00:33:23.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.828+0000 7f4116ffd640 1 -- 172.21.15.32:0/3122230113 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f412c004230 con 0x557db2d97980 2024-04-25T00:33:23.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.828+0000 7f4140bda740 1 -- 172.21.15.32:0/3122230113 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f412c019b10 con 0x557db2d97980 2024-04-25T00:33:23.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.828+0000 7f4116ffd640 1 -- 172.21.15.32:0/3122230113 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f412c033050 con 0x557db2d97980 2024-04-25T00:33:23.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.828+0000 7f4140bda740 10 client.15693 fetch_fsmap learned FSMap version 25 2024-04-25T00:33:23.850 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.828+0000 7f4140bda740 1 -- 172.21.15.32:0/3122230113 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x557db2b36f70 con 0x557db2d97980 2024-04-25T00:33:23.850 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.828+0000 7f4116ffd640 1 -- 172.21.15.32:0/3122230113 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f412c019b10 con 0x557db2d97980 2024-04-25T00:33:23.850 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.828+0000 7f4116ffd640 1 -- 172.21.15.32:0/3122230113 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7f412c00e980 con 0x557db2d97980 2024-04-25T00:33:23.850 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.829+0000 7f4140bda740 10 client.15693 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:33:23.850 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.829+0000 7f4140bda740 10 client.15693 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:33:23.850 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.829+0000 7f4140bda740 20 client.15693 populate_metadata read hostname 'smithi032' 2024-04-25T00:33:23.850 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.829+0000 7f4140bda740 1 --2- 172.21.15.32:0/3122230113 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x557db2db1040 0x557db2db3430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.850 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.829+0000 7f4140bda740 4 client.15693 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:33:23.850 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.829+0000 7f4140bda740 1 -- 172.21.15.32:0/3122230113 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x557db2cce5a0 con 0x557db2db1040 2024-04-25T00:33:23.850 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.829+0000 7f41377fe640 1 --2- 172.21.15.32:0/3122230113 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x557db2db1040 0x557db2db3430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.850 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.829+0000 7f41377fe640 1 --2- 172.21.15.32:0/3122230113 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x557db2db1040 0x557db2db3430 crc :-1 s=READY pgs=40 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2bc776d5b9784a6b server_cookie=4318d2b51900d2be in_seq=0 out_seq=0 2024-04-25T00:33:23.850 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.829+0000 7f4116ffd640 10 client.15693 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:23.851 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.830+0000 7f4116ffd640 1 -- 172.21.15.32:0/3122230113 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x557db2cce5a0 con 0x557db2db1040 2024-04-25T00:33:23.851 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.830+0000 7f4116ffd640 10 client.15693 handle_command_reply: tid=0 2024-04-25T00:33:23.851 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.830+0000 7f4140bda740 1 client.15693 shutdown 2024-04-25T00:33:23.851 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.830+0000 7f4140bda740 1 -- 172.21.15.32:0/3122230113 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2d97980 msgr2=0x557db2d9e420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.851 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.830+0000 7f4140bda740 1 --2- 172.21.15.32:0/3122230113 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2d97980 0x557db2d9e420 secure :-1 s=READY pgs=504 cs=0 l=1 rev1=1 crypto rx=0x7f412c019a60 tx=0x7f412c019a90 comp rx=0 tx=0).stop 2024-04-25T00:33:23.851 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.830+0000 7f4140bda740 1 -- 172.21.15.32:0/3122230113 shutdown_connections 2024-04-25T00:33:23.851 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.830+0000 7f4140bda740 1 --2- 172.21.15.32:0/3122230113 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x557db2db1040 0x557db2db3430 crc :-1 s=READY pgs=40 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.851 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.830+0000 7f4140bda740 1 --2- 172.21.15.32:0/3122230113 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557db2da1010 0x557db2d9e960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.851 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.830+0000 7f4140bda740 1 --2- 172.21.15.32:0/3122230113 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2d97980 0x557db2d9e420 unknown :-1 s=CLOSED pgs=504 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.851 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.830+0000 7f4116ffd640 0 client.15693 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:23.851 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.830+0000 7f4140bda740 1 --2- 172.21.15.32:0/3122230113 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557db2d96290 0x557db2d9dee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.852 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.830+0000 7f4140bda740 1 -- 172.21.15.32:0/3122230113 >> 172.21.15.32:0/3122230113 conn(0x557db2d5e950 msgr2=0x557db2d985f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:23.852 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.830+0000 7f4140bda740 1 -- 172.21.15.32:0/3122230113 shutdown_connections 2024-04-25T00:33:23.852 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.830+0000 7f4140bda740 1 -- 172.21.15.32:0/3122230113 wait complete. 2024-04-25T00:33:23.852 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.830+0000 7f4140bda740 20 client.15693 trim_cache size 0 max 16384 2024-04-25T00:33:23.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.848+0000 7f4140bda740 1 Processor -- start 2024-04-25T00:33:23.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.848+0000 7f4140bda740 1 -- start start 2024-04-25T00:33:23.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.848+0000 7f4140bda740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557db2d96290 0x557db2dace00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.848+0000 7f4140bda740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557db2d97980 0x557db2d917d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.848+0000 7f4140bda740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2da1010 0x557db2db8c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.848+0000 7f4140bda740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f4124012820 con 0x557db2da1010 2024-04-25T00:33:23.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.848+0000 7f4140bda740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f41240100f0 con 0x557db2d97980 2024-04-25T00:33:23.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.848+0000 7f4140bda740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f412400df40 con 0x557db2d96290 2024-04-25T00:33:23.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.848+0000 7f4137fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2da1010 0x557db2db8c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.848+0000 7f4137fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2da1010 0x557db2db8c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:57946/0 (socket says 172.21.15.32:57946) 2024-04-25T00:33:23.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.848+0000 7f4137fff640 1 -- 172.21.15.32:0/4223602279 learned_addr learned my addr 172.21.15.32:0/4223602279 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:23.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.849+0000 7f41377fe640 1 --2- 172.21.15.32:0/4223602279 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557db2d96290 0x557db2dace00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.849+0000 7f4136ffd640 1 --2- 172.21.15.32:0/4223602279 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557db2d97980 0x557db2d917d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.849+0000 7f4137fff640 1 -- 172.21.15.32:0/4223602279 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557db2d96290 msgr2=0x557db2dace00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:23.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.849+0000 7f4137fff640 1 --2- 172.21.15.32:0/4223602279 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557db2d96290 0x557db2dace00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.849+0000 7f4137fff640 1 -- 172.21.15.32:0/4223602279 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557db2d97980 msgr2=0x557db2d917d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:23.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.849+0000 7f4137fff640 1 --2- 172.21.15.32:0/4223602279 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557db2d97980 0x557db2d917d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.849+0000 7f4137fff640 1 -- 172.21.15.32:0/4223602279 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557db2d9ce90 con 0x557db2da1010 2024-04-25T00:33:23.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.849+0000 7f41377fe640 1 --2- 172.21.15.32:0/4223602279 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557db2d96290 0x557db2dace00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:23.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.849+0000 7f4137fff640 1 --2- 172.21.15.32:0/4223602279 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2da1010 0x557db2db8c80 secure :-1 s=READY pgs=505 cs=0 l=1 rev1=1 crypto rx=0x7f41300108e0 tx=0x7f4130018000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e194d0805bbac6d server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:23.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.849+0000 7f411d7fa640 1 -- 172.21.15.32:0/4223602279 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f413001fb10 con 0x557db2da1010 2024-04-25T00:33:23.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.849+0000 7f411d7fa640 1 -- 172.21.15.32:0/4223602279 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f413001fcb0 con 0x557db2da1010 2024-04-25T00:33:23.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.849+0000 7f411d7fa640 1 -- 172.21.15.32:0/4223602279 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f413001d830 con 0x557db2da1010 2024-04-25T00:33:23.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.850+0000 7f4140bda740 1 -- 172.21.15.32:0/4223602279 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2da1010 msgr2=0x557db2db8c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.850+0000 7f4140bda740 1 --2- 172.21.15.32:0/4223602279 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2da1010 0x557db2db8c80 secure :-1 s=READY pgs=505 cs=0 l=1 rev1=1 crypto rx=0x7f41300108e0 tx=0x7f4130018000 comp rx=0 tx=0).stop 2024-04-25T00:33:23.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.850+0000 7f4140bda740 1 -- 172.21.15.32:0/4223602279 shutdown_connections 2024-04-25T00:33:23.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.850+0000 7f4140bda740 1 --2- 172.21.15.32:0/4223602279 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2da1010 0x557db2db8c80 unknown :-1 s=CLOSED pgs=505 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.850+0000 7f4140bda740 1 --2- 172.21.15.32:0/4223602279 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557db2d97980 0x557db2d917d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.850+0000 7f4140bda740 1 --2- 172.21.15.32:0/4223602279 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557db2d96290 0x557db2dace00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.850+0000 7f4140bda740 1 -- 172.21.15.32:0/4223602279 >> 172.21.15.32:0/4223602279 conn(0x557db2d5e950 msgr2=0x557db2d910c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:23.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.850+0000 7f4140bda740 1 -- 172.21.15.32:0/4223602279 shutdown_connections 2024-04-25T00:33:23.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.850+0000 7f4140bda740 1 -- 172.21.15.32:0/4223602279 wait complete. 2024-04-25T00:33:23.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.851+0000 7f4140bda740 1 -- start start 2024-04-25T00:33:23.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.851+0000 7f4140bda740 1 Processor -- start 2024-04-25T00:33:23.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.851+0000 7f4140bda740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2d96290 0x557db2da53b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.851+0000 7f4140bda740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557db2d97980 0x557db2da58f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.851+0000 7f4140bda740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557db2da1010 0x557db2e164f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.851+0000 7f4140bda740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f412400e5b0 con 0x557db2d96290 2024-04-25T00:33:23.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.851+0000 7f4140bda740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f412400fa20 con 0x557db2d97980 2024-04-25T00:33:23.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.851+0000 7f4140bda740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f412400ec80 con 0x557db2da1010 2024-04-25T00:33:23.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.851+0000 7f4140bda740 10 client.0 _ll_register_callbacks cb 0x7f412c01c070 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:33:23.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.851+0000 7f41377fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2d96290 0x557db2da53b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.851+0000 7f41377fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2d96290 0x557db2da53b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:57954/0 (socket says 172.21.15.32:57954) 2024-04-25T00:33:23.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.851+0000 7f41377fe640 1 -- 172.21.15.32:0/1627334057 learned_addr learned my addr 172.21.15.32:0/1627334057 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:23.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.852+0000 7f4136ffd640 1 --2- 172.21.15.32:0/1627334057 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557db2d97980 0x557db2da58f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.852+0000 7f41377fe640 1 -- 172.21.15.32:0/1627334057 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557db2da1010 msgr2=0x557db2e164f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:23.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.852+0000 7f41377fe640 1 --2- 172.21.15.32:0/1627334057 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557db2da1010 0x557db2e164f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.852+0000 7f41377fe640 1 -- 172.21.15.32:0/1627334057 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557db2d97980 msgr2=0x557db2da58f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:23.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.852+0000 7f41377fe640 1 --2- 172.21.15.32:0/1627334057 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557db2d97980 0x557db2da58f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.852+0000 7f41377fe640 1 -- 172.21.15.32:0/1627334057 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557db29ce170 con 0x557db2d96290 2024-04-25T00:33:23.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.852+0000 7f41377fe640 1 --2- 172.21.15.32:0/1627334057 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2d96290 0x557db2da53b0 secure :-1 s=READY pgs=506 cs=0 l=1 rev1=1 crypto rx=0x7f41280ed490 tx=0x7f4128009260 comp rx=0 tx=0).ready entity=mon.0 client_cookie=97a47abc005c156f server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:23.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.852+0000 7f4116ffd640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:33:23.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.852+0000 7f4116ffd640 1 -- 172.21.15.32:0/1627334057 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4128049040 con 0x557db2d96290 2024-04-25T00:33:23.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.852+0000 7f4116ffd640 1 -- 172.21.15.32:0/1627334057 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f41280ed080 con 0x557db2d96290 2024-04-25T00:33:23.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.852+0000 7f4140bda740 1 -- 172.21.15.32:0/1627334057 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f413001fcb0 con 0x557db2d96290 2024-04-25T00:33:23.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.852+0000 7f4116ffd640 1 -- 172.21.15.32:0/1627334057 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f41280bbba0 con 0x557db2d96290 2024-04-25T00:33:23.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.852+0000 7f4140bda740 10 client.15699 fetch_fsmap learned FSMap version 25 2024-04-25T00:33:23.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.852+0000 7f4140bda740 1 -- 172.21.15.32:0/1627334057 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x557db2e17b60 con 0x557db2d96290 2024-04-25T00:33:23.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.852+0000 7f4116ffd640 1 -- 172.21.15.32:0/1627334057 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f413001fcb0 con 0x557db2d96290 2024-04-25T00:33:23.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.853+0000 7f4116ffd640 1 -- 172.21.15.32:0/1627334057 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7f4128031bd0 con 0x557db2d96290 2024-04-25T00:33:23.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.853+0000 7f4140bda740 10 client.15699 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:33:23.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.853+0000 7f4140bda740 10 client.15699 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:33:23.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.853+0000 7f4140bda740 20 client.15699 populate_metadata read hostname 'smithi032' 2024-04-25T00:33:23.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.853+0000 7f4140bda740 1 --2- 172.21.15.32:0/1627334057 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x557db2e18230 0x557db2e1a6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.853+0000 7f4140bda740 4 client.15699 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-25T00:33:23.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.853+0000 7f4140bda740 1 -- 172.21.15.32:0/1627334057 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) -- 0x557db2d07800 con 0x557db2e18230 2024-04-25T00:33:23.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.853+0000 7f4136ffd640 1 --2- 172.21.15.32:0/1627334057 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x557db2e18230 0x557db2e1a6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.853+0000 7f4136ffd640 1 --2- 172.21.15.32:0/1627334057 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x557db2e18230 0x557db2e1a6b0 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b6dc5bad1e9ecfc4 server_cookie=c121773af3b2bd00 in_seq=0 out_seq=0 2024-04-25T00:33:23.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.853+0000 7f4116ffd640 10 client.15699 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:23.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.854+0000 7f4116ffd640 1 -- 172.21.15.32:0/1627334057 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+110 (crc 0 0 0) 0x557db2d07800 con 0x557db2e18230 2024-04-25T00:33:23.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.854+0000 7f4116ffd640 10 client.15699 handle_command_reply: tid=0 2024-04-25T00:33:23.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.854+0000 7f4140bda740 1 client.15699 shutdown 2024-04-25T00:33:23.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.855+0000 7f4140bda740 1 -- 172.21.15.32:0/1627334057 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2d96290 msgr2=0x557db2da53b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.855+0000 7f4140bda740 1 --2- 172.21.15.32:0/1627334057 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2d96290 0x557db2da53b0 secure :-1 s=READY pgs=506 cs=0 l=1 rev1=1 crypto rx=0x7f41280ed490 tx=0x7f4128009260 comp rx=0 tx=0).stop 2024-04-25T00:33:23.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.855+0000 7f4140bda740 1 -- 172.21.15.32:0/1627334057 shutdown_connections 2024-04-25T00:33:23.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.855+0000 7f4140bda740 1 --2- 172.21.15.32:0/1627334057 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x557db2e18230 0x557db2e1a6b0 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.855+0000 7f4140bda740 1 --2- 172.21.15.32:0/1627334057 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557db2da1010 0x557db2e164f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.855+0000 7f4140bda740 1 --2- 172.21.15.32:0/1627334057 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557db2d97980 0x557db2da58f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.855+0000 7f4116ffd640 0 client.15699 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:23.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.855+0000 7f4140bda740 1 --2- 172.21.15.32:0/1627334057 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557db2d96290 0x557db2da53b0 unknown :-1 s=CLOSED pgs=506 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.855+0000 7f4140bda740 1 -- 172.21.15.32:0/1627334057 >> 172.21.15.32:0/1627334057 conn(0x557db2d5e950 msgr2=0x557db2d91950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:23.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.855+0000 7f4140bda740 1 -- 172.21.15.32:0/1627334057 shutdown_connections 2024-04-25T00:33:23.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.855+0000 7f4140bda740 1 -- 172.21.15.32:0/1627334057 wait complete. 2024-04-25T00:33:23.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.855+0000 7f4140bda740 20 client.15699 trim_cache size 0 max 16384 2024-04-25T00:33:23.864 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:33:23.864 INFO:teuthology.orchestra.run.smithi032.stdout: "return_code": 0, 2024-04-25T00:33:23.864 INFO:teuthology.orchestra.run.smithi032.stdout: "scrub_tag": "fbd5e042-b5f0-4894-b901-3e8a2b75cac7", 2024-04-25T00:33:23.864 INFO:teuthology.orchestra.run.smithi032.stdout: "mode": "asynchronous" 2024-04-25T00:33:23.864 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:33:23.865 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.856+0000 7f413ddee640 1 -- 172.21.15.32:0/582164350 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f41200a2e10 msgr2=0x7f41200a52d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.865 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.856+0000 7f413ddee640 1 --2- 172.21.15.32:0/582164350 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f41200a2e10 0x7f41200a52d0 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f4130005fd0 tx=0x7f4130009500 comp rx=0 tx=0).stop 2024-04-25T00:33:23.865 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.856+0000 7f413ddee640 1 -- 172.21.15.32:0/582164350 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4138101710 msgr2=0x7f4138073a40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.865 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.856+0000 7f413ddee640 1 --2- 172.21.15.32:0/582164350 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4138101710 0x7f4138073a40 secure :-1 s=READY pgs=501 cs=0 l=1 rev1=1 crypto rx=0x7f4128009230 tx=0x7f4128004040 comp rx=0 tx=0).stop 2024-04-25T00:33:23.865 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.856+0000 7f413ddee640 1 -- 172.21.15.32:0/582164350 shutdown_connections 2024-04-25T00:33:23.865 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.856+0000 7f413ddee640 1 --2- 172.21.15.32:0/582164350 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f41200a2e10 0x7f41200a52d0 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.856+0000 7f413ddee640 1 --2- 172.21.15.32:0/582164350 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4138107720 0x7f41380744c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.856+0000 7f413ddee640 1 --2- 172.21.15.32:0/582164350 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f41381020d0 0x7f4138073f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.856+0000 7f413ddee640 1 --2- 172.21.15.32:0/582164350 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4138101710 0x7f4138073a40 unknown :-1 s=CLOSED pgs=501 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.856+0000 7f413ddee640 1 -- 172.21.15.32:0/582164350 >> 172.21.15.32:0/582164350 conn(0x7f4138078b90 msgr2=0x7f41380ff1a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:23.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.856+0000 7f413ddee640 1 -- 172.21.15.32:0/582164350 shutdown_connections 2024-04-25T00:33:23.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.856+0000 7f413ddee640 1 -- 172.21.15.32:0/582164350 wait complete. 2024-04-25T00:33:23.869 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-25T00:33:23.869 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-25T00:33:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: Updating smithi032:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:33:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: Updating smithi121:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:33:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: Updating smithi130:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T00:33:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: pgmap v5: 129 pgs: 129 active+clean; 1.1 MiB data, 373 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:33:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-25T00:33:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-04-25T00:33:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: Updating smithi032:/etc/ceph/ceph.client.admin.keyring 2024-04-25T00:33:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: Updating smithi130:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T00:33:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: Updating smithi121:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T00:33:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: Updating smithi032:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T00:33:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: mgrmap e24: x(active, since 4s), standbys: z, y 2024-04-25T00:33:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:23.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:23.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:23.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:23.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:23.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:23.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:23.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3856611077' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:33:23.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:33:23.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:33:23.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:33:23.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:33:23.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/538135899' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:33:23.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:23 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3554938134' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:33:23.978 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.974+0000 7fa83bf7e640 1 -- 172.21.15.32:0/488101901 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa8341066b0 msgr2=0x7fa834106a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.978 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.974+0000 7fa83bf7e640 1 --2- 172.21.15.32:0/488101901 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa8341066b0 0x7fa834106a90 secure :-1 s=READY pgs=507 cs=0 l=1 rev1=1 crypto rx=0x7fa830009930 tx=0x7fa83002a420 comp rx=0 tx=0).stop 2024-04-25T00:33:23.978 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.974+0000 7fa83bf7e640 1 -- 172.21.15.32:0/488101901 shutdown_connections 2024-04-25T00:33:23.978 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.974+0000 7fa83bf7e640 1 --2- 172.21.15.32:0/488101901 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa834100fd0 0x7fa83410d4e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.978 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.974+0000 7fa83bf7e640 1 --2- 172.21.15.32:0/488101901 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa834100610 0x7fa834100a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.978 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.974+0000 7fa83bf7e640 1 --2- 172.21.15.32:0/488101901 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa8341066b0 0x7fa834106a90 unknown :-1 s=CLOSED pgs=507 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.978 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.974+0000 7fa83bf7e640 1 -- 172.21.15.32:0/488101901 >> 172.21.15.32:0/488101901 conn(0x7fa834077740 msgr2=0x7fa834077b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:23.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.974+0000 7fa83bf7e640 1 -- 172.21.15.32:0/488101901 shutdown_connections 2024-04-25T00:33:23.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.974+0000 7fa83bf7e640 1 -- 172.21.15.32:0/488101901 wait complete. 2024-04-25T00:33:23.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.975+0000 7fa83bf7e640 1 Processor -- start 2024-04-25T00:33:23.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.975+0000 7fa83bf7e640 1 -- start start 2024-04-25T00:33:23.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.975+0000 7fa83bf7e640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa834100610 0x7fa83410df70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.975+0000 7fa83bf7e640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa834100fd0 0x7fa83410e4b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.975+0000 7fa83bf7e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa8341066b0 0x7fa83410e9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.975+0000 7fa83bf7e640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fa834111b30 con 0x7fa8341066b0 2024-04-25T00:33:23.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.975+0000 7fa83bf7e640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fa8341119b0 con 0x7fa834100fd0 2024-04-25T00:33:23.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.975+0000 7fa83bf7e640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fa834111cb0 con 0x7fa834100610 2024-04-25T00:33:23.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.975+0000 7fa83a51b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa8341066b0 0x7fa83410e9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.975+0000 7fa83a51b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa8341066b0 0x7fa83410e9f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:57974/0 (socket says 172.21.15.32:57974) 2024-04-25T00:33:23.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.975+0000 7fa839d1a640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa834100610 0x7fa83410df70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.975+0000 7fa83a51b640 1 -- 172.21.15.32:0/2241383976 learned_addr learned my addr 172.21.15.32:0/2241383976 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:23.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.975+0000 7fa839519640 1 --2- 172.21.15.32:0/2241383976 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa834100fd0 0x7fa83410e4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.975+0000 7fa83a51b640 1 -- 172.21.15.32:0/2241383976 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa834100610 msgr2=0x7fa83410df70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.975+0000 7fa83a51b640 1 --2- 172.21.15.32:0/2241383976 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa834100610 0x7fa83410df70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.975+0000 7fa83a51b640 1 -- 172.21.15.32:0/2241383976 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa834100fd0 msgr2=0x7fa83410e4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.975+0000 7fa83a51b640 1 --2- 172.21.15.32:0/2241383976 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa834100fd0 0x7fa83410e4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.975+0000 7fa83a51b640 1 -- 172.21.15.32:0/2241383976 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa8341a6530 con 0x7fa8341066b0 2024-04-25T00:33:23.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.975+0000 7fa839519640 1 --2- 172.21.15.32:0/2241383976 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa834100fd0 0x7fa83410e4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:23.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.976+0000 7fa83a51b640 1 --2- 172.21.15.32:0/2241383976 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa8341066b0 0x7fa83410e9f0 secure :-1 s=READY pgs=508 cs=0 l=1 rev1=1 crypto rx=0x7fa82c00eb10 tx=0x7fa82c00c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:23.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.976+0000 7fa82affd640 1 -- 172.21.15.32:0/2241383976 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa82c004b20 con 0x7fa8341066b0 2024-04-25T00:33:23.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.976+0000 7fa82affd640 1 -- 172.21.15.32:0/2241383976 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa82c004cc0 con 0x7fa8341066b0 2024-04-25T00:33:23.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.976+0000 7fa83bf7e640 1 -- 172.21.15.32:0/2241383976 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa8341a6820 con 0x7fa8341066b0 2024-04-25T00:33:23.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.976+0000 7fa82affd640 1 -- 172.21.15.32:0/2241383976 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa82c010760 con 0x7fa8341066b0 2024-04-25T00:33:23.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.976+0000 7fa83bf7e640 1 -- 172.21.15.32:0/2241383976 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa8341a6d70 con 0x7fa8341066b0 2024-04-25T00:33:23.987 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.978+0000 7fa82affd640 1 -- 172.21.15.32:0/2241383976 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 24) ==== 138009+0+0 (secure 0 0 0) 0x7fa82c005020 con 0x7fa8341066b0 2024-04-25T00:33:23.987 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.978+0000 7fa83ed6a740 1 Processor -- start 2024-04-25T00:33:23.987 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.979+0000 7fa83ed6a740 1 -- start start 2024-04-25T00:33:23.987 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.979+0000 7fa83ed6a740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131c630 0x55fe9131ca10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.979+0000 7fa83ed6a740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe9131cf50 0x55fe913270e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.979+0000 7fa83ed6a740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe91327620 0x55fe91329a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.979+0000 7fa83ed6a740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55fe912816b0 con 0x55fe9131c630 2024-04-25T00:33:23.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.979+0000 7fa83ed6a740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55fe91229880 con 0x55fe91327620 2024-04-25T00:33:23.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.979+0000 7fa83ed6a740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55fe9127d670 con 0x55fe9131cf50 2024-04-25T00:33:23.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.979+0000 7fa82affd640 1 --2- 172.21.15.32:0/2241383976 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa8200a2e10 0x7fa8200a52d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.979+0000 7fa839519640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131c630 0x55fe9131ca10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.979+0000 7fa839519640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131c630 0x55fe9131ca10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:57986/0 (socket says 172.21.15.32:57986) 2024-04-25T00:33:23.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.979+0000 7fa839519640 1 -- 172.21.15.32:0/3638172403 learned_addr learned my addr 172.21.15.32:0/3638172403 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:23.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.979+0000 7fa83a51b640 1 --2- 172.21.15.32:0/2241383976 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa8200a2e10 0x7fa8200a52d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.979+0000 7fa839d1a640 1 --2- 172.21.15.32:0/3638172403 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe9131cf50 0x55fe913270e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.979+0000 7fa82affd640 1 -- 172.21.15.32:0/2241383976 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fa82c0cfee0 con 0x7fa8341066b0 2024-04-25T00:33:23.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.979+0000 7fa839519640 1 -- 172.21.15.32:0/3638172403 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe9131cf50 msgr2=0x55fe913270e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:23.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.979+0000 7fa839519640 1 --2- 172.21.15.32:0/3638172403 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe9131cf50 0x55fe913270e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.989 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.979+0000 7fa839519640 1 -- 172.21.15.32:0/3638172403 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe91327620 msgr2=0x55fe91329a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:23.989 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.979+0000 7fa839519640 1 --2- 172.21.15.32:0/3638172403 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe91327620 0x55fe91329a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.989 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.979+0000 7fa839519640 1 -- 172.21.15.32:0/3638172403 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55fe9105f7d0 con 0x55fe9131c630 2024-04-25T00:33:23.989 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.979+0000 7fa839d1a640 1 --2- 172.21.15.32:0/3638172403 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe9131cf50 0x55fe913270e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:23.989 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.980+0000 7fa839519640 1 --2- 172.21.15.32:0/3638172403 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131c630 0x55fe9131ca10 secure :-1 s=READY pgs=509 cs=0 l=1 rev1=1 crypto rx=0x7fa824009d90 tx=0x7fa8240137d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3cd09743d1b44756 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:23.989 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.980+0000 7fa83a51b640 1 --2- 172.21.15.32:0/2241383976 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa8200a2e10 0x7fa8200a52d0 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7fa82c0cd220 tx=0x7fa82c00efd0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:23.989 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.980+0000 7fa81ffff640 1 -- 172.21.15.32:0/3638172403 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa82400f440 con 0x55fe9131c630 2024-04-25T00:33:23.989 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.980+0000 7fa81ffff640 1 -- 172.21.15.32:0/3638172403 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa824017030 con 0x55fe9131c630 2024-04-25T00:33:23.989 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.980+0000 7fa81ffff640 1 -- 172.21.15.32:0/3638172403 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa8240117a0 con 0x55fe9131c630 2024-04-25T00:33:23.989 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.980+0000 7fa83ed6a740 1 -- 172.21.15.32:0/3638172403 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131c630 msgr2=0x55fe9131ca10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.990 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.980+0000 7fa83ed6a740 1 --2- 172.21.15.32:0/3638172403 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131c630 0x55fe9131ca10 secure :-1 s=READY pgs=509 cs=0 l=1 rev1=1 crypto rx=0x7fa824009d90 tx=0x7fa8240137d0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.990 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.980+0000 7fa83ed6a740 1 -- 172.21.15.32:0/3638172403 shutdown_connections 2024-04-25T00:33:23.990 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.981+0000 7fa83ed6a740 1 --2- 172.21.15.32:0/3638172403 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe91327620 0x55fe91329a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.990 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.981+0000 7fa83ed6a740 1 --2- 172.21.15.32:0/3638172403 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe9131cf50 0x55fe913270e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.990 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.981+0000 7fa83ed6a740 1 --2- 172.21.15.32:0/3638172403 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131c630 0x55fe9131ca10 unknown :-1 s=CLOSED pgs=509 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.990 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.981+0000 7fa83ed6a740 1 -- 172.21.15.32:0/3638172403 >> 172.21.15.32:0/3638172403 conn(0x55fe91318950 msgr2=0x55fe91318d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:23.990 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.981+0000 7fa83ed6a740 1 -- 172.21.15.32:0/3638172403 shutdown_connections 2024-04-25T00:33:23.990 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.981+0000 7fa83ed6a740 1 -- 172.21.15.32:0/3638172403 wait complete. 2024-04-25T00:33:23.990 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.981+0000 7fa83ed6a740 1 -- start start 2024-04-25T00:33:23.990 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.981+0000 7fa83ed6a740 1 Processor -- start 2024-04-25T00:33:23.990 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.981+0000 7fa83ed6a740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe9131c630 0x55fe913245a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.981+0000 7fa83ed6a740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131cf50 0x55fe91324ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.981+0000 7fa83ed6a740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe91327620 0x55fe91325020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.981+0000 7fa83ed6a740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55fe9132a410 con 0x55fe9131cf50 2024-04-25T00:33:23.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.981+0000 7fa83ed6a740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55fe9123cf50 con 0x55fe91327620 2024-04-25T00:33:23.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.981+0000 7fa83ed6a740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55fe9132a590 con 0x55fe9131c630 2024-04-25T00:33:23.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.982+0000 7fa83ed6a740 10 client.0 _ll_register_callbacks cb 0x55fe91278000 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:33:23.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.982+0000 7fa839d1a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131cf50 0x55fe91324ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.982+0000 7fa839d1a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131cf50 0x55fe91324ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:58000/0 (socket says 172.21.15.32:58000) 2024-04-25T00:33:23.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.982+0000 7fa839d1a640 1 -- 172.21.15.32:0/3656368811 learned_addr learned my addr 172.21.15.32:0/3656368811 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:23.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.982+0000 7fa839519640 1 --2- 172.21.15.32:0/3656368811 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe91327620 0x55fe91325020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.982+0000 7fa839519640 1 --2- 172.21.15.32:0/3656368811 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe9131c630 0x55fe913245a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.982+0000 7fa839d1a640 1 -- 172.21.15.32:0/3656368811 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe9131c630 msgr2=0x55fe913245a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:23.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.982+0000 7fa839d1a640 1 --2- 172.21.15.32:0/3656368811 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe9131c630 0x55fe913245a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.982+0000 7fa839d1a640 1 -- 172.21.15.32:0/3656368811 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe91327620 msgr2=0x55fe91325020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:23.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.982+0000 7fa839d1a640 1 --2- 172.21.15.32:0/3656368811 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe91327620 0x55fe91325020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.982+0000 7fa839d1a640 1 -- 172.21.15.32:0/3656368811 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55fe90f953f0 con 0x55fe9131cf50 2024-04-25T00:33:23.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.982+0000 7fa839519640 1 --2- 172.21.15.32:0/3656368811 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe9131c630 0x55fe913245a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:33:23.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.982+0000 7fa839d1a640 1 --2- 172.21.15.32:0/3656368811 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131cf50 0x55fe91324ae0 secure :-1 s=READY pgs=510 cs=0 l=1 rev1=1 crypto rx=0x7fa830009900 tx=0x7fa830031860 comp rx=0 tx=0).ready entity=mon.0 client_cookie=431be1662a67a4eb server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:23.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.982+0000 7fa81d7fa640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:33:23.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.982+0000 7fa81d7fa640 1 -- 172.21.15.32:0/3656368811 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa830035040 con 0x55fe9131cf50 2024-04-25T00:33:23.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.982+0000 7fa81d7fa640 1 -- 172.21.15.32:0/3656368811 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fa830004240 con 0x55fe9131cf50 2024-04-25T00:33:23.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.982+0000 7fa83ed6a740 1 -- 172.21.15.32:0/3656368811 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa824017030 con 0x55fe9131cf50 2024-04-25T00:33:23.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.983+0000 7fa81d7fa640 1 -- 172.21.15.32:0/3656368811 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7fa830004440 con 0x55fe9131cf50 2024-04-25T00:33:23.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.983+0000 7fa83ed6a740 10 client.15711 fetch_fsmap learned FSMap version 25 2024-04-25T00:33:23.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.983+0000 7fa83ed6a740 1 -- 172.21.15.32:0/3656368811 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x55fe90f550f0 con 0x55fe9131cf50 2024-04-25T00:33:23.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.983+0000 7fa81d7fa640 1 -- 172.21.15.32:0/3656368811 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa824017030 con 0x55fe9131cf50 2024-04-25T00:33:23.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.983+0000 7fa81d7fa640 1 -- 172.21.15.32:0/3656368811 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7fa830004bb0 con 0x55fe9131cf50 2024-04-25T00:33:23.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.983+0000 7fa83ed6a740 10 client.15711 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:33:23.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.983+0000 7fa83ed6a740 10 client.15711 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:33:23.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.983+0000 7fa83ed6a740 20 client.15711 populate_metadata read hostname 'smithi032' 2024-04-25T00:33:23.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.983+0000 7fa83ed6a740 1 --2- 172.21.15.32:0/3656368811 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55fe913378e0 0x55fe91339cf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:23.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.983+0000 7fa83ed6a740 4 client.15711 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:33:23.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.983+0000 7fa83ed6a740 1 -- 172.21.15.32:0/3656368811 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55fe9128ebf0 con 0x55fe913378e0 2024-04-25T00:33:23.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.984+0000 7fa839519640 1 --2- 172.21.15.32:0/3656368811 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55fe913378e0 0x55fe91339cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:23.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.984+0000 7fa81d7fa640 10 client.15711 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:23.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.984+0000 7fa839519640 1 --2- 172.21.15.32:0/3656368811 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55fe913378e0 0x55fe91339cf0 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=62b94b45db160019 server_cookie=91cf7ae451ded4a9 in_seq=0 out_seq=0 2024-04-25T00:33:23.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.985+0000 7fa81d7fa640 1 -- 172.21.15.32:0/3656368811 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x55fe9128ebf0 con 0x55fe913378e0 2024-04-25T00:33:23.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.985+0000 7fa81d7fa640 10 client.15711 handle_command_reply: tid=0 2024-04-25T00:33:23.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.985+0000 7fa83ed6a740 1 client.15711 shutdown 2024-04-25T00:33:23.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.985+0000 7fa83ed6a740 1 -- 172.21.15.32:0/3656368811 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131cf50 msgr2=0x55fe91324ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:23.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.985+0000 7fa83ed6a740 1 --2- 172.21.15.32:0/3656368811 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131cf50 0x55fe91324ae0 secure :-1 s=READY pgs=510 cs=0 l=1 rev1=1 crypto rx=0x7fa830009900 tx=0x7fa830031860 comp rx=0 tx=0).stop 2024-04-25T00:33:23.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.985+0000 7fa83ed6a740 1 -- 172.21.15.32:0/3656368811 shutdown_connections 2024-04-25T00:33:23.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.985+0000 7fa83ed6a740 1 --2- 172.21.15.32:0/3656368811 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55fe913378e0 0x55fe91339cf0 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.994 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.985+0000 7fa83ed6a740 1 --2- 172.21.15.32:0/3656368811 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe91327620 0x55fe91325020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.994 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.985+0000 7fa83ed6a740 1 --2- 172.21.15.32:0/3656368811 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131cf50 0x55fe91324ae0 unknown :-1 s=CLOSED pgs=510 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.994 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.985+0000 7fa81d7fa640 0 client.15711 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:23.994 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.985+0000 7fa83ed6a740 1 --2- 172.21.15.32:0/3656368811 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe9131c630 0x55fe913245a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:23.994 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.985+0000 7fa83ed6a740 1 -- 172.21.15.32:0/3656368811 >> 172.21.15.32:0/3656368811 conn(0x55fe91318950 msgr2=0x55fe9131e750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:23.994 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.985+0000 7fa83ed6a740 1 -- 172.21.15.32:0/3656368811 shutdown_connections 2024-04-25T00:33:23.994 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.985+0000 7fa83ed6a740 1 -- 172.21.15.32:0/3656368811 wait complete. 2024-04-25T00:33:23.994 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:23.985+0000 7fa83ed6a740 20 client.15711 trim_cache size 0 max 16384 2024-04-25T00:33:24.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.003+0000 7fa83ed6a740 1 Processor -- start 2024-04-25T00:33:24.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.003+0000 7fa83ed6a740 1 -- start start 2024-04-25T00:33:24.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.003+0000 7fa83ed6a740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131c630 0x55fe913332d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:24.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.003+0000 7fa83ed6a740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe9131cf50 0x55fe9131ed90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:24.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.003+0000 7fa83ed6a740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe91327620 0x55fe913263d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:24.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.003+0000 7fa83ed6a740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fa8341b37e0 con 0x55fe9131c630 2024-04-25T00:33:24.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.003+0000 7fa83ed6a740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fa8341b10b0 con 0x55fe9131cf50 2024-04-25T00:33:24.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.003+0000 7fa83ed6a740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fa8341aef00 con 0x55fe91327620 2024-04-25T00:33:24.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.003+0000 7fa839519640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131c630 0x55fe913332d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:24.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.003+0000 7fa839519640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131c630 0x55fe913332d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:58008/0 (socket says 172.21.15.32:58008) 2024-04-25T00:33:24.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.003+0000 7fa839519640 1 -- 172.21.15.32:0/2435998057 learned_addr learned my addr 172.21.15.32:0/2435998057 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:24.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.003+0000 7fa839d1a640 1 --2- 172.21.15.32:0/2435998057 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe9131cf50 0x55fe9131ed90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:24.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.003+0000 7fa839519640 1 -- 172.21.15.32:0/2435998057 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe91327620 msgr2=0x55fe913263d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:24.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.003+0000 7fa839519640 1 --2- 172.21.15.32:0/2435998057 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe91327620 0x55fe913263d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.003+0000 7fa839519640 1 -- 172.21.15.32:0/2435998057 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe9131cf50 msgr2=0x55fe9131ed90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:24.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.003+0000 7fa839519640 1 --2- 172.21.15.32:0/2435998057 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe9131cf50 0x55fe9131ed90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.003+0000 7fa839519640 1 -- 172.21.15.32:0/2435998057 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55fe9131cd80 con 0x55fe9131c630 2024-04-25T00:33:24.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.003+0000 7fa839d1a640 1 --2- 172.21.15.32:0/2435998057 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe9131cf50 0x55fe9131ed90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:24.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.004+0000 7fa839519640 1 --2- 172.21.15.32:0/2435998057 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131c630 0x55fe913332d0 secure :-1 s=READY pgs=511 cs=0 l=1 rev1=1 crypto rx=0x7fa824009d30 tx=0x7fa824011440 comp rx=0 tx=0).ready entity=mon.0 client_cookie=642bf221f19d12d3 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:24.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.004+0000 7fa81f7fe640 1 -- 172.21.15.32:0/2435998057 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa8240283a0 con 0x55fe9131c630 2024-04-25T00:33:24.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.004+0000 7fa81f7fe640 1 -- 172.21.15.32:0/2435998057 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa824017040 con 0x55fe9131c630 2024-04-25T00:33:24.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.004+0000 7fa81f7fe640 1 -- 172.21.15.32:0/2435998057 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa82400f730 con 0x55fe9131c630 2024-04-25T00:33:24.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.004+0000 7fa83ed6a740 1 -- 172.21.15.32:0/2435998057 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131c630 msgr2=0x55fe913332d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:24.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.004+0000 7fa83ed6a740 1 --2- 172.21.15.32:0/2435998057 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131c630 0x55fe913332d0 secure :-1 s=READY pgs=511 cs=0 l=1 rev1=1 crypto rx=0x7fa824009d30 tx=0x7fa824011440 comp rx=0 tx=0).stop 2024-04-25T00:33:24.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.004+0000 7fa83ed6a740 1 -- 172.21.15.32:0/2435998057 shutdown_connections 2024-04-25T00:33:24.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.004+0000 7fa83ed6a740 1 --2- 172.21.15.32:0/2435998057 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe91327620 0x55fe913263d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.004+0000 7fa83ed6a740 1 --2- 172.21.15.32:0/2435998057 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe9131cf50 0x55fe9131ed90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.004+0000 7fa83ed6a740 1 --2- 172.21.15.32:0/2435998057 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131c630 0x55fe913332d0 unknown :-1 s=CLOSED pgs=511 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.004+0000 7fa83ed6a740 1 -- 172.21.15.32:0/2435998057 >> 172.21.15.32:0/2435998057 conn(0x55fe91318950 msgr2=0x55fe91325cc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:24.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.005+0000 7fa83ed6a740 1 -- 172.21.15.32:0/2435998057 shutdown_connections 2024-04-25T00:33:24.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.005+0000 7fa83ed6a740 1 -- 172.21.15.32:0/2435998057 wait complete. 2024-04-25T00:33:24.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.005+0000 7fa83ed6a740 1 -- start start 2024-04-25T00:33:24.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.005+0000 7fa83ed6a740 1 Processor -- start 2024-04-25T00:33:24.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.005+0000 7fa83ed6a740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe9131c630 0x55fe9132fd50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:24.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.005+0000 7fa83ed6a740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131cf50 0x55fe913210a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:24.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.005+0000 7fa83ed6a740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe91327620 0x55fe91321790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:24.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.005+0000 7fa83ed6a740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fa8341af570 con 0x55fe9131cf50 2024-04-25T00:33:24.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.005+0000 7fa83ed6a740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fa8341b09e0 con 0x55fe91327620 2024-04-25T00:33:24.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.005+0000 7fa83ed6a740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fa8341afc40 con 0x55fe9131c630 2024-04-25T00:33:24.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.005+0000 7fa83ed6a740 10 client.0 _ll_register_callbacks cb 0x7fa824005e30 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:33:24.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.005+0000 7fa839d1a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131cf50 0x55fe913210a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:24.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.005+0000 7fa839d1a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131cf50 0x55fe913210a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:58020/0 (socket says 172.21.15.32:58020) 2024-04-25T00:33:24.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.005+0000 7fa839d1a640 1 -- 172.21.15.32:0/2608654204 learned_addr learned my addr 172.21.15.32:0/2608654204 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:24.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.006+0000 7fa839d1a640 1 -- 172.21.15.32:0/2608654204 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe9131c630 msgr2=0x55fe9132fd50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:24.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.006+0000 7fa839519640 1 --2- 172.21.15.32:0/2608654204 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe9131c630 0x55fe9132fd50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:24.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.006+0000 7fa839519640 1 --2- 172.21.15.32:0/2608654204 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe91327620 0x55fe91321790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:24.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.006+0000 7fa839d1a640 1 --2- 172.21.15.32:0/2608654204 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe9131c630 0x55fe9132fd50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.006+0000 7fa839d1a640 1 -- 172.21.15.32:0/2608654204 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe91327620 msgr2=0x55fe91321790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:24.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.006+0000 7fa839d1a640 1 --2- 172.21.15.32:0/2608654204 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe91327620 0x55fe91321790 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.006+0000 7fa839d1a640 1 -- 172.21.15.32:0/2608654204 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55fe9139ce30 con 0x55fe9131cf50 2024-04-25T00:33:24.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.006+0000 7fa839d1a640 1 --2- 172.21.15.32:0/2608654204 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131cf50 0x55fe913210a0 secure :-1 s=READY pgs=512 cs=0 l=1 rev1=1 crypto rx=0x7fa830006010 tx=0x7fa830031b00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d02d1db9c3442bcb server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:24.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.006+0000 7fa81d7fa640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:33:24.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.006+0000 7fa81d7fa640 1 -- 172.21.15.32:0/2608654204 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa830035040 con 0x55fe9131cf50 2024-04-25T00:33:24.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.006+0000 7fa81d7fa640 1 -- 172.21.15.32:0/2608654204 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fa830004b80 con 0x55fe9131cf50 2024-04-25T00:33:24.014 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.006+0000 7fa83ed6a740 1 -- 172.21.15.32:0/2608654204 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa824017040 con 0x55fe9131cf50 2024-04-25T00:33:24.014 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.006+0000 7fa81d7fa640 1 -- 172.21.15.32:0/2608654204 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7fa830004fd0 con 0x55fe9131cf50 2024-04-25T00:33:24.014 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.006+0000 7fa83ed6a740 10 client.15717 fetch_fsmap learned FSMap version 25 2024-04-25T00:33:24.014 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.006+0000 7fa83ed6a740 1 -- 172.21.15.32:0/2608654204 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x55fe9139dfe0 con 0x55fe9131cf50 2024-04-25T00:33:24.014 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.006+0000 7fa81d7fa640 1 -- 172.21.15.32:0/2608654204 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa824017040 con 0x55fe9131cf50 2024-04-25T00:33:24.014 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.007+0000 7fa81d7fa640 1 -- 172.21.15.32:0/2608654204 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7fa830003cf0 con 0x55fe9131cf50 2024-04-25T00:33:24.014 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.007+0000 7fa83ed6a740 10 client.15717 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:33:24.014 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.007+0000 7fa83ed6a740 10 client.15717 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:33:24.014 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.007+0000 7fa83ed6a740 20 client.15717 populate_metadata read hostname 'smithi032' 2024-04-25T00:33:24.014 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.007+0000 7fa83ed6a740 1 --2- 172.21.15.32:0/2608654204 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55fe9139e780 0x55fe913a0ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:24.014 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.007+0000 7fa83ed6a740 4 client.15717 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-25T00:33:24.014 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.007+0000 7fa83ed6a740 1 -- 172.21.15.32:0/2608654204 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "scrub status"}) -- 0x55fe91288f40 con 0x55fe9139e780 2024-04-25T00:33:24.014 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.007+0000 7fa839519640 1 --2- 172.21.15.32:0/2608654204 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55fe9139e780 0x55fe913a0ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:24.014 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.007+0000 7fa839519640 1 --2- 172.21.15.32:0/2608654204 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55fe9139e780 0x55fe913a0ba0 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3b8faa53bbb282a server_cookie=4977e65c178afc8d in_seq=0 out_seq=0 2024-04-25T00:33:24.015 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.007+0000 7fa81d7fa640 10 client.15717 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:24.015 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.008+0000 7fa81d7fa640 1 -- 172.21.15.32:0/2608654204 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+259 (crc 0 0 0) 0x55fe91288f40 con 0x55fe9139e780 2024-04-25T00:33:24.015 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.008+0000 7fa81d7fa640 10 client.15717 handle_command_reply: tid=0 2024-04-25T00:33:24.015 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.008+0000 7fa83ed6a740 1 client.15717 shutdown 2024-04-25T00:33:24.015 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.008+0000 7fa83ed6a740 1 -- 172.21.15.32:0/2608654204 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131cf50 msgr2=0x55fe913210a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:24.015 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.008+0000 7fa83ed6a740 1 --2- 172.21.15.32:0/2608654204 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131cf50 0x55fe913210a0 secure :-1 s=READY pgs=512 cs=0 l=1 rev1=1 crypto rx=0x7fa830006010 tx=0x7fa830031b00 comp rx=0 tx=0).stop 2024-04-25T00:33:24.015 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.008+0000 7fa83ed6a740 1 -- 172.21.15.32:0/2608654204 shutdown_connections 2024-04-25T00:33:24.015 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.008+0000 7fa83ed6a740 1 --2- 172.21.15.32:0/2608654204 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55fe9139e780 0x55fe913a0ba0 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.015 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.008+0000 7fa83ed6a740 1 --2- 172.21.15.32:0/2608654204 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe91327620 0x55fe91321790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.015 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.008+0000 7fa83ed6a740 1 --2- 172.21.15.32:0/2608654204 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe9131cf50 0x55fe913210a0 unknown :-1 s=CLOSED pgs=512 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.015 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.008+0000 7fa81d7fa640 0 client.15717 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:24.015 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.008+0000 7fa83ed6a740 1 --2- 172.21.15.32:0/2608654204 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe9131c630 0x55fe9132fd50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.015 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.008+0000 7fa83ed6a740 1 -- 172.21.15.32:0/2608654204 >> 172.21.15.32:0/2608654204 conn(0x55fe91318950 msgr2=0x55fe91324dd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:24.015 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.008+0000 7fa83ed6a740 1 -- 172.21.15.32:0/2608654204 shutdown_connections 2024-04-25T00:33:24.015 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.008+0000 7fa83ed6a740 1 -- 172.21.15.32:0/2608654204 wait complete. 2024-04-25T00:33:24.016 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.008+0000 7fa83ed6a740 20 client.15717 trim_cache size 0 max 16384 2024-04-25T00:33:24.016 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:33:24.016 INFO:teuthology.orchestra.run.smithi032.stdout: "status": "IDLE (waiting for more scrubs)", 2024-04-25T00:33:24.016 INFO:teuthology.orchestra.run.smithi032.stdout: "scrubs": { 2024-04-25T00:33:24.016 INFO:teuthology.orchestra.run.smithi032.stdout: "fbd5e042-b5f0-4894-b901-3e8a2b75cac7": { 2024-04-25T00:33:24.016 INFO:teuthology.orchestra.run.smithi032.stdout: "path": "/", 2024-04-25T00:33:24.016 INFO:teuthology.orchestra.run.smithi032.stdout: "tag": "fbd5e042-b5f0-4894-b901-3e8a2b75cac7", 2024-04-25T00:33:24.016 INFO:teuthology.orchestra.run.smithi032.stdout: "options": "recursive,force" 2024-04-25T00:33:24.016 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:33:24.016 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:33:24.017 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:33:24.017 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.008+0000 7fa83bf7e640 1 -- 172.21.15.32:0/2241383976 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa8200a2e10 msgr2=0x7fa8200a52d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:24.017 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.008+0000 7fa83bf7e640 1 --2- 172.21.15.32:0/2241383976 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa8200a2e10 0x7fa8200a52d0 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7fa82c0cd220 tx=0x7fa82c00efd0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.017 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.008+0000 7fa83bf7e640 1 -- 172.21.15.32:0/2241383976 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa8341066b0 msgr2=0x7fa83410e9f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:24.017 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.008+0000 7fa83bf7e640 1 --2- 172.21.15.32:0/2241383976 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa8341066b0 0x7fa83410e9f0 secure :-1 s=READY pgs=508 cs=0 l=1 rev1=1 crypto rx=0x7fa82c00eb10 tx=0x7fa82c00c2d0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.018 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.009+0000 7fa83bf7e640 1 -- 172.21.15.32:0/2241383976 shutdown_connections 2024-04-25T00:33:24.018 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.009+0000 7fa83bf7e640 1 --2- 172.21.15.32:0/2241383976 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa8200a2e10 0x7fa8200a52d0 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.018 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.009+0000 7fa83bf7e640 1 --2- 172.21.15.32:0/2241383976 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa8341066b0 0x7fa83410e9f0 unknown :-1 s=CLOSED pgs=508 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.018 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.009+0000 7fa83bf7e640 1 --2- 172.21.15.32:0/2241383976 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa834100fd0 0x7fa83410e4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.018 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.009+0000 7fa83bf7e640 1 --2- 172.21.15.32:0/2241383976 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa834100610 0x7fa83410df70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.018 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.009+0000 7fa83bf7e640 1 -- 172.21.15.32:0/2241383976 >> 172.21.15.32:0/2241383976 conn(0x7fa834077740 msgr2=0x7fa8340ff2a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:24.018 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.009+0000 7fa83bf7e640 1 -- 172.21.15.32:0/2241383976 shutdown_connections 2024-04-25T00:33:24.018 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.009+0000 7fa83bf7e640 1 -- 172.21.15.32:0/2241383976 wait complete. 2024-04-25T00:33:24.021 INFO:tasks.cephfs.filesystem:scrub status for tag:fbd5e042-b5f0-4894-b901-3e8a2b75cac7 - {'path': '/', 'tag': 'fbd5e042-b5f0-4894-b901-3e8a2b75cac7', 'options': 'recursive,force'} 2024-04-25T00:33:24.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.045+0000 7f90a06e1640 1 -- 172.21.15.32:0/2477513410 --> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] -- mgr_command(tid 0: {"prefix": "fs snap-schedule add", "fs": "cephfs", "path": "/", "snap_schedule": "1m", "target": ["mon-mgr", ""]}) -- 0x7f9098061b80 con 0x7f90840a2ec0 2024-04-25T00:33:24.112 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.110+0000 7f908f7fe640 1 -- 172.21.15.32:0/2477513410 <== mgr.15483 v2:172.21.15.32:6800/3307874911 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+24 (secure 0 0 0) 0x7f9098061b80 con 0x7f90840a2ec0 2024-04-25T00:33:24.112 INFO:teuthology.orchestra.run.smithi032.stdout:Schedule set for path / 2024-04-25T00:33:24.114 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.112+0000 7f90a06e1640 1 -- 172.21.15.32:0/2477513410 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f90840a2ec0 msgr2=0x7f90840a5380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:24.114 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.112+0000 7f90a06e1640 1 --2- 172.21.15.32:0/2477513410 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f90840a2ec0 0x7f90840a5380 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f9094009b50 tx=0x7f9094005f90 comp rx=0 tx=0).stop 2024-04-25T00:33:24.114 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.113+0000 7f90a06e1640 1 -- 172.21.15.32:0/2477513410 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9098106630 msgr2=0x7f909819c8d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:24.114 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.113+0000 7f90a06e1640 1 --2- 172.21.15.32:0/2477513410 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9098106630 0x7f909819c8d0 secure :-1 s=READY pgs=502 cs=0 l=1 rev1=1 crypto rx=0x7f9090005cc0 tx=0x7f90900093d0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.114 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.113+0000 7f90a06e1640 1 -- 172.21.15.32:0/2477513410 shutdown_connections 2024-04-25T00:33:24.115 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.113+0000 7f90a06e1640 1 --2- 172.21.15.32:0/2477513410 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f90840a2ec0 0x7f90840a5380 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.115 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.113+0000 7f90a06e1640 1 --2- 172.21.15.32:0/2477513410 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9098106630 0x7f909819c8d0 unknown :-1 s=CLOSED pgs=502 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.115 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.113+0000 7f90a06e1640 1 --2- 172.21.15.32:0/2477513410 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9098101030 0x7f9098198550 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.115 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.113+0000 7f90a06e1640 1 --2- 172.21.15.32:0/2477513410 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9098100670 0x7f9098198010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.115 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.113+0000 7f90a06e1640 1 -- 172.21.15.32:0/2477513410 >> 172.21.15.32:0/2477513410 conn(0x7f90980777c0 msgr2=0x7f9098069130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:24.115 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.113+0000 7f90a06e1640 1 -- 172.21.15.32:0/2477513410 shutdown_connections 2024-04-25T00:33:24.115 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.113+0000 7f90a06e1640 1 -- 172.21.15.32:0/2477513410 wait complete. 2024-04-25T00:33:24.125 DEBUG:teuthology.orchestra.run.smithi032:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule retention add --fs=cephfs --path=/ --retention-spec-or-period=6m3h' 2024-04-25T00:33:24.274 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.269+0000 7f682641c640 1 -- 172.21.15.32:0/2067726101 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f68201032f0 msgr2=0x7f682010b7d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:24.274 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.269+0000 7f682641c640 1 --2- 172.21.15.32:0/2067726101 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f68201032f0 0x7f682010b7d0 secure :-1 s=READY pgs=513 cs=0 l=1 rev1=1 crypto rx=0x7f6814009900 tx=0x7f681402a510 comp rx=0 tx=0).stop 2024-04-25T00:33:24.274 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.269+0000 7f682641c640 1 -- 172.21.15.32:0/2067726101 shutdown_connections 2024-04-25T00:33:24.274 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.269+0000 7f682641c640 1 --2- 172.21.15.32:0/2067726101 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f682010bd10 0x7f682010e1b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.274 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.269+0000 7f682641c640 1 --2- 172.21.15.32:0/2067726101 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f68201032f0 0x7f682010b7d0 unknown :-1 s=CLOSED pgs=513 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.274 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.269+0000 7f682641c640 1 --2- 172.21.15.32:0/2067726101 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f68201029d0 0x7f6820102db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.274 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.269+0000 7f682641c640 1 -- 172.21.15.32:0/2067726101 >> 172.21.15.32:0/2067726101 conn(0x7f68200777c0 msgr2=0x7f6820077bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:24.274 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.270+0000 7f682641c640 1 -- 172.21.15.32:0/2067726101 shutdown_connections 2024-04-25T00:33:24.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.270+0000 7f682641c640 1 -- 172.21.15.32:0/2067726101 wait complete. 2024-04-25T00:33:24.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.270+0000 7f682641c640 1 Processor -- start 2024-04-25T00:33:24.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.270+0000 7f682641c640 1 -- start start 2024-04-25T00:33:24.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.270+0000 7f682641c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f68201029d0 0x7f6820100620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:24.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.270+0000 7f682641c640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f68201032f0 0x7f6820100b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:24.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.270+0000 7f682641c640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f682010bd10 0x7f68201010a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:24.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.270+0000 7f682641c640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f6820111b10 con 0x7f68201029d0 2024-04-25T00:33:24.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.270+0000 7f682641c640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f6820111990 con 0x7f682010bd10 2024-04-25T00:33:24.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.270+0000 7f682641c640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f6820111c90 con 0x7f68201032f0 2024-04-25T00:33:24.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.271+0000 7f681ffff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f68201029d0 0x7f6820100620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:24.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.271+0000 7f681ffff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f68201029d0 0x7f6820100620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:58034/0 (socket says 172.21.15.32:58034) 2024-04-25T00:33:24.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.271+0000 7f681ffff640 1 -- 172.21.15.32:0/2054576897 learned_addr learned my addr 172.21.15.32:0/2054576897 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:24.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.271+0000 7f681f7fe640 1 --2- 172.21.15.32:0/2054576897 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f68201032f0 0x7f6820100b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:24.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.271+0000 7f681ffff640 1 -- 172.21.15.32:0/2054576897 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f68201032f0 msgr2=0x7f6820100b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:24.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.271+0000 7f681ffff640 1 --2- 172.21.15.32:0/2054576897 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f68201032f0 0x7f6820100b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.271+0000 7f681ffff640 1 -- 172.21.15.32:0/2054576897 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f682010bd10 msgr2=0x7f68201010a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:24.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.271+0000 7f681ffff640 1 --2- 172.21.15.32:0/2054576897 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f682010bd10 0x7f68201010a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.271+0000 7f681ffff640 1 -- 172.21.15.32:0/2054576897 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f68201a6470 con 0x7f68201029d0 2024-04-25T00:33:24.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.271+0000 7f681f7fe640 1 --2- 172.21.15.32:0/2054576897 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f68201032f0 0x7f6820100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:24.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.271+0000 7f681ffff640 1 --2- 172.21.15.32:0/2054576897 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f68201029d0 0x7f6820100620 secure :-1 s=READY pgs=514 cs=0 l=1 rev1=1 crypto rx=0x7f681000d950 tx=0x7f681000de10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:24.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.271+0000 7f681d7fa640 1 -- 172.21.15.32:0/2054576897 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6810017070 con 0x7f68201029d0 2024-04-25T00:33:24.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.271+0000 7f682641c640 1 -- 172.21.15.32:0/2054576897 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f68201a6760 con 0x7f68201029d0 2024-04-25T00:33:24.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.271+0000 7f681d7fa640 1 -- 172.21.15.32:0/2054576897 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6810009c30 con 0x7f68201029d0 2024-04-25T00:33:24.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.271+0000 7f681d7fa640 1 -- 172.21.15.32:0/2054576897 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f681000f6b0 con 0x7f68201029d0 2024-04-25T00:33:24.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.271+0000 7f682641c640 1 -- 172.21.15.32:0/2054576897 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f68201a6cb0 con 0x7f68201029d0 2024-04-25T00:33:24.277 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.272+0000 7f682641c640 1 -- 172.21.15.32:0/2054576897 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f682004f500 con 0x7f68201029d0 2024-04-25T00:33:24.277 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.273+0000 7f681d7fa640 1 -- 172.21.15.32:0/2054576897 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 24) ==== 138009+0+0 (secure 0 0 0) 0x7f68100109d0 con 0x7f68201029d0 2024-04-25T00:33:24.278 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.274+0000 7f681d7fa640 1 --2- 172.21.15.32:0/2054576897 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f68080a2e70 0x7f68080a5330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:24.278 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.274+0000 7f681f7fe640 1 --2- 172.21.15.32:0/2054576897 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f68080a2e70 0x7f68080a5330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:24.278 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.274+0000 7f681d7fa640 1 -- 172.21.15.32:0/2054576897 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f68100cf640 con 0x7f68201029d0 2024-04-25T00:33:24.278 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.274+0000 7f681f7fe640 1 --2- 172.21.15.32:0/2054576897 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f68080a2e70 0x7f68080a5330 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f6820102060 tx=0x7f6814005a20 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:24.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.278+0000 7f681d7fa640 1 -- 172.21.15.32:0/2054576897 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f6810009de0 con 0x7f68201029d0 2024-04-25T00:33:24.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.485+0000 7f682641c640 1 -- 172.21.15.32:0/2054576897 --> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] -- mgr_command(tid 0: {"prefix": "fs snap-schedule retention add", "fs": "cephfs", "path": "/", "retention_spec_or_period": "6m3h", "target": ["mon-mgr", ""]}) -- 0x7f6820061b80 con 0x7f68080a2e70 2024-04-25T00:33:24.550 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.548+0000 7f681d7fa640 1 -- 172.21.15.32:0/2054576897 <== mgr.15483 v2:172.21.15.32:6800/3307874911 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+26 (secure 0 0 0) 0x7f6820061b80 con 0x7f68080a2e70 2024-04-25T00:33:24.550 INFO:teuthology.orchestra.run.smithi032.stdout:Retention added to path / 2024-04-25T00:33:24.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.550+0000 7f682641c640 1 -- 172.21.15.32:0/2054576897 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f68080a2e70 msgr2=0x7f68080a5330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:24.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.550+0000 7f682641c640 1 --2- 172.21.15.32:0/2054576897 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f68080a2e70 0x7f68080a5330 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f6820102060 tx=0x7f6814005a20 comp rx=0 tx=0).stop 2024-04-25T00:33:24.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.550+0000 7f682641c640 1 -- 172.21.15.32:0/2054576897 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f68201029d0 msgr2=0x7f6820100620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:24.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.550+0000 7f682641c640 1 --2- 172.21.15.32:0/2054576897 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f68201029d0 0x7f6820100620 secure :-1 s=READY pgs=514 cs=0 l=1 rev1=1 crypto rx=0x7f681000d950 tx=0x7f681000de10 comp rx=0 tx=0).stop 2024-04-25T00:33:24.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.550+0000 7f682641c640 1 -- 172.21.15.32:0/2054576897 shutdown_connections 2024-04-25T00:33:24.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.550+0000 7f682641c640 1 --2- 172.21.15.32:0/2054576897 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f68080a2e70 0x7f68080a5330 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.550+0000 7f682641c640 1 --2- 172.21.15.32:0/2054576897 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f682010bd10 0x7f68201010a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.553 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.550+0000 7f682641c640 1 --2- 172.21.15.32:0/2054576897 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f68201032f0 0x7f6820100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.553 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.550+0000 7f682641c640 1 --2- 172.21.15.32:0/2054576897 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f68201029d0 0x7f6820100620 unknown :-1 s=CLOSED pgs=514 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.553 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.550+0000 7f682641c640 1 -- 172.21.15.32:0/2054576897 >> 172.21.15.32:0/2054576897 conn(0x7f68200777c0 msgr2=0x7f68200fd710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:24.553 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.550+0000 7f682641c640 1 -- 172.21.15.32:0/2054576897 shutdown_connections 2024-04-25T00:33:24.553 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.550+0000 7f682641c640 1 -- 172.21.15.32:0/2054576897 wait complete. 2024-04-25T00:33:24.563 DEBUG:teuthology.orchestra.run.smithi032:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule status --fs=cephfs --path=/' 2024-04-25T00:33:24.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.664+0000 7fa30b7b6640 1 -- 172.21.15.32:0/1878865154 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa304106670 msgr2=0x7fa304106a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:24.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.664+0000 7fa30b7b6640 1 --2- 172.21.15.32:0/1878865154 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa304106670 0x7fa304106a50 secure :-1 s=READY pgs=515 cs=0 l=1 rev1=1 crypto rx=0x7fa300009930 tx=0x7fa30002a420 comp rx=0 tx=0).stop 2024-04-25T00:33:24.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.664+0000 7fa30b7b6640 1 -- 172.21.15.32:0/1878865154 shutdown_connections 2024-04-25T00:33:24.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.664+0000 7fa30b7b6640 1 --2- 172.21.15.32:0/1878865154 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa304101020 0x7fa30410d530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.664+0000 7fa30b7b6640 1 --2- 172.21.15.32:0/1878865154 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa304100660 0x7fa304100ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.664+0000 7fa30b7b6640 1 --2- 172.21.15.32:0/1878865154 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa304106670 0x7fa304106a50 unknown :-1 s=CLOSED pgs=515 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.664+0000 7fa30b7b6640 1 -- 172.21.15.32:0/1878865154 >> 172.21.15.32:0/1878865154 conn(0x7fa3040769c0 msgr2=0x7fa304076dd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:24.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.664+0000 7fa30b7b6640 1 -- 172.21.15.32:0/1878865154 shutdown_connections 2024-04-25T00:33:24.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.664+0000 7fa30b7b6640 1 -- 172.21.15.32:0/1878865154 wait complete. 2024-04-25T00:33:24.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.665+0000 7fa30b7b6640 1 Processor -- start 2024-04-25T00:33:24.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.665+0000 7fa30b7b6640 1 -- start start 2024-04-25T00:33:24.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.665+0000 7fa30b7b6640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa304100660 0x7fa304198da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:24.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.665+0000 7fa30b7b6640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa304101020 0x7fa3041992e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:24.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.665+0000 7fa30b7b6640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa304106670 0x7fa3041a3e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:24.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.665+0000 7fa30b7b6640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fa304110a40 con 0x7fa304101020 2024-04-25T00:33:24.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.665+0000 7fa30b7b6640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fa3041108c0 con 0x7fa304106670 2024-04-25T00:33:24.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.665+0000 7fa30b7b6640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fa304110bc0 con 0x7fa304100660 2024-04-25T00:33:24.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.665+0000 7fa308d51640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa304101020 0x7fa3041992e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:24.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.665+0000 7fa308d51640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa304101020 0x7fa3041992e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:58058/0 (socket says 172.21.15.32:58058) 2024-04-25T00:33:24.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.665+0000 7fa308d51640 1 -- 172.21.15.32:0/1516403219 learned_addr learned my addr 172.21.15.32:0/1516403219 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:24.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.665+0000 7fa309552640 1 --2- 172.21.15.32:0/1516403219 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa304100660 0x7fa304198da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:24.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.666+0000 7fa309d53640 1 --2- 172.21.15.32:0/1516403219 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa304106670 0x7fa3041a3e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:24.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.666+0000 7fa308d51640 1 -- 172.21.15.32:0/1516403219 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa304100660 msgr2=0x7fa304198da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:24.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.666+0000 7fa308d51640 1 --2- 172.21.15.32:0/1516403219 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa304100660 0x7fa304198da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.666+0000 7fa308d51640 1 -- 172.21.15.32:0/1516403219 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa304106670 msgr2=0x7fa3041a3e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:24.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.666+0000 7fa308d51640 1 --2- 172.21.15.32:0/1516403219 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa304106670 0x7fa3041a3e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.666+0000 7fa308d51640 1 -- 172.21.15.32:0/1516403219 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa3041a4360 con 0x7fa304101020 2024-04-25T00:33:24.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.666+0000 7fa308d51640 1 --2- 172.21.15.32:0/1516403219 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa304101020 0x7fa3041992e0 secure :-1 s=READY pgs=516 cs=0 l=1 rev1=1 crypto rx=0x7fa2f400d950 tx=0x7fa2f400de10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:24.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.666+0000 7fa2fa7fc640 1 -- 172.21.15.32:0/1516403219 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa2f4017030 con 0x7fa304101020 2024-04-25T00:33:24.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.666+0000 7fa30b7b6640 1 -- 172.21.15.32:0/1516403219 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa3041a4530 con 0x7fa304101020 2024-04-25T00:33:24.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.666+0000 7fa2fa7fc640 1 -- 172.21.15.32:0/1516403219 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa2f4004020 con 0x7fa304101020 2024-04-25T00:33:24.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.666+0000 7fa2fa7fc640 1 -- 172.21.15.32:0/1516403219 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa2f40047a0 con 0x7fa304101020 2024-04-25T00:33:24.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.666+0000 7fa30b7b6640 1 -- 172.21.15.32:0/1516403219 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa3041a4a40 con 0x7fa304101020 2024-04-25T00:33:24.671 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.667+0000 7fa30b7b6640 1 -- 172.21.15.32:0/1516403219 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa30404f500 con 0x7fa304101020 2024-04-25T00:33:24.671 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.668+0000 7fa2fa7fc640 1 -- 172.21.15.32:0/1516403219 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 24) ==== 138009+0+0 (secure 0 0 0) 0x7fa2f40041d0 con 0x7fa304101020 2024-04-25T00:33:24.671 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.669+0000 7fa2fa7fc640 1 --2- 172.21.15.32:0/1516403219 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa2f00a2e40 0x7fa2f00a5300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:24.672 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.669+0000 7fa309552640 1 --2- 172.21.15.32:0/1516403219 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa2f00a2e40 0x7fa2f00a5300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:24.672 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.669+0000 7fa2fa7fc640 1 -- 172.21.15.32:0/1516403219 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fa2f40cf420 con 0x7fa304101020 2024-04-25T00:33:24.672 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.669+0000 7fa309552640 1 --2- 172.21.15.32:0/1516403219 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa2f00a2e40 0x7fa2f00a5300 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7fa300009900 tx=0x7fa3000058d0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:24.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.673+0000 7fa2fa7fc640 1 -- 172.21.15.32:0/1516403219 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fa2f409bac0 con 0x7fa304101020 2024-04-25T00:33:24.757 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:24 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1102113382' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:33:24.757 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:24 smithi121 ceph-mon[32231]: scrub queued for path: / 2024-04-25T00:33:24.757 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:24 smithi121 ceph-mon[32231]: scrub summary: idle+waiting paths [/] 2024-04-25T00:33:24.757 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:24 smithi121 ceph-mon[32231]: scrub summary: active paths [/] 2024-04-25T00:33:24.757 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:24 smithi121 ceph-mon[32231]: scrub summary: idle+waiting paths [/] 2024-04-25T00:33:24.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:24 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1102113382' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:33:24.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:24 smithi130 ceph-mon[32525]: scrub queued for path: / 2024-04-25T00:33:24.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:24 smithi130 ceph-mon[32525]: scrub summary: idle+waiting paths [/] 2024-04-25T00:33:24.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:24 smithi130 ceph-mon[32525]: scrub summary: active paths [/] 2024-04-25T00:33:24.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:24 smithi130 ceph-mon[32525]: scrub summary: idle+waiting paths [/] 2024-04-25T00:33:24.885 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.883+0000 7fa30b7b6640 1 -- 172.21.15.32:0/1516403219 --> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] -- mgr_command(tid 0: {"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}) -- 0x7fa3040fea50 con 0x7fa2f00a2e40 2024-04-25T00:33:24.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.891+0000 7fa2fa7fc640 1 -- 172.21.15.32:0/1516403219 <== mgr.15483 v2:172.21.15.32:6800/3307874911 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+298 (secure 0 0 0) 0x7fa3040fea50 con 0x7fa2f00a2e40 2024-04-25T00:33:24.892 INFO:teuthology.orchestra.run.smithi032.stdout:{"fs": "cephfs", "subvol": null, "group": null, "path": "/", "rel_path": "/", "schedule": "1m", "retention": {"m": 6, "h": 3}, "start": "2024-04-25T00:00:00", "created": "2024-04-25T00:33:24", "first": null, "last": null, "last_pruned": null, "created_count": 0, "pruned_count": 0, "active": true} 2024-04-25T00:33:24.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.893+0000 7fa30b7b6640 1 -- 172.21.15.32:0/1516403219 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa2f00a2e40 msgr2=0x7fa2f00a5300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:24.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.893+0000 7fa30b7b6640 1 --2- 172.21.15.32:0/1516403219 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa2f00a2e40 0x7fa2f00a5300 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7fa300009900 tx=0x7fa3000058d0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.893+0000 7fa30b7b6640 1 -- 172.21.15.32:0/1516403219 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa304101020 msgr2=0x7fa3041992e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:24.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.893+0000 7fa30b7b6640 1 --2- 172.21.15.32:0/1516403219 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa304101020 0x7fa3041992e0 secure :-1 s=READY pgs=516 cs=0 l=1 rev1=1 crypto rx=0x7fa2f400d950 tx=0x7fa2f400de10 comp rx=0 tx=0).stop 2024-04-25T00:33:24.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.893+0000 7fa30b7b6640 1 -- 172.21.15.32:0/1516403219 shutdown_connections 2024-04-25T00:33:24.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.893+0000 7fa30b7b6640 1 --2- 172.21.15.32:0/1516403219 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa2f00a2e40 0x7fa2f00a5300 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.893+0000 7fa30b7b6640 1 --2- 172.21.15.32:0/1516403219 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa304106670 0x7fa3041a3e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.893+0000 7fa30b7b6640 1 --2- 172.21.15.32:0/1516403219 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa304101020 0x7fa3041992e0 unknown :-1 s=CLOSED pgs=516 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.893+0000 7fa30b7b6640 1 --2- 172.21.15.32:0/1516403219 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa304100660 0x7fa304198da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:24.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.893+0000 7fa30b7b6640 1 -- 172.21.15.32:0/1516403219 >> 172.21.15.32:0/1516403219 conn(0x7fa3040769c0 msgr2=0x7fa3040fe340 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:24.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.893+0000 7fa30b7b6640 1 -- 172.21.15.32:0/1516403219 shutdown_connections 2024-04-25T00:33:24.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:24.893+0000 7fa30b7b6640 1 -- 172.21.15.32:0/1516403219 wait complete. 2024-04-25T00:33:24.904 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:24 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1102113382' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:33:24.904 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:24 smithi032 ceph-mon[29357]: scrub queued for path: / 2024-04-25T00:33:24.904 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:24 smithi032 ceph-mon[29357]: scrub summary: idle+waiting paths [/] 2024-04-25T00:33:24.904 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:24 smithi032 ceph-mon[29357]: scrub summary: active paths [/] 2024-04-25T00:33:24.905 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:24 smithi032 ceph-mon[29357]: scrub summary: idle+waiting paths [/] 2024-04-25T00:33:24.905 DEBUG:teuthology.orchestra.run.smithi032:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule list --fs=cephfs --path=/ --recursive=true' 2024-04-25T00:33:25.011 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.008+0000 7f7b6b370640 1 -- 172.21.15.32:0/814455654 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b64101000 msgr2=0x7f7b6410d510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:25.011 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.008+0000 7f7b6b370640 1 --2- 172.21.15.32:0/814455654 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b64101000 0x7f7b6410d510 secure :-1 s=READY pgs=517 cs=0 l=1 rev1=1 crypto rx=0x7f7b60009cd0 tx=0x7f7b6002a8e0 comp rx=0 tx=0).stop 2024-04-25T00:33:25.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.008+0000 7f7b6b370640 1 -- 172.21.15.32:0/814455654 shutdown_connections 2024-04-25T00:33:25.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.008+0000 7f7b6b370640 1 --2- 172.21.15.32:0/814455654 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b64101000 0x7f7b6410d510 unknown :-1 s=CLOSED pgs=517 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:25.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.008+0000 7f7b6b370640 1 --2- 172.21.15.32:0/814455654 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7b64100640 0x7f7b64100ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:25.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.008+0000 7f7b6b370640 1 --2- 172.21.15.32:0/814455654 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7b64106650 0x7f7b64106a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:25.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.008+0000 7f7b6b370640 1 -- 172.21.15.32:0/814455654 >> 172.21.15.32:0/814455654 conn(0x7f7b640777c0 msgr2=0x7f7b64077bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:25.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.008+0000 7f7b6b370640 1 -- 172.21.15.32:0/814455654 shutdown_connections 2024-04-25T00:33:25.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.008+0000 7f7b6b370640 1 -- 172.21.15.32:0/814455654 wait complete. 2024-04-25T00:33:25.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.008+0000 7f7b6b370640 1 Processor -- start 2024-04-25T00:33:25.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.008+0000 7f7b6b370640 1 -- start start 2024-04-25T00:33:25.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.009+0000 7f7b6b370640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b64100640 0x7f7b640fd5a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:25.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.009+0000 7f7b6b370640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7b64101000 0x7f7b640fdae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:25.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.009+0000 7f7b6b370640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7b64106650 0x7f7b640fe020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:25.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.009+0000 7f7b6b370640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f7b64110a60 con 0x7f7b64100640 2024-04-25T00:33:25.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.009+0000 7f7b6b370640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f7b641108e0 con 0x7f7b64101000 2024-04-25T00:33:25.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.009+0000 7f7b6b370640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f7b64110be0 con 0x7f7b64106650 2024-04-25T00:33:25.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.009+0000 7f7b6910c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b64100640 0x7f7b640fd5a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:25.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.009+0000 7f7b6910c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b64100640 0x7f7b640fd5a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:58076/0 (socket says 172.21.15.32:58076) 2024-04-25T00:33:25.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.009+0000 7f7b6910c640 1 -- 172.21.15.32:0/3973547986 learned_addr learned my addr 172.21.15.32:0/3973547986 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:25.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.009+0000 7f7b6990d640 1 --2- 172.21.15.32:0/3973547986 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7b64106650 0x7f7b640fe020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:25.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.009+0000 7f7b6890b640 1 --2- 172.21.15.32:0/3973547986 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7b64101000 0x7f7b640fdae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:25.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.009+0000 7f7b6910c640 1 -- 172.21.15.32:0/3973547986 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7b64106650 msgr2=0x7f7b640fe020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:25.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.009+0000 7f7b6910c640 1 --2- 172.21.15.32:0/3973547986 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7b64106650 0x7f7b640fe020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:25.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.009+0000 7f7b6910c640 1 -- 172.21.15.32:0/3973547986 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7b64101000 msgr2=0x7f7b640fdae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:25.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.009+0000 7f7b6910c640 1 --2- 172.21.15.32:0/3973547986 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7b64101000 0x7f7b640fdae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:25.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.009+0000 7f7b6910c640 1 -- 172.21.15.32:0/3973547986 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7b641a64b0 con 0x7f7b64100640 2024-04-25T00:33:25.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.009+0000 7f7b6910c640 1 --2- 172.21.15.32:0/3973547986 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b64100640 0x7f7b640fd5a0 secure :-1 s=READY pgs=518 cs=0 l=1 rev1=1 crypto rx=0x7f7b5c00e9f0 tx=0x7f7b5c00edf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:25.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.009+0000 7f7b5a7fc640 1 -- 172.21.15.32:0/3973547986 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b5c00cd80 con 0x7f7b64100640 2024-04-25T00:33:25.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.009+0000 7f7b5a7fc640 1 -- 172.21.15.32:0/3973547986 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7b5c005500 con 0x7f7b64100640 2024-04-25T00:33:25.014 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.009+0000 7f7b6b370640 1 -- 172.21.15.32:0/3973547986 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7b641a67a0 con 0x7f7b64100640 2024-04-25T00:33:25.014 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.009+0000 7f7b5a7fc640 1 -- 172.21.15.32:0/3973547986 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b5c010690 con 0x7f7b64100640 2024-04-25T00:33:25.014 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.009+0000 7f7b6b370640 1 -- 172.21.15.32:0/3973547986 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7b641a6ca0 con 0x7f7b64100640 2024-04-25T00:33:25.015 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.010+0000 7f7b6b370640 1 -- 172.21.15.32:0/3973547986 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7b6404f500 con 0x7f7b64100640 2024-04-25T00:33:25.015 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.012+0000 7f7b5a7fc640 1 -- 172.21.15.32:0/3973547986 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 24) ==== 138009+0+0 (secure 0 0 0) 0x7f7b5c0046c0 con 0x7f7b64100640 2024-04-25T00:33:25.015 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.012+0000 7f7b5a7fc640 1 --2- 172.21.15.32:0/3973547986 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7b500a2e40 0x7f7b500a5300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:25.015 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.012+0000 7f7b6890b640 1 --2- 172.21.15.32:0/3973547986 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7b500a2e40 0x7f7b500a5300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:25.015 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.012+0000 7f7b5a7fc640 1 -- 172.21.15.32:0/3973547986 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f7b5c0cff20 con 0x7f7b64100640 2024-04-25T00:33:25.015 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.013+0000 7f7b6890b640 1 --2- 172.21.15.32:0/3973547986 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7b500a2e40 0x7f7b500a5300 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f7b640ff030 tx=0x7f7b54005f90 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:25.017 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.016+0000 7f7b5a7fc640 1 -- 172.21.15.32:0/3973547986 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f7b5c053740 con 0x7f7b64100640 2024-04-25T00:33:25.223 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.221+0000 7f7b6b370640 1 -- 172.21.15.32:0/3973547986 --> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] -- mgr_command(tid 0: {"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}) -- 0x7f7b64109c30 con 0x7f7b500a2e40 2024-04-25T00:33:25.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.228+0000 7f7b5a7fc640 1 -- 172.21.15.32:0/3973547986 <== mgr.15483 v2:172.21.15.32:6800/3307874911 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+10 (secure 0 0 0) 0x7f7b5401b3f0 con 0x7f7b500a2e40 2024-04-25T00:33:25.229 INFO:teuthology.orchestra.run.smithi032.stdout:/ 1m 6m3h 2024-04-25T00:33:25.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.230+0000 7f7b6b370640 1 -- 172.21.15.32:0/3973547986 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7b500a2e40 msgr2=0x7f7b500a5300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:25.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.230+0000 7f7b6b370640 1 --2- 172.21.15.32:0/3973547986 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7b500a2e40 0x7f7b500a5300 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f7b640ff030 tx=0x7f7b54005f90 comp rx=0 tx=0).stop 2024-04-25T00:33:25.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.230+0000 7f7b6b370640 1 -- 172.21.15.32:0/3973547986 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b64100640 msgr2=0x7f7b640fd5a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:25.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.230+0000 7f7b6b370640 1 --2- 172.21.15.32:0/3973547986 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b64100640 0x7f7b640fd5a0 secure :-1 s=READY pgs=518 cs=0 l=1 rev1=1 crypto rx=0x7f7b5c00e9f0 tx=0x7f7b5c00edf0 comp rx=0 tx=0).stop 2024-04-25T00:33:25.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.230+0000 7f7b6b370640 1 -- 172.21.15.32:0/3973547986 shutdown_connections 2024-04-25T00:33:25.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.230+0000 7f7b6b370640 1 --2- 172.21.15.32:0/3973547986 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7b500a2e40 0x7f7b500a5300 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:25.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.230+0000 7f7b6b370640 1 --2- 172.21.15.32:0/3973547986 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7b64106650 0x7f7b640fe020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:25.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.230+0000 7f7b6b370640 1 --2- 172.21.15.32:0/3973547986 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7b64101000 0x7f7b640fdae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:25.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.230+0000 7f7b6b370640 1 --2- 172.21.15.32:0/3973547986 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b64100640 0x7f7b640fd5a0 unknown :-1 s=CLOSED pgs=518 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:25.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.230+0000 7f7b6b370640 1 -- 172.21.15.32:0/3973547986 >> 172.21.15.32:0/3973547986 conn(0x7f7b640777c0 msgr2=0x7f7b6406a1f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:25.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.230+0000 7f7b6b370640 1 -- 172.21.15.32:0/3973547986 shutdown_connections 2024-04-25T00:33:25.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:25.230+0000 7f7b6b370640 1 -- 172.21.15.32:0/3973547986 wait complete. 2024-04-25T00:33:25.242 DEBUG:teuthology.orchestra.run.smithi032:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'date +%s > START_TIME' 2024-04-25T00:33:25.271 INFO:teuthology.run_tasks:Running task full_sequential_finally... 2024-04-25T00:33:25.285 INFO:teuthology.run_tasks:Running task workunit... 2024-04-25T00:33:25.299 INFO:tasks.workunit:Pulling workunits from ref d2c45d977142e024a59b68a8f1dce8f0b9840551 2024-04-25T00:33:25.299 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-04-25T00:33:25.299 INFO:tasks.workunit:timeout=3h 2024-04-25T00:33:25.300 INFO:tasks.workunit:cleanup=True 2024-04-25T00:33:25.300 DEBUG:teuthology.orchestra.run.smithi032:> stat -- /home/ubuntu/cephtest/mnt.0 2024-04-25T00:33:25.327 INFO:teuthology.orchestra.run.smithi032.stdout: File: /home/ubuntu/cephtest/mnt.0 2024-04-25T00:33:25.327 INFO:teuthology.orchestra.run.smithi032.stdout: Size: 0 Blocks: 0 IO Block: 65536 directory 2024-04-25T00:33:25.327 INFO:teuthology.orchestra.run.smithi032.stdout:Device: 73h/115d Inode: 1099511628283 Links: 2 2024-04-25T00:33:25.327 INFO:teuthology.orchestra.run.smithi032.stdout:Access: (1777/drwxrwxrwt) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-25T00:33:25.327 INFO:teuthology.orchestra.run.smithi032.stdout:Context: system_u:object_r:unlabeled_t:s0 2024-04-25T00:33:25.327 INFO:teuthology.orchestra.run.smithi032.stdout:Access: 2024-04-25 00:31:16.401405329 +0000 2024-04-25T00:33:25.327 INFO:teuthology.orchestra.run.smithi032.stdout:Modify: 2024-04-25 00:31:16.401405329 +0000 2024-04-25T00:33:25.328 INFO:teuthology.orchestra.run.smithi032.stdout:Change: 2024-04-25 00:33:15.721121620 +0000 2024-04-25T00:33:25.328 INFO:teuthology.orchestra.run.smithi032.stdout: Birth: 2024-04-25 00:31:16.401405329 +0000 2024-04-25T00:33:25.328 INFO:tasks.workunit:Did not need to create dir /home/ubuntu/cephtest/mnt.0 2024-04-25T00:33:25.328 DEBUG:teuthology.orchestra.run.smithi032:> cd -- /home/ubuntu/cephtest/mnt.0 && sudo install -d -m 0755 --owner=ubuntu -- client.0 2024-04-25T00:33:25.399 DEBUG:teuthology.orchestra.run.smithi032:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 && git checkout d2c45d977142e024a59b68a8f1dce8f0b9840551 2024-04-25T00:33:25.455 INFO:tasks.workunit.client.0.smithi032.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-04-25T00:33:25.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:25 smithi121 ceph-mon[32231]: pgmap v6: 129 pgs: 129 active+clean; 1.1 MiB data, 373 MiB used, 1.0 TiB / 1.0 TiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-25T00:33:25.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:25 smithi121 ceph-mon[32231]: from='client.15687 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule add", "fs": "cephfs", "path": "/", "snap_schedule": "1m", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:33:25.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:25 smithi121 ceph-mon[32231]: from='client.15723 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule retention add", "fs": "cephfs", "path": "/", "retention_spec_or_period": "6m3h", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:33:25.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:25 smithi130 ceph-mon[32525]: pgmap v6: 129 pgs: 129 active+clean; 1.1 MiB data, 373 MiB used, 1.0 TiB / 1.0 TiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-25T00:33:25.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:25 smithi130 ceph-mon[32525]: from='client.15687 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule add", "fs": "cephfs", "path": "/", "snap_schedule": "1m", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:33:25.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:25 smithi130 ceph-mon[32525]: from='client.15723 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule retention add", "fs": "cephfs", "path": "/", "retention_spec_or_period": "6m3h", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:33:25.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:25 smithi032 ceph-mon[29357]: pgmap v6: 129 pgs: 129 active+clean; 1.1 MiB data, 373 MiB used, 1.0 TiB / 1.0 TiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-25T00:33:25.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:25 smithi032 ceph-mon[29357]: from='client.15687 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule add", "fs": "cephfs", "path": "/", "snap_schedule": "1m", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:33:25.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:25 smithi032 ceph-mon[29357]: from='client.15723 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule retention add", "fs": "cephfs", "path": "/", "retention_spec_or_period": "6m3h", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:33:26.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:26 smithi121 ceph-mon[32231]: from='client.15729 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:33:26.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:26 smithi121 ceph-mon[32231]: from='client.15735 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:33:26.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:26 smithi130 ceph-mon[32525]: from='client.15729 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:33:26.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:26 smithi130 ceph-mon[32525]: from='client.15735 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:33:26.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:26 smithi032 ceph-mon[29357]: from='client.15729 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:33:26.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:26 smithi032 ceph-mon[29357]: from='client.15735 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:33:27.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:27 smithi121 ceph-mon[32231]: pgmap v7: 129 pgs: 129 active+clean; 1.1 MiB data, 377 MiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 292 B/s wr, 22 op/s 2024-04-25T00:33:27.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:27 smithi130 ceph-mon[32525]: pgmap v7: 129 pgs: 129 active+clean; 1.1 MiB data, 377 MiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 292 B/s wr, 22 op/s 2024-04-25T00:33:27.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:27 smithi032 ceph-mon[29357]: pgmap v7: 129 pgs: 129 active+clean; 1.1 MiB data, 377 MiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 292 B/s wr, 22 op/s 2024-04-25T00:33:29.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:29 smithi121 ceph-mon[32231]: pgmap v8: 129 pgs: 129 active+clean; 1.1 MiB data, 377 MiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 227 B/s wr, 17 op/s 2024-04-25T00:33:29.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:29 smithi130 ceph-mon[32525]: pgmap v8: 129 pgs: 129 active+clean; 1.1 MiB data, 377 MiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 227 B/s wr, 17 op/s 2024-04-25T00:33:29.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:29 smithi032 ceph-mon[29357]: pgmap v8: 129 pgs: 129 active+clean; 1.1 MiB data, 377 MiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 227 B/s wr, 17 op/s 2024-04-25T00:33:31.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:31 smithi121 ceph-mon[32231]: pgmap v9: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 84 KiB/s wr, 52 op/s 2024-04-25T00:33:31.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:31 smithi130 ceph-mon[32525]: pgmap v9: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 84 KiB/s wr, 52 op/s 2024-04-25T00:33:31.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:31 smithi032 ceph-mon[29357]: pgmap v9: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 84 KiB/s wr, 52 op/s 2024-04-25T00:33:33.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:33 smithi121 ceph-mon[32231]: pgmap v10: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 77 KiB/s wr, 48 op/s 2024-04-25T00:33:33.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:33 smithi130 ceph-mon[32525]: pgmap v10: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 77 KiB/s wr, 48 op/s 2024-04-25T00:33:33.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:33 smithi032 ceph-mon[29357]: pgmap v10: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 77 KiB/s wr, 48 op/s 2024-04-25T00:33:35.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:35 smithi121 ceph-mon[32231]: pgmap v11: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 77 KiB/s wr, 49 op/s 2024-04-25T00:33:35.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:35 smithi130 ceph-mon[32525]: pgmap v11: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 77 KiB/s wr, 49 op/s 2024-04-25T00:33:35.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:35 smithi032 ceph-mon[29357]: pgmap v11: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 77 KiB/s wr, 49 op/s 2024-04-25T00:33:37.801 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:37 smithi032 ceph-mon[29357]: pgmap v12: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 77 KiB/s wr, 50 op/s 2024-04-25T00:33:37.801 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:37 smithi032 ceph-mon[29357]: scrub summary: idle 2024-04-25T00:33:37.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:37 smithi121 ceph-mon[32231]: pgmap v12: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 77 KiB/s wr, 50 op/s 2024-04-25T00:33:37.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:37 smithi121 ceph-mon[32231]: scrub summary: idle 2024-04-25T00:33:37.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:37 smithi130 ceph-mon[32525]: pgmap v12: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 77 KiB/s wr, 50 op/s 2024-04-25T00:33:37.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:37 smithi130 ceph-mon[32525]: scrub summary: idle 2024-04-25T00:33:39.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:39 smithi121 ceph-mon[32231]: pgmap v13: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 77 KiB/s wr, 37 op/s 2024-04-25T00:33:39.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:39 smithi130 ceph-mon[32525]: pgmap v13: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 77 KiB/s wr, 37 op/s 2024-04-25T00:33:39.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:39 smithi032 ceph-mon[29357]: pgmap v13: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 77 KiB/s wr, 37 op/s 2024-04-25T00:33:41.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:41 smithi121 ceph-mon[32231]: pgmap v14: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 77 KiB/s wr, 39 op/s 2024-04-25T00:33:41.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:41 smithi130 ceph-mon[32525]: pgmap v14: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 77 KiB/s wr, 39 op/s 2024-04-25T00:33:41.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:41 smithi032 ceph-mon[29357]: pgmap v14: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 77 KiB/s wr, 39 op/s 2024-04-25T00:33:43.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:43 smithi121 ceph-mon[32231]: pgmap v15: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:33:43.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:43 smithi130 ceph-mon[32525]: pgmap v15: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:33:43.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:43 smithi032 ceph-mon[29357]: pgmap v15: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:33:45.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:45 smithi121 ceph-mon[32231]: pgmap v16: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:33:45.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:45 smithi130 ceph-mon[32525]: pgmap v16: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:33:45.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:45 smithi032 ceph-mon[29357]: pgmap v16: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:33:47.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:47 smithi121 ceph-mon[32231]: pgmap v17: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:33:47.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:47 smithi130 ceph-mon[32525]: pgmap v17: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:33:47.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:47 smithi032 ceph-mon[29357]: pgmap v17: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:33:49.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:49 smithi121 ceph-mon[32231]: pgmap v18: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:33:49.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:49 smithi130 ceph-mon[32525]: pgmap v18: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:33:49.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:49 smithi032 ceph-mon[29357]: pgmap v18: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:33:51.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:51 smithi121 ceph-mon[32231]: pgmap v19: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 7 op/s 2024-04-25T00:33:51.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:51 smithi130 ceph-mon[32525]: pgmap v19: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 7 op/s 2024-04-25T00:33:51.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:51 smithi032 ceph-mon[29357]: pgmap v19: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 7 op/s 2024-04-25T00:33:53.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:53 smithi121 ceph-mon[32231]: pgmap v20: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:33:53.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:53 smithi130 ceph-mon[32525]: pgmap v20: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:33:53.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:53 smithi032 ceph-mon[29357]: pgmap v20: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:33:54.021 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-25T00:33:54.022 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-25T00:33:54.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.148+0000 7f7b4c8fc640 1 -- 172.21.15.32:0/4066372631 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b481066b0 msgr2=0x7f7b48106a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.148+0000 7f7b4c8fc640 1 --2- 172.21.15.32:0/4066372631 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b481066b0 0x7f7b48106a90 secure :-1 s=READY pgs=519 cs=0 l=1 rev1=1 crypto rx=0x7f7b38009930 tx=0x7f7b3802a420 comp rx=0 tx=0).stop 2024-04-25T00:33:54.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.148+0000 7f7b4c8fc640 1 -- 172.21.15.32:0/4066372631 shutdown_connections 2024-04-25T00:33:54.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.148+0000 7f7b4c8fc640 1 --2- 172.21.15.32:0/4066372631 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7b48100fd0 0x7f7b4810d4e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.148+0000 7f7b4c8fc640 1 --2- 172.21.15.32:0/4066372631 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7b48100610 0x7f7b48100a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.148+0000 7f7b4c8fc640 1 --2- 172.21.15.32:0/4066372631 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b481066b0 0x7f7b48106a90 unknown :-1 s=CLOSED pgs=519 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.148+0000 7f7b4c8fc640 1 -- 172.21.15.32:0/4066372631 >> 172.21.15.32:0/4066372631 conn(0x7f7b48077740 msgr2=0x7f7b48077b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:54.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.148+0000 7f7b4c8fc640 1 -- 172.21.15.32:0/4066372631 shutdown_connections 2024-04-25T00:33:54.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.148+0000 7f7b4c8fc640 1 -- 172.21.15.32:0/4066372631 wait complete. 2024-04-25T00:33:54.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.148+0000 7f7b4c8fc640 1 Processor -- start 2024-04-25T00:33:54.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.149+0000 7f7b4c8fc640 1 -- start start 2024-04-25T00:33:54.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.149+0000 7f7b4c8fc640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b48100610 0x7f7b48196f70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.149+0000 7f7b4c8fc640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7b48100fd0 0x7f7b481974b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.149+0000 7f7b4c8fc640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7b481066b0 0x7f7b4819b830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.149+0000 7f7b4c8fc640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f7b48110a30 con 0x7f7b48100610 2024-04-25T00:33:54.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.149+0000 7f7b4c8fc640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f7b481108b0 con 0x7f7b481066b0 2024-04-25T00:33:54.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.149+0000 7f7b4c8fc640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f7b48110bb0 con 0x7f7b48100fd0 2024-04-25T00:33:54.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.149+0000 7f7b4659c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b48100610 0x7f7b48196f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.149+0000 7f7b4659c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b48100610 0x7f7b48196f70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53430/0 (socket says 172.21.15.32:53430) 2024-04-25T00:33:54.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.149+0000 7f7b4659c640 1 -- 172.21.15.32:0/1980269426 learned_addr learned my addr 172.21.15.32:0/1980269426 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:54.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.149+0000 7f7b46d9d640 1 --2- 172.21.15.32:0/1980269426 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7b481066b0 0x7f7b4819b830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.149+0000 7f7b45d9b640 1 --2- 172.21.15.32:0/1980269426 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7b48100fd0 0x7f7b481974b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.149+0000 7f7b4659c640 1 -- 172.21.15.32:0/1980269426 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7b48100fd0 msgr2=0x7f7b481974b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.149+0000 7f7b4659c640 1 --2- 172.21.15.32:0/1980269426 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7b48100fd0 0x7f7b481974b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.149+0000 7f7b4659c640 1 -- 172.21.15.32:0/1980269426 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7b481066b0 msgr2=0x7f7b4819b830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.149+0000 7f7b4659c640 1 --2- 172.21.15.32:0/1980269426 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7b481066b0 0x7f7b4819b830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.149+0000 7f7b4659c640 1 -- 172.21.15.32:0/1980269426 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7b4819bf60 con 0x7f7b48100610 2024-04-25T00:33:54.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.149+0000 7f7b46d9d640 1 --2- 172.21.15.32:0/1980269426 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7b481066b0 0x7f7b4819b830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:54.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.149+0000 7f7b4659c640 1 --2- 172.21.15.32:0/1980269426 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b48100610 0x7f7b48196f70 secure :-1 s=READY pgs=520 cs=0 l=1 rev1=1 crypto rx=0x7f7b38005940 tx=0x7f7b380042f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.150+0000 7f7b377fe640 1 -- 172.21.15.32:0/1980269426 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b3803f030 con 0x7f7b48100610 2024-04-25T00:33:54.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.150+0000 7f7b4c8fc640 1 -- 172.21.15.32:0/1980269426 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7b4819c1f0 con 0x7f7b48100610 2024-04-25T00:33:54.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.150+0000 7f7b377fe640 1 -- 172.21.15.32:0/1980269426 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7b38003ce0 con 0x7f7b48100610 2024-04-25T00:33:54.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.150+0000 7f7b377fe640 1 -- 172.21.15.32:0/1980269426 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b38035070 con 0x7f7b48100610 2024-04-25T00:33:54.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.150+0000 7f7b4c8fc640 1 -- 172.21.15.32:0/1980269426 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7b480688d0 con 0x7f7b48100610 2024-04-25T00:33:54.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.152+0000 7f7b377fe640 1 -- 172.21.15.32:0/1980269426 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 24) ==== 138009+0+0 (secure 0 0 0) 0x7f7b38042030 con 0x7f7b48100610 2024-04-25T00:33:54.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.152+0000 7f7b4f60a740 1 Processor -- start 2024-04-25T00:33:54.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.153+0000 7f7b4f60a740 1 -- start start 2024-04-25T00:33:54.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.153+0000 7f7b4f60a740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954a360 0x55bfa954a740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.153+0000 7f7b4f60a740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfa954ad10 0x55bfa95594f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.153+0000 7f7b4f60a740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55bfa9559a30 0x55bfa955be20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.153+0000 7f7b377fe640 1 --2- 172.21.15.32:0/1980269426 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7b300a2d90 0x7f7b300a5250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.153+0000 7f7b4f60a740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55bfa94af6b0 con 0x55bfa954a360 2024-04-25T00:33:54.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.153+0000 7f7b4f60a740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55bfa9457880 con 0x55bfa9559a30 2024-04-25T00:33:54.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.153+0000 7f7b4f60a740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55bfa94ab670 con 0x55bfa954ad10 2024-04-25T00:33:54.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.153+0000 7f7b46d9d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954a360 0x55bfa954a740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.153+0000 7f7b46d9d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954a360 0x55bfa954a740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53438/0 (socket says 172.21.15.32:53438) 2024-04-25T00:33:54.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.153+0000 7f7b46d9d640 1 -- 172.21.15.32:0/855737237 learned_addr learned my addr 172.21.15.32:0/855737237 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:54.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.153+0000 7f7b4659c640 1 --2- 172.21.15.32:0/855737237 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfa954ad10 0x55bfa95594f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.153+0000 7f7b46d9d640 1 --2- 172.21.15.32:0/1980269426 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7b300a2d90 0x7f7b300a5250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.153+0000 7f7b377fe640 1 -- 172.21.15.32:0/1980269426 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f7b380eff20 con 0x7f7b48100610 2024-04-25T00:33:54.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.153+0000 7f7b45d9b640 1 --2- 172.21.15.32:0/855737237 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55bfa9559a30 0x55bfa955be20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.153+0000 7f7b46d9d640 1 -- 172.21.15.32:0/855737237 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfa954ad10 msgr2=0x55bfa95594f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.153+0000 7f7b46d9d640 1 --2- 172.21.15.32:0/855737237 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfa954ad10 0x55bfa95594f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.153+0000 7f7b46d9d640 1 -- 172.21.15.32:0/855737237 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55bfa9559a30 msgr2=0x55bfa955be20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.153+0000 7f7b46d9d640 1 --2- 172.21.15.32:0/855737237 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55bfa9559a30 0x55bfa955be20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.153+0000 7f7b46d9d640 1 -- 172.21.15.32:0/855737237 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55bfa928d7d0 con 0x55bfa954a360 2024-04-25T00:33:54.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.153+0000 7f7b4659c640 1 --2- 172.21.15.32:0/855737237 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfa954ad10 0x55bfa95594f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:54.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.154+0000 7f7b46d9d640 1 --2- 172.21.15.32:0/855737237 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954a360 0x55bfa954a740 secure :-1 s=READY pgs=521 cs=0 l=1 rev1=1 crypto rx=0x7f7b40012cf0 tx=0x7f7b40010740 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d0d3c27c39bf1289 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.154+0000 7f7b46d9d640 1 --2- 172.21.15.32:0/1980269426 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7b300a2d90 0x7f7b300a5250 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f7b40010c40 tx=0x7f7b40017000 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.154+0000 7f7b34ff9640 1 -- 172.21.15.32:0/855737237 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b40022070 con 0x55bfa954a360 2024-04-25T00:33:54.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.154+0000 7f7b34ff9640 1 -- 172.21.15.32:0/855737237 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7b40009cb0 con 0x55bfa954a360 2024-04-25T00:33:54.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.154+0000 7f7b34ff9640 1 -- 172.21.15.32:0/855737237 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b4000cdd0 con 0x55bfa954a360 2024-04-25T00:33:54.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.155+0000 7f7b4f60a740 1 -- 172.21.15.32:0/855737237 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954a360 msgr2=0x55bfa954a740 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.155+0000 7f7b4f60a740 1 --2- 172.21.15.32:0/855737237 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954a360 0x55bfa954a740 secure :-1 s=READY pgs=521 cs=0 l=1 rev1=1 crypto rx=0x7f7b40012cf0 tx=0x7f7b40010740 comp rx=0 tx=0).stop 2024-04-25T00:33:54.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.155+0000 7f7b4f60a740 1 -- 172.21.15.32:0/855737237 shutdown_connections 2024-04-25T00:33:54.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.155+0000 7f7b4f60a740 1 --2- 172.21.15.32:0/855737237 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55bfa9559a30 0x55bfa955be20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.155+0000 7f7b4f60a740 1 --2- 172.21.15.32:0/855737237 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfa954ad10 0x55bfa95594f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.155+0000 7f7b4f60a740 1 --2- 172.21.15.32:0/855737237 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954a360 0x55bfa954a740 unknown :-1 s=CLOSED pgs=521 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.155+0000 7f7b4f60a740 1 -- 172.21.15.32:0/855737237 >> 172.21.15.32:0/855737237 conn(0x55bfa9546950 msgr2=0x55bfa9546d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:54.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.155+0000 7f7b4f60a740 1 -- 172.21.15.32:0/855737237 shutdown_connections 2024-04-25T00:33:54.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.155+0000 7f7b4f60a740 1 -- 172.21.15.32:0/855737237 wait complete. 2024-04-25T00:33:54.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.155+0000 7f7b4f60a740 1 -- start start 2024-04-25T00:33:54.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.155+0000 7f7b4f60a740 1 Processor -- start 2024-04-25T00:33:54.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.156+0000 7f7b4f60a740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954a360 0x55bfa9556a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.156+0000 7f7b4f60a740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55bfa954ad10 0x55bfa9556fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.156+0000 7f7b4f60a740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfa9559a30 0x55bfa9557510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.156+0000 7f7b4f60a740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55bfa955c790 con 0x55bfa954a360 2024-04-25T00:33:54.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.156+0000 7f7b4f60a740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55bfa946af50 con 0x55bfa954ad10 2024-04-25T00:33:54.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.156+0000 7f7b4f60a740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55bfa955c910 con 0x55bfa9559a30 2024-04-25T00:33:54.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.156+0000 7f7b4f60a740 10 client.0 _ll_register_callbacks cb 0x55bfa94a6000 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:33:54.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.156+0000 7f7b4659c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954a360 0x55bfa9556a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.156+0000 7f7b4659c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954a360 0x55bfa9556a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53444/0 (socket says 172.21.15.32:53444) 2024-04-25T00:33:54.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.156+0000 7f7b4659c640 1 -- 172.21.15.32:0/3983159811 learned_addr learned my addr 172.21.15.32:0/3983159811 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:54.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.156+0000 7f7b45d9b640 1 --2- 172.21.15.32:0/3983159811 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55bfa954ad10 0x55bfa9556fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.156+0000 7f7b46d9d640 1 --2- 172.21.15.32:0/3983159811 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfa9559a30 0x55bfa9557510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.156+0000 7f7b4659c640 1 -- 172.21.15.32:0/3983159811 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfa9559a30 msgr2=0x55bfa9557510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.156+0000 7f7b4659c640 1 --2- 172.21.15.32:0/3983159811 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfa9559a30 0x55bfa9557510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.156+0000 7f7b4659c640 1 -- 172.21.15.32:0/3983159811 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55bfa954ad10 msgr2=0x55bfa9556fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.156+0000 7f7b4659c640 1 --2- 172.21.15.32:0/3983159811 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55bfa954ad10 0x55bfa9556fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.156+0000 7f7b4659c640 1 -- 172.21.15.32:0/3983159811 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55bfa91c33f0 con 0x55bfa954a360 2024-04-25T00:33:54.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.156+0000 7f7b46d9d640 1 --2- 172.21.15.32:0/3983159811 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfa9559a30 0x55bfa9557510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:33:54.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.156+0000 7f7b4659c640 1 --2- 172.21.15.32:0/3983159811 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954a360 0x55bfa9556a90 secure :-1 s=READY pgs=522 cs=0 l=1 rev1=1 crypto rx=0x7f7b380ed230 tx=0x7f7b38041d10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4e1539b8eeadb12b server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.156+0000 7f7b2dffb640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:33:54.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.156+0000 7f7b2dffb640 1 -- 172.21.15.32:0/3983159811 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b38049040 con 0x55bfa954a360 2024-04-25T00:33:54.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.156+0000 7f7b4f60a740 1 -- 172.21.15.32:0/3983159811 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f7b40009cb0 con 0x55bfa954a360 2024-04-25T00:33:54.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.156+0000 7f7b2dffb640 1 -- 172.21.15.32:0/3983159811 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f7b38041d80 con 0x55bfa954a360 2024-04-25T00:33:54.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.157+0000 7f7b2dffb640 1 -- 172.21.15.32:0/3983159811 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f7b380bb820 con 0x55bfa954a360 2024-04-25T00:33:54.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.157+0000 7f7b4f60a740 10 client.15747 fetch_fsmap learned FSMap version 25 2024-04-25T00:33:54.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.157+0000 7f7b4f60a740 1 -- 172.21.15.32:0/3983159811 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x55bfa91830f0 con 0x55bfa954a360 2024-04-25T00:33:54.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.157+0000 7f7b2dffb640 1 -- 172.21.15.32:0/3983159811 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7b40009cb0 con 0x55bfa954a360 2024-04-25T00:33:54.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.157+0000 7f7b2dffb640 1 -- 172.21.15.32:0/3983159811 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7f7b380b8530 con 0x55bfa954a360 2024-04-25T00:33:54.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.157+0000 7f7b4f60a740 10 client.15747 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:33:54.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.157+0000 7f7b4f60a740 10 client.15747 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:33:54.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.157+0000 7f7b4f60a740 20 client.15747 populate_metadata read hostname 'smithi032' 2024-04-25T00:33:54.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.157+0000 7f7b4f60a740 1 --2- 172.21.15.32:0/3983159811 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55bfa9569d40 0x55bfa956c1e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.157+0000 7f7b4f60a740 4 client.15747 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:33:54.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.157+0000 7f7b4f60a740 1 -- 172.21.15.32:0/3983159811 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55bfa94bcbf0 con 0x55bfa9569d40 2024-04-25T00:33:54.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.157+0000 7f7b45d9b640 1 --2- 172.21.15.32:0/3983159811 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55bfa9569d40 0x55bfa956c1e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.158+0000 7f7b45d9b640 1 --2- 172.21.15.32:0/3983159811 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55bfa9569d40 0x55bfa956c1e0 crc :-1 s=READY pgs=44 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=773a585f52182ca6 server_cookie=7a1ec7128cc5c85b in_seq=0 out_seq=0 2024-04-25T00:33:54.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.158+0000 7f7b2dffb640 10 client.15747 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:54.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.159+0000 7f7b2dffb640 1 -- 172.21.15.32:0/3983159811 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x55bfa94bcbf0 con 0x55bfa9569d40 2024-04-25T00:33:54.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.159+0000 7f7b2dffb640 10 client.15747 handle_command_reply: tid=0 2024-04-25T00:33:54.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.159+0000 7f7b4f60a740 1 client.15747 shutdown 2024-04-25T00:33:54.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.159+0000 7f7b4f60a740 1 -- 172.21.15.32:0/3983159811 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954a360 msgr2=0x55bfa9556a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.159+0000 7f7b4f60a740 1 --2- 172.21.15.32:0/3983159811 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954a360 0x55bfa9556a90 secure :-1 s=READY pgs=522 cs=0 l=1 rev1=1 crypto rx=0x7f7b380ed230 tx=0x7f7b38041d10 comp rx=0 tx=0).stop 2024-04-25T00:33:54.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.159+0000 7f7b4f60a740 1 -- 172.21.15.32:0/3983159811 shutdown_connections 2024-04-25T00:33:54.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.159+0000 7f7b4f60a740 1 --2- 172.21.15.32:0/3983159811 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55bfa9569d40 0x55bfa956c1e0 crc :-1 s=READY pgs=44 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.159+0000 7f7b4f60a740 1 --2- 172.21.15.32:0/3983159811 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfa9559a30 0x55bfa9557510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.159+0000 7f7b4f60a740 1 --2- 172.21.15.32:0/3983159811 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55bfa954ad10 0x55bfa9556fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.159+0000 7f7b2dffb640 0 client.15747 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:54.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.159+0000 7f7b4f60a740 1 --2- 172.21.15.32:0/3983159811 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954a360 0x55bfa9556a90 unknown :-1 s=CLOSED pgs=522 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.159+0000 7f7b4f60a740 1 -- 172.21.15.32:0/3983159811 >> 172.21.15.32:0/3983159811 conn(0x55bfa9546950 msgr2=0x55bfa954c7c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:54.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.159+0000 7f7b4f60a740 1 -- 172.21.15.32:0/3983159811 shutdown_connections 2024-04-25T00:33:54.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.159+0000 7f7b4f60a740 1 -- 172.21.15.32:0/3983159811 wait complete. 2024-04-25T00:33:54.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.159+0000 7f7b4f60a740 20 client.15747 trim_cache size 0 max 16384 2024-04-25T00:33:54.181 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.179+0000 7f7b4f60a740 1 Processor -- start 2024-04-25T00:33:54.181 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.179+0000 7f7b4f60a740 1 -- start start 2024-04-25T00:33:54.181 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.179+0000 7f7b4f60a740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfa954a360 0x55bfa95591c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.181 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.179+0000 7f7b4f60a740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954ad10 0x55bfa95559d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.181 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.179+0000 7f7b4f60a740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55bfa9559a30 0x55bfa9551db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.181 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.179+0000 7f7b4f60a740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f7b481af3d0 con 0x55bfa954ad10 2024-04-25T00:33:54.181 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.179+0000 7f7b4f60a740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f7b481acca0 con 0x55bfa9559a30 2024-04-25T00:33:54.181 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.179+0000 7f7b4f60a740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f7b481aaaf0 con 0x55bfa954a360 2024-04-25T00:33:54.181 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.179+0000 7f7b45d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954ad10 0x55bfa95559d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.181 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.179+0000 7f7b45d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954ad10 0x55bfa95559d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53446/0 (socket says 172.21.15.32:53446) 2024-04-25T00:33:54.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.179+0000 7f7b45d9b640 1 -- 172.21.15.32:0/2743837260 learned_addr learned my addr 172.21.15.32:0/2743837260 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:54.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.179+0000 7f7b46d9d640 1 --2- 172.21.15.32:0/2743837260 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55bfa9559a30 0x55bfa9551db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.179+0000 7f7b4659c640 1 --2- 172.21.15.32:0/2743837260 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfa954a360 0x55bfa95591c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.179+0000 7f7b45d9b640 1 -- 172.21.15.32:0/2743837260 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfa954a360 msgr2=0x55bfa95591c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.179+0000 7f7b45d9b640 1 --2- 172.21.15.32:0/2743837260 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfa954a360 0x55bfa95591c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.179+0000 7f7b45d9b640 1 -- 172.21.15.32:0/2743837260 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55bfa9559a30 msgr2=0x55bfa9551db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.179+0000 7f7b45d9b640 1 --2- 172.21.15.32:0/2743837260 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55bfa9559a30 0x55bfa9551db0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.179+0000 7f7b45d9b640 1 -- 172.21.15.32:0/2743837260 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55bfa9565650 con 0x55bfa954ad10 2024-04-25T00:33:54.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.179+0000 7f7b46d9d640 1 --2- 172.21.15.32:0/2743837260 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55bfa9559a30 0x55bfa9551db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:54.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.179+0000 7f7b45d9b640 1 --2- 172.21.15.32:0/2743837260 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954ad10 0x55bfa95559d0 secure :-1 s=READY pgs=523 cs=0 l=1 rev1=1 crypto rx=0x7f7b480fea70 tx=0x7f7b3c00bd20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f93fa6639e8f16b6 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.180+0000 7f7b2ffff640 1 -- 172.21.15.32:0/2743837260 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b3c003d60 con 0x55bfa954ad10 2024-04-25T00:33:54.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.180+0000 7f7b2ffff640 1 -- 172.21.15.32:0/2743837260 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7b3c011440 con 0x55bfa954ad10 2024-04-25T00:33:54.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.180+0000 7f7b2ffff640 1 -- 172.21.15.32:0/2743837260 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b3c010690 con 0x55bfa954ad10 2024-04-25T00:33:54.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.180+0000 7f7b4f60a740 1 -- 172.21.15.32:0/2743837260 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954ad10 msgr2=0x55bfa95559d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.180+0000 7f7b4f60a740 1 --2- 172.21.15.32:0/2743837260 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954ad10 0x55bfa95559d0 secure :-1 s=READY pgs=523 cs=0 l=1 rev1=1 crypto rx=0x7f7b480fea70 tx=0x7f7b3c00bd20 comp rx=0 tx=0).stop 2024-04-25T00:33:54.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.180+0000 7f7b4f60a740 1 -- 172.21.15.32:0/2743837260 shutdown_connections 2024-04-25T00:33:54.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.180+0000 7f7b4f60a740 1 --2- 172.21.15.32:0/2743837260 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55bfa9559a30 0x55bfa9551db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.180+0000 7f7b4f60a740 1 --2- 172.21.15.32:0/2743837260 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954ad10 0x55bfa95559d0 unknown :-1 s=CLOSED pgs=523 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.180+0000 7f7b4f60a740 1 --2- 172.21.15.32:0/2743837260 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfa954a360 0x55bfa95591c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.180+0000 7f7b4f60a740 1 -- 172.21.15.32:0/2743837260 >> 172.21.15.32:0/2743837260 conn(0x55bfa9546950 msgr2=0x55bfa954c7c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:54.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.180+0000 7f7b4f60a740 1 -- 172.21.15.32:0/2743837260 shutdown_connections 2024-04-25T00:33:54.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.180+0000 7f7b4f60a740 1 -- 172.21.15.32:0/2743837260 wait complete. 2024-04-25T00:33:54.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.181+0000 7f7b4f60a740 1 -- start start 2024-04-25T00:33:54.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.181+0000 7f7b4f60a740 1 Processor -- start 2024-04-25T00:33:54.188 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.181+0000 7f7b4f60a740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfa954a360 0x55bfa955d7d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.188 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.181+0000 7f7b4f60a740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954ad10 0x55bfa955dd10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.188 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.181+0000 7f7b4f60a740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55bfa9559a30 0x55bfa955e250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.188 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.181+0000 7f7b4f60a740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f7b481ab160 con 0x55bfa954ad10 2024-04-25T00:33:54.188 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.181+0000 7f7b4f60a740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f7b481ac5d0 con 0x55bfa9559a30 2024-04-25T00:33:54.188 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.181+0000 7f7b4f60a740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f7b481ab830 con 0x55bfa954a360 2024-04-25T00:33:54.188 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.181+0000 7f7b4f60a740 10 client.0 _ll_register_callbacks cb 0x7f7b38003f20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:33:54.188 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.181+0000 7f7b45d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954ad10 0x55bfa955dd10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.188 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.181+0000 7f7b45d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954ad10 0x55bfa955dd10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53458/0 (socket says 172.21.15.32:53458) 2024-04-25T00:33:54.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.181+0000 7f7b45d9b640 1 -- 172.21.15.32:0/1121834444 learned_addr learned my addr 172.21.15.32:0/1121834444 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:54.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.181+0000 7f7b4659c640 1 --2- 172.21.15.32:0/1121834444 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfa954a360 0x55bfa955d7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.181+0000 7f7b46d9d640 1 --2- 172.21.15.32:0/1121834444 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55bfa9559a30 0x55bfa955e250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.181+0000 7f7b45d9b640 1 -- 172.21.15.32:0/1121834444 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfa954a360 msgr2=0x55bfa955d7d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.181+0000 7f7b45d9b640 1 --2- 172.21.15.32:0/1121834444 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfa954a360 0x55bfa955d7d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.181+0000 7f7b45d9b640 1 -- 172.21.15.32:0/1121834444 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55bfa9559a30 msgr2=0x55bfa955e250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.181+0000 7f7b45d9b640 1 --2- 172.21.15.32:0/1121834444 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55bfa9559a30 0x55bfa955e250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.181+0000 7f7b45d9b640 1 -- 172.21.15.32:0/1121834444 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55bfa95cf570 con 0x55bfa954ad10 2024-04-25T00:33:54.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.182+0000 7f7b2dffb640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:33:54.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.182+0000 7f7b45d9b640 1 --2- 172.21.15.32:0/1121834444 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954ad10 0x55bfa955dd10 secure :-1 s=READY pgs=524 cs=0 l=1 rev1=1 crypto rx=0x7f7b3c000c00 tx=0x7f7b3c0044b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8095154df5012056 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.182+0000 7f7b2dffb640 1 -- 172.21.15.32:0/1121834444 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b3c01d670 con 0x55bfa954ad10 2024-04-25T00:33:54.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.182+0000 7f7b4f60a740 1 -- 172.21.15.32:0/1121834444 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f7b3c011440 con 0x55bfa954ad10 2024-04-25T00:33:54.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.182+0000 7f7b2dffb640 1 -- 172.21.15.32:0/1121834444 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f7b3c01d810 con 0x55bfa954ad10 2024-04-25T00:33:54.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.182+0000 7f7b2dffb640 1 -- 172.21.15.32:0/1121834444 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f7b3c037050 con 0x55bfa954ad10 2024-04-25T00:33:54.190 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.182+0000 7f7b4f60a740 10 client.15753 fetch_fsmap learned FSMap version 25 2024-04-25T00:33:54.190 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.182+0000 7f7b4f60a740 1 -- 172.21.15.32:0/1121834444 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x55bfa95d0880 con 0x55bfa954ad10 2024-04-25T00:33:54.190 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.182+0000 7f7b2dffb640 1 -- 172.21.15.32:0/1121834444 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7b3c011440 con 0x55bfa954ad10 2024-04-25T00:33:54.190 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.182+0000 7f7b2dffb640 1 -- 172.21.15.32:0/1121834444 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7f7b3c0046c0 con 0x55bfa954ad10 2024-04-25T00:33:54.190 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.182+0000 7f7b4f60a740 10 client.15753 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:33:54.190 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.182+0000 7f7b4f60a740 10 client.15753 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:33:54.190 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.182+0000 7f7b4f60a740 20 client.15753 populate_metadata read hostname 'smithi032' 2024-04-25T00:33:54.190 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.182+0000 7f7b4f60a740 1 --2- 172.21.15.32:0/1121834444 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55bfa95d0f20 0x55bfa95d3340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.190 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.182+0000 7f7b4f60a740 4 client.15753 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-25T00:33:54.190 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.182+0000 7f7b4f60a740 1 -- 172.21.15.32:0/1121834444 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "scrub status"}) -- 0x55bfa94b6f40 con 0x55bfa95d0f20 2024-04-25T00:33:54.190 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.183+0000 7f7b4659c640 1 --2- 172.21.15.32:0/1121834444 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55bfa95d0f20 0x55bfa95d3340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.190 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.183+0000 7f7b4659c640 1 --2- 172.21.15.32:0/1121834444 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55bfa95d0f20 0x55bfa95d3340 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e35fd55ec3e6c42e server_cookie=c38d5e82de3d724c in_seq=0 out_seq=0 2024-04-25T00:33:54.190 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.183+0000 7f7b2dffb640 10 client.15753 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:54.191 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.183+0000 7f7b2dffb640 1 -- 172.21.15.32:0/1121834444 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+63 (crc 0 0 0) 0x55bfa94b6f40 con 0x55bfa95d0f20 2024-04-25T00:33:54.191 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.183+0000 7f7b2dffb640 10 client.15753 handle_command_reply: tid=0 2024-04-25T00:33:54.191 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.183+0000 7f7b4f60a740 1 client.15753 shutdown 2024-04-25T00:33:54.191 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.183+0000 7f7b4f60a740 1 -- 172.21.15.32:0/1121834444 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954ad10 msgr2=0x55bfa955dd10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.191 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.183+0000 7f7b4f60a740 1 --2- 172.21.15.32:0/1121834444 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954ad10 0x55bfa955dd10 secure :-1 s=READY pgs=524 cs=0 l=1 rev1=1 crypto rx=0x7f7b3c000c00 tx=0x7f7b3c0044b0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.191 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.183+0000 7f7b4f60a740 1 -- 172.21.15.32:0/1121834444 shutdown_connections 2024-04-25T00:33:54.191 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.183+0000 7f7b4f60a740 1 --2- 172.21.15.32:0/1121834444 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55bfa95d0f20 0x55bfa95d3340 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.191 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.183+0000 7f7b4f60a740 1 --2- 172.21.15.32:0/1121834444 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55bfa9559a30 0x55bfa955e250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.191 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.183+0000 7f7b4f60a740 1 --2- 172.21.15.32:0/1121834444 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55bfa954ad10 0x55bfa955dd10 unknown :-1 s=CLOSED pgs=524 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.191 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.183+0000 7f7b2dffb640 0 client.15753 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:54.191 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.183+0000 7f7b4f60a740 1 --2- 172.21.15.32:0/1121834444 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfa954a360 0x55bfa955d7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.191 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.183+0000 7f7b4f60a740 1 -- 172.21.15.32:0/1121834444 >> 172.21.15.32:0/1121834444 conn(0x55bfa9546950 msgr2=0x55bfa9556a50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:54.191 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.184+0000 7f7b4f60a740 1 -- 172.21.15.32:0/1121834444 shutdown_connections 2024-04-25T00:33:54.191 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.184+0000 7f7b4f60a740 1 -- 172.21.15.32:0/1121834444 wait complete. 2024-04-25T00:33:54.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.184+0000 7f7b4f60a740 20 client.15753 trim_cache size 0 max 16384 2024-04-25T00:33:54.192 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:33:54.192 INFO:teuthology.orchestra.run.smithi032.stdout: "status": "no active scrubs running", 2024-04-25T00:33:54.192 INFO:teuthology.orchestra.run.smithi032.stdout: "scrubs": {} 2024-04-25T00:33:54.192 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:33:54.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.184+0000 7f7b4c8fc640 1 -- 172.21.15.32:0/1980269426 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7b300a2d90 msgr2=0x7f7b300a5250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.184+0000 7f7b4c8fc640 1 --2- 172.21.15.32:0/1980269426 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7b300a2d90 0x7f7b300a5250 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f7b40010c40 tx=0x7f7b40017000 comp rx=0 tx=0).stop 2024-04-25T00:33:54.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.184+0000 7f7b4c8fc640 1 -- 172.21.15.32:0/1980269426 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b48100610 msgr2=0x7f7b48196f70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.184+0000 7f7b4c8fc640 1 --2- 172.21.15.32:0/1980269426 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b48100610 0x7f7b48196f70 secure :-1 s=READY pgs=520 cs=0 l=1 rev1=1 crypto rx=0x7f7b38005940 tx=0x7f7b380042f0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.184+0000 7f7b4c8fc640 1 -- 172.21.15.32:0/1980269426 shutdown_connections 2024-04-25T00:33:54.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.184+0000 7f7b4c8fc640 1 --2- 172.21.15.32:0/1980269426 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7b300a2d90 0x7f7b300a5250 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.184+0000 7f7b4c8fc640 1 --2- 172.21.15.32:0/1980269426 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7b481066b0 0x7f7b4819b830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.184+0000 7f7b4c8fc640 1 --2- 172.21.15.32:0/1980269426 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7b48100fd0 0x7f7b481974b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.184+0000 7f7b4c8fc640 1 --2- 172.21.15.32:0/1980269426 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b48100610 0x7f7b48196f70 unknown :-1 s=CLOSED pgs=520 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.184+0000 7f7b4c8fc640 1 -- 172.21.15.32:0/1980269426 >> 172.21.15.32:0/1980269426 conn(0x7f7b48077740 msgr2=0x7f7b480ff280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:54.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.184+0000 7f7b4c8fc640 1 -- 172.21.15.32:0/1980269426 shutdown_connections 2024-04-25T00:33:54.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.184+0000 7f7b4c8fc640 1 -- 172.21.15.32:0/1980269426 wait complete. 2024-04-25T00:33:54.196 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-25T00:33:54.196 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:33:54.301 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.299+0000 7f4815190640 1 -- 172.21.15.32:0/2712153413 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4810106630 msgr2=0x7f4810106a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.301 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.299+0000 7f4815190640 1 --2- 172.21.15.32:0/2712153413 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4810106630 0x7f4810106a10 secure :-1 s=READY pgs=525 cs=0 l=1 rev1=1 crypto rx=0x7f4800009930 tx=0x7f480002a420 comp rx=0 tx=0).stop 2024-04-25T00:33:54.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.299+0000 7f4815190640 1 -- 172.21.15.32:0/2712153413 shutdown_connections 2024-04-25T00:33:54.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.299+0000 7f4815190640 1 --2- 172.21.15.32:0/2712153413 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4810100fe0 0x7f481010d4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.299+0000 7f4815190640 1 --2- 172.21.15.32:0/2712153413 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4810100620 0x7f4810100aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.299+0000 7f4815190640 1 --2- 172.21.15.32:0/2712153413 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4810106630 0x7f4810106a10 unknown :-1 s=CLOSED pgs=525 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.299+0000 7f4815190640 1 -- 172.21.15.32:0/2712153413 >> 172.21.15.32:0/2712153413 conn(0x7f4810077740 msgr2=0x7f4810077b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:54.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.299+0000 7f4815190640 1 -- 172.21.15.32:0/2712153413 shutdown_connections 2024-04-25T00:33:54.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.299+0000 7f4815190640 1 -- 172.21.15.32:0/2712153413 wait complete. 2024-04-25T00:33:54.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.299+0000 7f4815190640 1 Processor -- start 2024-04-25T00:33:54.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.299+0000 7f4815190640 1 -- start start 2024-04-25T00:33:54.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.300+0000 7f4815190640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4810100620 0x7f4810197f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.300+0000 7f4815190640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4810100fe0 0x7f4810198490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.300+0000 7f4815190640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4810106630 0x7f481019c7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.300+0000 7f4815190640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f4810110a00 con 0x7f4810100620 2024-04-25T00:33:54.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.300+0000 7f4815190640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f4810110880 con 0x7f4810100fe0 2024-04-25T00:33:54.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.300+0000 7f4815190640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f4810110b80 con 0x7f4810106630 2024-04-25T00:33:54.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.300+0000 7f480ed9d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4810100620 0x7f4810197f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.300+0000 7f480ed9d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4810100620 0x7f4810197f50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53464/0 (socket says 172.21.15.32:53464) 2024-04-25T00:33:54.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.300+0000 7f480ed9d640 1 -- 172.21.15.32:0/4114945454 learned_addr learned my addr 172.21.15.32:0/4114945454 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:54.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.300+0000 7f4807fff640 1 --2- 172.21.15.32:0/4114945454 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4810100fe0 0x7f4810198490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.300+0000 7f480f59e640 1 --2- 172.21.15.32:0/4114945454 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4810106630 0x7f481019c7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.300+0000 7f480ed9d640 1 -- 172.21.15.32:0/4114945454 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4810106630 msgr2=0x7f481019c7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.300+0000 7f480ed9d640 1 --2- 172.21.15.32:0/4114945454 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4810106630 0x7f481019c7c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.300+0000 7f480ed9d640 1 -- 172.21.15.32:0/4114945454 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4810100fe0 msgr2=0x7f4810198490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.300+0000 7f480ed9d640 1 --2- 172.21.15.32:0/4114945454 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4810100fe0 0x7f4810198490 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.300+0000 7f480ed9d640 1 -- 172.21.15.32:0/4114945454 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f481019cef0 con 0x7f4810100620 2024-04-25T00:33:54.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.300+0000 7f480f59e640 1 --2- 172.21.15.32:0/4114945454 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4810106630 0x7f481019c7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:54.304 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.300+0000 7f480ed9d640 1 --2- 172.21.15.32:0/4114945454 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4810100620 0x7f4810197f50 secure :-1 s=READY pgs=526 cs=0 l=1 rev1=1 crypto rx=0x7f4800009a50 tx=0x7f4800004270 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.304 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.301+0000 7f480cd99640 1 -- 172.21.15.32:0/4114945454 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f480003f030 con 0x7f4810100620 2024-04-25T00:33:54.304 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.301+0000 7f4815190640 1 -- 172.21.15.32:0/4114945454 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f481019d180 con 0x7f4810100620 2024-04-25T00:33:54.304 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.301+0000 7f480cd99640 1 -- 172.21.15.32:0/4114945454 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4800003ad0 con 0x7f4810100620 2024-04-25T00:33:54.304 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.301+0000 7f480cd99640 1 -- 172.21.15.32:0/4114945454 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4800035070 con 0x7f4810100620 2024-04-25T00:33:54.304 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.301+0000 7f4815190640 1 -- 172.21.15.32:0/4114945454 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f48101a4a70 con 0x7f4810100620 2024-04-25T00:33:54.305 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.302+0000 7f4815190640 1 -- 172.21.15.32:0/4114945454 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4810068560 con 0x7f4810100620 2024-04-25T00:33:54.305 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.304+0000 7f480cd99640 1 -- 172.21.15.32:0/4114945454 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 24) ==== 138009+0+0 (secure 0 0 0) 0x7f4800042030 con 0x7f4810100620 2024-04-25T00:33:54.305 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.304+0000 7f480cd99640 1 --2- 172.21.15.32:0/4114945454 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f47fc0a2d90 0x7f47fc0a5250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.305 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.304+0000 7f4807fff640 1 --2- 172.21.15.32:0/4114945454 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f47fc0a2d90 0x7f47fc0a5250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.305 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.304+0000 7f480cd99640 1 -- 172.21.15.32:0/4114945454 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f48000f0470 con 0x7f4810100620 2024-04-25T00:33:54.305 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.305+0000 7f4807fff640 1 --2- 172.21.15.32:0/4114945454 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f47fc0a2d90 0x7f47fc0a5250 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f4810100150 tx=0x7f47f800a2d0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.309 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.308+0000 7f480cd99640 1 -- 172.21.15.32:0/4114945454 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f48000bcc10 con 0x7f4810100620 2024-04-25T00:33:54.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.542+0000 7f4815190640 1 -- 172.21.15.32:0/4114945454 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f4810109ba0 con 0x7f4810100620 2024-04-25T00:33:54.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.542+0000 7f480cd99640 1 -- 172.21.15.32:0/4114945454 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 25 v25) ==== 94+0+11876 (secure 0 0 0) 0x7f48000bc510 con 0x7f4810100620 2024-04-25T00:33:54.543 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:33:54.543 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":25,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14643,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6839/187492674","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6838","nonce":187492674},{"type":"v1","addr":"172.21.15.32:6839","nonce":187492674}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14655,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6841/1948785545","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6840","nonce":1948785545},{"type":"v1","addr":"172.21.15.32:6841","nonce":1948785545}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":25,"flags":306,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":true},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-25T00:30:47.866875+0000","modified":"2024-04-25T00:33:16.420908+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24506,"mds_1":24481,"mds_2":24476},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24443":{"gid":24443,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24445":{"gid":24445,"name":"f","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24457":{"gid":24457,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24476":{"gid":24476,"name":"h","rank":2,"incarnation":22,"state":"up:active","state_seq":12,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24481":{"gid":24481,"name":"i","rank":1,"incarnation":20,"state":"up:active","state_seq":11,"addr":"172.21.15.130:6839/4267125809","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":4267125809},{"type":"v1","addr":"172.21.15.130:6839","nonce":4267125809}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24506":{"gid":24506,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.121:6839/258444853","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6838","nonce":258444853},{"type":"v1","addr":"172.21.15.121:6839","nonce":258444853}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24506,"qdb_cluster":[24506,24476,24481]},"id":1}]} 2024-04-25T00:33:54.543 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 25 2024-04-25T00:33:54.545 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.544+0000 7f4815190640 1 -- 172.21.15.32:0/4114945454 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f47fc0a2d90 msgr2=0x7f47fc0a5250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.545 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.544+0000 7f4815190640 1 --2- 172.21.15.32:0/4114945454 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f47fc0a2d90 0x7f47fc0a5250 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f4810100150 tx=0x7f47f800a2d0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.545 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.544+0000 7f4815190640 1 -- 172.21.15.32:0/4114945454 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4810100620 msgr2=0x7f4810197f50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.545 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.544+0000 7f4815190640 1 --2- 172.21.15.32:0/4114945454 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4810100620 0x7f4810197f50 secure :-1 s=READY pgs=526 cs=0 l=1 rev1=1 crypto rx=0x7f4800009a50 tx=0x7f4800004270 comp rx=0 tx=0).stop 2024-04-25T00:33:54.545 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.544+0000 7f4815190640 1 -- 172.21.15.32:0/4114945454 shutdown_connections 2024-04-25T00:33:54.545 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.544+0000 7f4815190640 1 --2- 172.21.15.32:0/4114945454 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f47fc0a2d90 0x7f47fc0a5250 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.545 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.544+0000 7f4815190640 1 --2- 172.21.15.32:0/4114945454 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4810106630 0x7f481019c7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.545 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.544+0000 7f4815190640 1 --2- 172.21.15.32:0/4114945454 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4810100fe0 0x7f4810198490 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.546 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.544+0000 7f4815190640 1 --2- 172.21.15.32:0/4114945454 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4810100620 0x7f4810197f50 unknown :-1 s=CLOSED pgs=526 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.546 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.544+0000 7f4815190640 1 -- 172.21.15.32:0/4114945454 >> 172.21.15.32:0/4114945454 conn(0x7f4810077740 msgr2=0x7f4810068b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:54.546 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.545+0000 7f4815190640 1 -- 172.21.15.32:0/4114945454 shutdown_connections 2024-04-25T00:33:54.546 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.545+0000 7f4815190640 1 -- 172.21.15.32:0/4114945454 wait complete. 2024-04-25T00:33:54.557 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-25T00:33:54.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.663+0000 7f09e325d640 1 -- 172.21.15.32:0/488251833 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f09dc1077a0 msgr2=0x7f09dc107b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.663+0000 7f09e325d640 1 --2- 172.21.15.32:0/488251833 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f09dc1077a0 0x7f09dc107b80 secure :-1 s=READY pgs=527 cs=0 l=1 rev1=1 crypto rx=0x7f09d0009930 tx=0x7f09d002a420 comp rx=0 tx=0).stop 2024-04-25T00:33:54.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.663+0000 7f09e325d640 1 -- 172.21.15.32:0/488251833 shutdown_connections 2024-04-25T00:33:54.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.663+0000 7f09e325d640 1 --2- 172.21.15.32:0/488251833 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f09dc102150 0x7f09dc10e660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.663+0000 7f09e325d640 1 --2- 172.21.15.32:0/488251833 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f09dc101790 0x7f09dc101c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.663+0000 7f09e325d640 1 --2- 172.21.15.32:0/488251833 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f09dc1077a0 0x7f09dc107b80 unknown :-1 s=CLOSED pgs=527 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.663+0000 7f09e325d640 1 -- 172.21.15.32:0/488251833 >> 172.21.15.32:0/488251833 conn(0x7f09dc0fd2a0 msgr2=0x7f09dc0ff6c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:54.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.663+0000 7f09e325d640 1 -- 172.21.15.32:0/488251833 shutdown_connections 2024-04-25T00:33:54.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.663+0000 7f09e325d640 1 -- 172.21.15.32:0/488251833 wait complete. 2024-04-25T00:33:54.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.663+0000 7f09e325d640 1 Processor -- start 2024-04-25T00:33:54.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.664+0000 7f09e325d640 1 -- start start 2024-04-25T00:33:54.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.664+0000 7f09e325d640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f09dc101790 0x7f09dc1980f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.664+0000 7f09e325d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f09dc102150 0x7f09dc198630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.664+0000 7f09e325d640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f09dc1077a0 0x7f09dc19c960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.664+0000 7f09e325d640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f09dc111bb0 con 0x7f09dc102150 2024-04-25T00:33:54.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.664+0000 7f09e325d640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f09dc111a30 con 0x7f09dc101790 2024-04-25T00:33:54.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.664+0000 7f09e325d640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f09dc111d30 con 0x7f09dc1077a0 2024-04-25T00:33:54.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.664+0000 7f09dbfff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f09dc102150 0x7f09dc198630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.664+0000 7f09dbfff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f09dc102150 0x7f09dc198630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53490/0 (socket says 172.21.15.32:53490) 2024-04-25T00:33:54.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.664+0000 7f09dbfff640 1 -- 172.21.15.32:0/3786897080 learned_addr learned my addr 172.21.15.32:0/3786897080 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:54.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.664+0000 7f09e0ff9640 1 --2- 172.21.15.32:0/3786897080 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f09dc101790 0x7f09dc1980f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.664+0000 7f09e17fa640 1 --2- 172.21.15.32:0/3786897080 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f09dc1077a0 0x7f09dc19c960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.664+0000 7f09dbfff640 1 -- 172.21.15.32:0/3786897080 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f09dc1077a0 msgr2=0x7f09dc19c960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.664+0000 7f09dbfff640 1 --2- 172.21.15.32:0/3786897080 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f09dc1077a0 0x7f09dc19c960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.664+0000 7f09dbfff640 1 -- 172.21.15.32:0/3786897080 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f09dc101790 msgr2=0x7f09dc1980f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.664+0000 7f09dbfff640 1 --2- 172.21.15.32:0/3786897080 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f09dc101790 0x7f09dc1980f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.664+0000 7f09dbfff640 1 -- 172.21.15.32:0/3786897080 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f09dc19d090 con 0x7f09dc102150 2024-04-25T00:33:54.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.664+0000 7f09dbfff640 1 --2- 172.21.15.32:0/3786897080 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f09dc102150 0x7f09dc198630 secure :-1 s=READY pgs=528 cs=0 l=1 rev1=1 crypto rx=0x7f09c800d770 tx=0x7f09c800dc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.665+0000 7f09d9ffb640 1 -- 172.21.15.32:0/3786897080 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f09c8017070 con 0x7f09dc102150 2024-04-25T00:33:54.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.665+0000 7f09e325d640 1 -- 172.21.15.32:0/3786897080 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f09dc19d380 con 0x7f09dc102150 2024-04-25T00:33:54.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.665+0000 7f09d9ffb640 1 -- 172.21.15.32:0/3786897080 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f09c8005500 con 0x7f09dc102150 2024-04-25T00:33:54.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.665+0000 7f09d9ffb640 1 -- 172.21.15.32:0/3786897080 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f09c8004e00 con 0x7f09dc102150 2024-04-25T00:33:54.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.665+0000 7f09e325d640 1 -- 172.21.15.32:0/3786897080 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f09dc1a4c10 con 0x7f09dc102150 2024-04-25T00:33:54.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.667+0000 7f09d9ffb640 1 -- 172.21.15.32:0/3786897080 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 24) ==== 138009+0+0 (secure 0 0 0) 0x7f09c8004fa0 con 0x7f09dc102150 2024-04-25T00:33:54.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.667+0000 7f09e5f69740 1 Processor -- start 2024-04-25T00:33:54.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.667+0000 7f09e5f69740 1 -- start start 2024-04-25T00:33:54.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.668+0000 7f09e5f69740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5581ef911580 0x5581ef911960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.668+0000 7f09e5f69740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5581ef911ea0 0x5581ef91c030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.668+0000 7f09e5f69740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef91c570 0x5581ef91e960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.668+0000 7f09e5f69740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5581ef876670 con 0x5581ef91c570 2024-04-25T00:33:54.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.668+0000 7f09e5f69740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5581ef81e970 con 0x5581ef911580 2024-04-25T00:33:54.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.668+0000 7f09e5f69740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5581ef872630 con 0x5581ef911ea0 2024-04-25T00:33:54.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.668+0000 7f09d9ffb640 1 --2- 172.21.15.32:0/3786897080 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f09cc0a2d90 0x7f09cc0a5250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.668+0000 7f09dbfff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef91c570 0x5581ef91e960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.668+0000 7f09e0ff9640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5581ef911ea0 0x5581ef91c030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.668+0000 7f09e0ff9640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5581ef911ea0 0x5581ef91c030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:47592/0 (socket says 172.21.15.32:47592) 2024-04-25T00:33:54.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.668+0000 7f09e0ff9640 1 -- 172.21.15.32:0/1782261365 learned_addr learned my addr 172.21.15.32:0/1782261365 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:54.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.668+0000 7f09e17fa640 1 --2- 172.21.15.32:0/3786897080 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f09cc0a2d90 0x7f09cc0a5250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.668+0000 7f09e17fa640 1 --2- 172.21.15.32:0/1782261365 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5581ef911580 0x5581ef911960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.668+0000 7f09d9ffb640 1 -- 172.21.15.32:0/3786897080 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f09c80cf420 con 0x7f09dc102150 2024-04-25T00:33:54.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.668+0000 7f09dbfff640 1 -- 172.21.15.32:0/1782261365 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5581ef911ea0 msgr2=0x5581ef91c030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.668+0000 7f09dbfff640 1 --2- 172.21.15.32:0/1782261365 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5581ef911ea0 0x5581ef91c030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.668+0000 7f09dbfff640 1 -- 172.21.15.32:0/1782261365 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5581ef911580 msgr2=0x5581ef911960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.668+0000 7f09dbfff640 1 --2- 172.21.15.32:0/1782261365 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5581ef911580 0x5581ef911960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.668+0000 7f09dbfff640 1 -- 172.21.15.32:0/1782261365 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5581ef842400 con 0x5581ef91c570 2024-04-25T00:33:54.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.669+0000 7f09e17fa640 1 --2- 172.21.15.32:0/3786897080 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f09cc0a2d90 0x7f09cc0a5250 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f09d40129a0 tx=0x7f09d400abc0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.669+0000 7f09dbfff640 1 --2- 172.21.15.32:0/1782261365 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef91c570 0x5581ef91e960 secure :-1 s=READY pgs=529 cs=0 l=1 rev1=1 crypto rx=0x7f09c80cc700 tx=0x7f09c80c5fb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=13589a25ee7e567b server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.669+0000 7f09c6ffd640 1 -- 172.21.15.32:0/1782261365 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f09c809bb20 con 0x5581ef91c570 2024-04-25T00:33:54.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.669+0000 7f09c6ffd640 1 -- 172.21.15.32:0/1782261365 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f09c809b420 con 0x5581ef91c570 2024-04-25T00:33:54.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.669+0000 7f09c6ffd640 1 -- 172.21.15.32:0/1782261365 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f09c8097a90 con 0x5581ef91c570 2024-04-25T00:33:54.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.670+0000 7f09e5f69740 1 -- 172.21.15.32:0/1782261365 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef91c570 msgr2=0x5581ef91e960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.670+0000 7f09e5f69740 1 --2- 172.21.15.32:0/1782261365 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef91c570 0x5581ef91e960 secure :-1 s=READY pgs=529 cs=0 l=1 rev1=1 crypto rx=0x7f09c80cc700 tx=0x7f09c80c5fb0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.670+0000 7f09e5f69740 1 -- 172.21.15.32:0/1782261365 shutdown_connections 2024-04-25T00:33:54.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.670+0000 7f09e5f69740 1 --2- 172.21.15.32:0/1782261365 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef91c570 0x5581ef91e960 unknown :-1 s=CLOSED pgs=529 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.670+0000 7f09e5f69740 1 --2- 172.21.15.32:0/1782261365 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5581ef911ea0 0x5581ef91c030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.670+0000 7f09e5f69740 1 --2- 172.21.15.32:0/1782261365 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5581ef911580 0x5581ef911960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.670+0000 7f09e5f69740 1 -- 172.21.15.32:0/1782261365 >> 172.21.15.32:0/1782261365 conn(0x5581ef90d8a0 msgr2=0x5581ef90dc80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:54.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.670+0000 7f09e5f69740 1 -- 172.21.15.32:0/1782261365 shutdown_connections 2024-04-25T00:33:54.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.670+0000 7f09e5f69740 1 -- 172.21.15.32:0/1782261365 wait complete. 2024-04-25T00:33:54.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.670+0000 7f09e5f69740 1 -- start start 2024-04-25T00:33:54.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.670+0000 7f09e5f69740 1 Processor -- start 2024-04-25T00:33:54.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.670+0000 7f09e5f69740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5581ef911580 0x5581ef91bf00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.670+0000 7f09e5f69740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5581ef911ea0 0x5581ef9191f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.670+0000 7f09e5f69740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef91c570 0x5581ef919770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.670+0000 7f09e5f69740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5581ef91f360 con 0x5581ef91c570 2024-04-25T00:33:54.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.671+0000 7f09e5f69740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5581ef832050 con 0x5581ef911580 2024-04-25T00:33:54.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.671+0000 7f09e5f69740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5581ef91f4e0 con 0x5581ef911ea0 2024-04-25T00:33:54.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.671+0000 7f09e5f69740 10 client.0 _ll_register_callbacks cb 0x5581ef589be0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:33:54.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.671+0000 7f09e17fa640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef91c570 0x5581ef919770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.671+0000 7f09e17fa640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef91c570 0x5581ef919770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53508/0 (socket says 172.21.15.32:53508) 2024-04-25T00:33:54.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.671+0000 7f09e17fa640 1 -- 172.21.15.32:0/684511411 learned_addr learned my addr 172.21.15.32:0/684511411 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:54.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.671+0000 7f09e0ff9640 1 --2- 172.21.15.32:0/684511411 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5581ef911580 0x5581ef91bf00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.671+0000 7f09dbfff640 1 --2- 172.21.15.32:0/684511411 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5581ef911ea0 0x5581ef9191f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.671+0000 7f09e17fa640 1 -- 172.21.15.32:0/684511411 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5581ef911ea0 msgr2=0x5581ef9191f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.671+0000 7f09e17fa640 1 --2- 172.21.15.32:0/684511411 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5581ef911ea0 0x5581ef9191f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.671+0000 7f09e17fa640 1 -- 172.21.15.32:0/684511411 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5581ef911580 msgr2=0x5581ef91bf00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.671+0000 7f09e17fa640 1 --2- 172.21.15.32:0/684511411 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5581ef911580 0x5581ef91bf00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.671+0000 7f09e17fa640 1 -- 172.21.15.32:0/684511411 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5581ef803dc0 con 0x5581ef91c570 2024-04-25T00:33:54.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.671+0000 7f09e0ff9640 1 --2- 172.21.15.32:0/684511411 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5581ef911580 0x5581ef91bf00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:54.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.671+0000 7f09dbfff640 1 --2- 172.21.15.32:0/684511411 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5581ef911ea0 0x5581ef9191f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:54.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.671+0000 7f09e17fa640 1 --2- 172.21.15.32:0/684511411 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef91c570 0x5581ef919770 secure :-1 s=READY pgs=530 cs=0 l=1 rev1=1 crypto rx=0x7f09d400aea0 tx=0x7f09d4019000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b4eeee6f9116c183 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.671+0000 7f09bbfff640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:33:54.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.671+0000 7f09bbfff640 1 -- 172.21.15.32:0/684511411 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f09d4029610 con 0x5581ef91c570 2024-04-25T00:33:54.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.671+0000 7f09bbfff640 1 -- 172.21.15.32:0/684511411 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f09d40297b0 con 0x5581ef91c570 2024-04-25T00:33:54.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.671+0000 7f09e5f69740 1 -- 172.21.15.32:0/684511411 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f09c809b420 con 0x5581ef91c570 2024-04-25T00:33:54.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.672+0000 7f09bbfff640 1 -- 172.21.15.32:0/684511411 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f09d4032050 con 0x5581ef91c570 2024-04-25T00:33:54.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.672+0000 7f09e5f69740 10 client.15771 fetch_fsmap learned FSMap version 25 2024-04-25T00:33:54.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.672+0000 7f09e5f69740 1 -- 172.21.15.32:0/684511411 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x5581ef58a3f0 con 0x5581ef91c570 2024-04-25T00:33:54.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.672+0000 7f09bbfff640 1 -- 172.21.15.32:0/684511411 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f09c809b420 con 0x5581ef91c570 2024-04-25T00:33:54.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.672+0000 7f09bbfff640 1 -- 172.21.15.32:0/684511411 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7f09d401d380 con 0x5581ef91c570 2024-04-25T00:33:54.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.672+0000 7f09e5f69740 10 client.15771 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:33:54.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.672+0000 7f09e5f69740 10 client.15771 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-25T00:33:54.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.672+0000 7f09e5f69740 20 client.15771 populate_metadata read hostname 'smithi032' 2024-04-25T00:33:54.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.672+0000 7f09e5f69740 1 --2- 172.21.15.32:0/684511411 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x5581ef92c630 0x5581ef92eaa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.672+0000 7f09e5f69740 4 client.15771 mds_command: new command op to 24476 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:33:54.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.672+0000 7f09e5f69740 1 -- 172.21.15.32:0/684511411 --> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5581ef883bb0 con 0x5581ef92c630 2024-04-25T00:33:54.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.672+0000 7f09e0ff9640 1 --2- 172.21.15.32:0/684511411 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x5581ef92c630 0x5581ef92eaa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.673+0000 7f09bbfff640 10 client.15771 ms_handle_connect on v2:172.21.15.121:6836/3456022860 2024-04-25T00:33:54.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.673+0000 7f09e0ff9640 1 --2- 172.21.15.32:0/684511411 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x5581ef92c630 0x5581ef92eaa0 crc :-1 s=READY pgs=34 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=90c5b4c004a6df8b server_cookie=578657488b80b4d5 in_seq=0 out_seq=0 2024-04-25T00:33:54.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.673+0000 7f09bbfff640 1 -- 172.21.15.32:0/684511411 <== mds.2 v2:172.21.15.121:6836/3456022860 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x5581ef883bb0 con 0x5581ef92c630 2024-04-25T00:33:54.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.673+0000 7f09bbfff640 10 client.15771 handle_command_reply: tid=0 2024-04-25T00:33:54.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.673+0000 7f09e5f69740 1 client.15771 shutdown 2024-04-25T00:33:54.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.674+0000 7f09e5f69740 1 -- 172.21.15.32:0/684511411 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef91c570 msgr2=0x5581ef919770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.674+0000 7f09e5f69740 1 --2- 172.21.15.32:0/684511411 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef91c570 0x5581ef919770 secure :-1 s=READY pgs=530 cs=0 l=1 rev1=1 crypto rx=0x7f09d400aea0 tx=0x7f09d4019000 comp rx=0 tx=0).stop 2024-04-25T00:33:54.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.674+0000 7f09e5f69740 1 -- 172.21.15.32:0/684511411 shutdown_connections 2024-04-25T00:33:54.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.674+0000 7f09e5f69740 1 --2- 172.21.15.32:0/684511411 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x5581ef92c630 0x5581ef92eaa0 crc :-1 s=READY pgs=34 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.674+0000 7f09e5f69740 1 --2- 172.21.15.32:0/684511411 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef91c570 0x5581ef919770 unknown :-1 s=CLOSED pgs=530 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.674+0000 7f09e5f69740 1 --2- 172.21.15.32:0/684511411 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5581ef911ea0 0x5581ef9191f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.674+0000 7f09e5f69740 1 --2- 172.21.15.32:0/684511411 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5581ef911580 0x5581ef91bf00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.674+0000 7f09bbfff640 0 client.15771 ms_handle_reset on v2:172.21.15.121:6836/3456022860 2024-04-25T00:33:54.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.674+0000 7f09e5f69740 1 -- 172.21.15.32:0/684511411 >> 172.21.15.32:0/684511411 conn(0x5581ef90d8a0 msgr2=0x5581ef90cb40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:54.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.674+0000 7f09e5f69740 1 -- 172.21.15.32:0/684511411 shutdown_connections 2024-04-25T00:33:54.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.674+0000 7f09e5f69740 1 -- 172.21.15.32:0/684511411 wait complete. 2024-04-25T00:33:54.682 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.674+0000 7f09e5f69740 20 client.15771 trim_cache size 0 max 16384 2024-04-25T00:33:54.689 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.687+0000 7f09e5f69740 1 Processor -- start 2024-04-25T00:33:54.689 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.687+0000 7f09e5f69740 1 -- start start 2024-04-25T00:33:54.689 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.687+0000 7f09e5f69740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef911580 0x5581ef928220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.689 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.687+0000 7f09e5f69740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5581ef911ea0 0x5581ef934370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.689 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.687+0000 7f09e5f69740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5581ef91c570 0x5581ef9348b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.689 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.687+0000 7f09e5f69740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f09dc1b14b0 con 0x5581ef911580 2024-04-25T00:33:54.689 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.687+0000 7f09e5f69740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f09dc1aed80 con 0x5581ef91c570 2024-04-25T00:33:54.689 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.687+0000 7f09e5f69740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f09dc1acbd0 con 0x5581ef911ea0 2024-04-25T00:33:54.689 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.687+0000 7f09e0ff9640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef911580 0x5581ef928220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.689 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.687+0000 7f09e0ff9640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef911580 0x5581ef928220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53520/0 (socket says 172.21.15.32:53520) 2024-04-25T00:33:54.689 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.687+0000 7f09e0ff9640 1 -- 172.21.15.32:0/3075116319 learned_addr learned my addr 172.21.15.32:0/3075116319 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:54.689 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.688+0000 7f09dbfff640 1 --2- 172.21.15.32:0/3075116319 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5581ef911ea0 0x5581ef934370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.688+0000 7f09e0ff9640 1 -- 172.21.15.32:0/3075116319 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5581ef911ea0 msgr2=0x5581ef934370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.688+0000 7f09e0ff9640 1 --2- 172.21.15.32:0/3075116319 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5581ef911ea0 0x5581ef934370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.688+0000 7f09e0ff9640 1 -- 172.21.15.32:0/3075116319 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5581ef91c570 msgr2=0x5581ef9348b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.688+0000 7f09e0ff9640 1 --2- 172.21.15.32:0/3075116319 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5581ef91c570 0x5581ef9348b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.688+0000 7f09e0ff9640 1 -- 172.21.15.32:0/3075116319 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5581ef6547d0 con 0x5581ef911580 2024-04-25T00:33:54.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.688+0000 7f09e0ff9640 1 --2- 172.21.15.32:0/3075116319 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef911580 0x5581ef928220 secure :-1 s=READY pgs=531 cs=0 l=1 rev1=1 crypto rx=0x7f09d002a940 tx=0x7f09d00058d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5759bd062bac5794 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.688+0000 7f09c67fc640 1 -- 172.21.15.32:0/3075116319 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f09d004b070 con 0x5581ef911580 2024-04-25T00:33:54.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.688+0000 7f09c67fc640 1 -- 172.21.15.32:0/3075116319 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f09d0046dc0 con 0x5581ef911580 2024-04-25T00:33:54.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.688+0000 7f09c67fc640 1 -- 172.21.15.32:0/3075116319 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f09d0035040 con 0x5581ef911580 2024-04-25T00:33:54.695 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.689+0000 7f09e5f69740 1 -- 172.21.15.32:0/3075116319 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef911580 msgr2=0x5581ef928220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.695 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.689+0000 7f09e5f69740 1 --2- 172.21.15.32:0/3075116319 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef911580 0x5581ef928220 secure :-1 s=READY pgs=531 cs=0 l=1 rev1=1 crypto rx=0x7f09d002a940 tx=0x7f09d00058d0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.695 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.689+0000 7f09e5f69740 1 -- 172.21.15.32:0/3075116319 shutdown_connections 2024-04-25T00:33:54.695 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.689+0000 7f09e5f69740 1 --2- 172.21.15.32:0/3075116319 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5581ef91c570 0x5581ef9348b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.695 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.689+0000 7f09e5f69740 1 --2- 172.21.15.32:0/3075116319 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5581ef911ea0 0x5581ef934370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.695 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.689+0000 7f09e5f69740 1 --2- 172.21.15.32:0/3075116319 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef911580 0x5581ef928220 unknown :-1 s=CLOSED pgs=531 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.695 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.689+0000 7f09e5f69740 1 -- 172.21.15.32:0/3075116319 >> 172.21.15.32:0/3075116319 conn(0x5581ef90d8a0 msgr2=0x5581ef90cb40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:54.695 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.689+0000 7f09e5f69740 1 -- 172.21.15.32:0/3075116319 shutdown_connections 2024-04-25T00:33:54.695 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.689+0000 7f09e5f69740 1 -- 172.21.15.32:0/3075116319 wait complete. 2024-04-25T00:33:54.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.689+0000 7f09e5f69740 1 -- start start 2024-04-25T00:33:54.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.689+0000 7f09e5f69740 1 Processor -- start 2024-04-25T00:33:54.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.690+0000 7f09e5f69740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef911580 0x5581ef9248b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.690+0000 7f09e5f69740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5581ef911ea0 0x5581ef921c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.690+0000 7f09e5f69740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5581ef91c570 0x5581ef922180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.690+0000 7f09e5f69740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f09dc1ad240 con 0x5581ef911580 2024-04-25T00:33:54.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.690+0000 7f09e5f69740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f09dc1ae6b0 con 0x5581ef91c570 2024-04-25T00:33:54.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.690+0000 7f09e5f69740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f09dc1ad910 con 0x5581ef911ea0 2024-04-25T00:33:54.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.690+0000 7f09e5f69740 10 client.0 _ll_register_callbacks cb 0x7f09d400fb90 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:33:54.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.690+0000 7f09e0ff9640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef911580 0x5581ef9248b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.690+0000 7f09e0ff9640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef911580 0x5581ef9248b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53522/0 (socket says 172.21.15.32:53522) 2024-04-25T00:33:54.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.690+0000 7f09e0ff9640 1 -- 172.21.15.32:0/1585666741 learned_addr learned my addr 172.21.15.32:0/1585666741 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:54.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.690+0000 7f09e17fa640 1 --2- 172.21.15.32:0/1585666741 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5581ef91c570 0x5581ef922180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.690+0000 7f09e0ff9640 1 -- 172.21.15.32:0/1585666741 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5581ef911ea0 msgr2=0x5581ef921c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.690+0000 7f09dbfff640 1 --2- 172.21.15.32:0/1585666741 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5581ef911ea0 0x5581ef921c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.690+0000 7f09e0ff9640 1 --2- 172.21.15.32:0/1585666741 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5581ef911ea0 0x5581ef921c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.690+0000 7f09e0ff9640 1 -- 172.21.15.32:0/1585666741 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5581ef91c570 msgr2=0x5581ef922180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.690+0000 7f09e0ff9640 1 --2- 172.21.15.32:0/1585666741 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5581ef91c570 0x5581ef922180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.690+0000 7f09e0ff9640 1 -- 172.21.15.32:0/1585666741 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5581ef922740 con 0x5581ef911580 2024-04-25T00:33:54.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.690+0000 7f09e17fa640 1 --2- 172.21.15.32:0/1585666741 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5581ef91c570 0x5581ef922180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:54.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.690+0000 7f09dbfff640 1 --2- 172.21.15.32:0/1585666741 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5581ef911ea0 0x5581ef921c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:33:54.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.690+0000 7f09e0ff9640 1 --2- 172.21.15.32:0/1585666741 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef911580 0x5581ef9248b0 secure :-1 s=READY pgs=532 cs=0 l=1 rev1=1 crypto rx=0x7f09d0049040 tx=0x7f09d0005fd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=98683a7b64108213 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.690+0000 7f09bbfff640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:33:54.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.690+0000 7f09bbfff640 1 -- 172.21.15.32:0/1585666741 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f09d004b070 con 0x5581ef911580 2024-04-25T00:33:54.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.691+0000 7f09bbfff640 1 -- 172.21.15.32:0/1585666741 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f09d0005040 con 0x5581ef911580 2024-04-25T00:33:54.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.691+0000 7f09e5f69740 1 -- 172.21.15.32:0/1585666741 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f09d0046dc0 con 0x5581ef911580 2024-04-25T00:33:54.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.691+0000 7f09bbfff640 1 -- 172.21.15.32:0/1585666741 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f09d005e050 con 0x5581ef911580 2024-04-25T00:33:54.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.691+0000 7f09e5f69740 10 client.15777 fetch_fsmap learned FSMap version 25 2024-04-25T00:33:54.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.691+0000 7f09e5f69740 1 -- 172.21.15.32:0/1585666741 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x5581ef9929a0 con 0x5581ef911580 2024-04-25T00:33:54.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.691+0000 7f09bbfff640 1 -- 172.21.15.32:0/1585666741 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f09d0046dc0 con 0x5581ef911580 2024-04-25T00:33:54.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.691+0000 7f09bbfff640 1 -- 172.21.15.32:0/1585666741 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7f09d0059030 con 0x5581ef911580 2024-04-25T00:33:54.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.691+0000 7f09e5f69740 10 client.15777 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:33:54.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.691+0000 7f09e5f69740 10 client.15777 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-25T00:33:54.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.691+0000 7f09e5f69740 20 client.15777 populate_metadata read hostname 'smithi032' 2024-04-25T00:33:54.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.691+0000 7f09e5f69740 1 --2- 172.21.15.32:0/1585666741 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x5581ef993110 0x5581ef995530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.691+0000 7f09e5f69740 4 client.15777 mds_command: new command op to 24476 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-25T00:33:54.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.691+0000 7f09e5f69740 1 -- 172.21.15.32:0/1585666741 --> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] -- command(tid 0: {"prefix": "damage ls"}) -- 0x5581ef87df00 con 0x5581ef993110 2024-04-25T00:33:54.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.691+0000 7f09dbfff640 1 --2- 172.21.15.32:0/1585666741 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x5581ef993110 0x5581ef995530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.692+0000 7f09dbfff640 1 --2- 172.21.15.32:0/1585666741 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x5581ef993110 0x5581ef995530 crc :-1 s=READY pgs=35 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=5dd03f08ae753a6e server_cookie=be9e5e0223e609f in_seq=0 out_seq=0 2024-04-25T00:33:54.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.692+0000 7f09bbfff640 10 client.15777 ms_handle_connect on v2:172.21.15.121:6836/3456022860 2024-04-25T00:33:54.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.692+0000 7f09bbfff640 1 -- 172.21.15.32:0/1585666741 <== mds.2 v2:172.21.15.121:6836/3456022860 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x5581ef87df00 con 0x5581ef993110 2024-04-25T00:33:54.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.692+0000 7f09bbfff640 10 client.15777 handle_command_reply: tid=0 2024-04-25T00:33:54.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.692+0000 7f09e5f69740 1 client.15777 shutdown 2024-04-25T00:33:54.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.692+0000 7f09e5f69740 1 -- 172.21.15.32:0/1585666741 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef911580 msgr2=0x5581ef9248b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.692+0000 7f09e5f69740 1 --2- 172.21.15.32:0/1585666741 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef911580 0x5581ef9248b0 secure :-1 s=READY pgs=532 cs=0 l=1 rev1=1 crypto rx=0x7f09d0049040 tx=0x7f09d0005fd0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.692+0000 7f09e5f69740 1 -- 172.21.15.32:0/1585666741 shutdown_connections 2024-04-25T00:33:54.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.692+0000 7f09e5f69740 1 --2- 172.21.15.32:0/1585666741 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x5581ef993110 0x5581ef995530 crc :-1 s=READY pgs=35 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.692+0000 7f09e5f69740 1 --2- 172.21.15.32:0/1585666741 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5581ef91c570 0x5581ef922180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.692+0000 7f09bbfff640 0 client.15777 ms_handle_reset on v2:172.21.15.121:6836/3456022860 2024-04-25T00:33:54.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.692+0000 7f09e5f69740 1 --2- 172.21.15.32:0/1585666741 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5581ef911ea0 0x5581ef921c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.692+0000 7f09e5f69740 1 --2- 172.21.15.32:0/1585666741 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5581ef911580 0x5581ef9248b0 unknown :-1 s=CLOSED pgs=532 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.692+0000 7f09e5f69740 1 -- 172.21.15.32:0/1585666741 >> 172.21.15.32:0/1585666741 conn(0x5581ef90d8a0 msgr2=0x5581ef927210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:54.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.692+0000 7f09e5f69740 1 -- 172.21.15.32:0/1585666741 shutdown_connections 2024-04-25T00:33:54.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.692+0000 7f09e5f69740 1 -- 172.21.15.32:0/1585666741 wait complete. 2024-04-25T00:33:54.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.692+0000 7f09e5f69740 20 client.15777 trim_cache size 0 max 16384 2024-04-25T00:33:54.700 INFO:teuthology.orchestra.run.smithi032.stdout:[] 2024-04-25T00:33:54.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.693+0000 7f09e325d640 1 -- 172.21.15.32:0/3786897080 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f09cc0a2d90 msgr2=0x7f09cc0a5250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.693+0000 7f09e325d640 1 --2- 172.21.15.32:0/3786897080 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f09cc0a2d90 0x7f09cc0a5250 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f09d40129a0 tx=0x7f09d400abc0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.693+0000 7f09e325d640 1 -- 172.21.15.32:0/3786897080 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f09dc102150 msgr2=0x7f09dc198630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.693+0000 7f09e325d640 1 --2- 172.21.15.32:0/3786897080 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f09dc102150 0x7f09dc198630 secure :-1 s=READY pgs=528 cs=0 l=1 rev1=1 crypto rx=0x7f09c800d770 tx=0x7f09c800dc30 comp rx=0 tx=0).stop 2024-04-25T00:33:54.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.693+0000 7f09e325d640 1 -- 172.21.15.32:0/3786897080 shutdown_connections 2024-04-25T00:33:54.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.693+0000 7f09e325d640 1 --2- 172.21.15.32:0/3786897080 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f09cc0a2d90 0x7f09cc0a5250 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.693+0000 7f09e325d640 1 --2- 172.21.15.32:0/3786897080 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f09dc1077a0 0x7f09dc19c960 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.693+0000 7f09e325d640 1 --2- 172.21.15.32:0/3786897080 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f09dc102150 0x7f09dc198630 unknown :-1 s=CLOSED pgs=528 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.693+0000 7f09e325d640 1 --2- 172.21.15.32:0/3786897080 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f09dc101790 0x7f09dc1980f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.693+0000 7f09e325d640 1 -- 172.21.15.32:0/3786897080 >> 172.21.15.32:0/3786897080 conn(0x7f09dc0fd2a0 msgr2=0x7f09dc0fdb50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:54.702 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.693+0000 7f09e325d640 1 -- 172.21.15.32:0/3786897080 shutdown_connections 2024-04-25T00:33:54.702 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.693+0000 7f09e325d640 1 -- 172.21.15.32:0/3786897080 wait complete. 2024-04-25T00:33:54.704 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-25T00:33:54.828 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.810+0000 7f015ff42640 1 -- 172.21.15.32:0/2141156666 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0158100fd0 msgr2=0x7f015810d4e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.828 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.810+0000 7f015ff42640 1 --2- 172.21.15.32:0/2141156666 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0158100fd0 0x7f015810d4e0 secure :-1 s=READY pgs=533 cs=0 l=1 rev1=1 crypto rx=0x7f0150009cd0 tx=0x7f015002a8e0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.828 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.810+0000 7f015ff42640 1 -- 172.21.15.32:0/2141156666 shutdown_connections 2024-04-25T00:33:54.828 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.810+0000 7f015ff42640 1 --2- 172.21.15.32:0/2141156666 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0158100fd0 0x7f015810d4e0 unknown :-1 s=CLOSED pgs=533 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.810+0000 7f015ff42640 1 --2- 172.21.15.32:0/2141156666 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0158100610 0x7f0158100a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.810+0000 7f015ff42640 1 --2- 172.21.15.32:0/2141156666 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f01581066b0 0x7f0158106a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.810+0000 7f015ff42640 1 -- 172.21.15.32:0/2141156666 >> 172.21.15.32:0/2141156666 conn(0x7f0158077740 msgr2=0x7f0158077b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:54.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.810+0000 7f015ff42640 1 -- 172.21.15.32:0/2141156666 shutdown_connections 2024-04-25T00:33:54.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.810+0000 7f015ff42640 1 -- 172.21.15.32:0/2141156666 wait complete. 2024-04-25T00:33:54.829 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.811+0000 7f015ff42640 1 Processor -- start 2024-04-25T00:33:54.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.811+0000 7f015ff42640 1 -- start start 2024-04-25T00:33:54.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.811+0000 7f015ff42640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0158100610 0x7f01581022b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.811+0000 7f015ff42640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0158100fd0 0x7f01581027f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.811+0000 7f015ff42640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f01581066b0 0x7f0158102d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.811+0000 7f015ff42640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f01581109f0 con 0x7f0158100fd0 2024-04-25T00:33:54.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.811+0000 7f015ff42640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f0158110870 con 0x7f01581066b0 2024-04-25T00:33:54.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.811+0000 7f015ff42640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f0158110b70 con 0x7f0158100610 2024-04-25T00:33:54.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.811+0000 7f015d4dd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0158100fd0 0x7f01581027f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.811+0000 7f015d4dd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0158100fd0 0x7f01581027f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53548/0 (socket says 172.21.15.32:53548) 2024-04-25T00:33:54.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.811+0000 7f015d4dd640 1 -- 172.21.15.32:0/2611677667 learned_addr learned my addr 172.21.15.32:0/2611677667 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:54.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.811+0000 7f015e4df640 1 --2- 172.21.15.32:0/2611677667 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f01581066b0 0x7f0158102d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.812+0000 7f015dcde640 1 --2- 172.21.15.32:0/2611677667 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0158100610 0x7f01581022b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.812+0000 7f015d4dd640 1 -- 172.21.15.32:0/2611677667 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0158100610 msgr2=0x7f01581022b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.812+0000 7f015d4dd640 1 --2- 172.21.15.32:0/2611677667 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0158100610 0x7f01581022b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.812+0000 7f015d4dd640 1 -- 172.21.15.32:0/2611677667 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f01581066b0 msgr2=0x7f0158102d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.812+0000 7f015d4dd640 1 --2- 172.21.15.32:0/2611677667 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f01581066b0 0x7f0158102d30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.812+0000 7f015d4dd640 1 -- 172.21.15.32:0/2611677667 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f01581a63d0 con 0x7f0158100fd0 2024-04-25T00:33:54.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.812+0000 7f015d4dd640 1 --2- 172.21.15.32:0/2611677667 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0158100fd0 0x7f01581027f0 secure :-1 s=READY pgs=534 cs=0 l=1 rev1=1 crypto rx=0x7f014800e9f0 tx=0x7f014800eeb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.812+0000 7f014effd640 1 -- 172.21.15.32:0/2611677667 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f014800cd80 con 0x7f0158100fd0 2024-04-25T00:33:54.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.812+0000 7f015ff42640 1 -- 172.21.15.32:0/2611677667 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f01581a66c0 con 0x7f0158100fd0 2024-04-25T00:33:54.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.812+0000 7f014effd640 1 -- 172.21.15.32:0/2611677667 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0148010910 con 0x7f0158100fd0 2024-04-25T00:33:54.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.812+0000 7f014effd640 1 -- 172.21.15.32:0/2611677667 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f01480188f0 con 0x7f0158100fd0 2024-04-25T00:33:54.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.812+0000 7f015ff42640 1 -- 172.21.15.32:0/2611677667 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f01581a6c60 con 0x7f0158100fd0 2024-04-25T00:33:54.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.814+0000 7f014effd640 1 -- 172.21.15.32:0/2611677667 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 24) ==== 138009+0+0 (secure 0 0 0) 0x7f0148010ac0 con 0x7f0158100fd0 2024-04-25T00:33:54.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.815+0000 7f0162c54740 1 Processor -- start 2024-04-25T00:33:54.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.815+0000 7f0162c54740 1 -- start start 2024-04-25T00:33:54.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.815+0000 7f0162c54740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c49580 0x55d173c49960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.815+0000 7f0162c54740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d173c49ea0 0x55d173c54030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.815+0000 7f0162c54740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d173c54570 0x55d173c56960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.815+0000 7f0162c54740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55d173bae670 con 0x55d173c49580 2024-04-25T00:33:54.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.815+0000 7f0162c54740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55d173b56970 con 0x55d173c49ea0 2024-04-25T00:33:54.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.815+0000 7f0162c54740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55d173baa630 con 0x55d173c54570 2024-04-25T00:33:54.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.815+0000 7f015e4df640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c49580 0x55d173c49960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.815+0000 7f015e4df640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c49580 0x55d173c49960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53558/0 (socket says 172.21.15.32:53558) 2024-04-25T00:33:54.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.815+0000 7f015e4df640 1 -- 172.21.15.32:0/1430608606 learned_addr learned my addr 172.21.15.32:0/1430608606 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:54.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.815+0000 7f014effd640 1 --2- 172.21.15.32:0/2611677667 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f01440a2dc0 0x7f01440a5280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.815+0000 7f015dcde640 1 --2- 172.21.15.32:0/1430608606 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d173c49ea0 0x55d173c54030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.815+0000 7f015e4df640 1 --2- 172.21.15.32:0/2611677667 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f01440a2dc0 0x7f01440a5280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.816+0000 7f015d4dd640 1 --2- 172.21.15.32:0/1430608606 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d173c54570 0x55d173c56960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.816+0000 7f015e4df640 1 -- 172.21.15.32:0/1430608606 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d173c54570 msgr2=0x55d173c56960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.816+0000 7f015e4df640 1 --2- 172.21.15.32:0/1430608606 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d173c54570 0x55d173c56960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.816+0000 7f015e4df640 1 -- 172.21.15.32:0/1430608606 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d173c49ea0 msgr2=0x55d173c54030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.816+0000 7f014effd640 1 -- 172.21.15.32:0/2611677667 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f01480cfe00 con 0x7f0158100fd0 2024-04-25T00:33:54.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.816+0000 7f015e4df640 1 --2- 172.21.15.32:0/1430608606 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d173c49ea0 0x55d173c54030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.816+0000 7f015e4df640 1 -- 172.21.15.32:0/1430608606 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d173b7a400 con 0x55d173c49580 2024-04-25T00:33:54.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.816+0000 7f015dcde640 1 --2- 172.21.15.32:0/1430608606 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d173c49ea0 0x55d173c54030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:54.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.816+0000 7f015e4df640 1 --2- 172.21.15.32:0/1430608606 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c49580 0x55d173c49960 secure :-1 s=READY pgs=535 cs=0 l=1 rev1=1 crypto rx=0x7f0150009430 tx=0x7f0150031a70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=eb3cd2cc9f77580b server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.816+0000 7f015e4df640 1 --2- 172.21.15.32:0/2611677667 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f01440a2dc0 0x7f01440a5280 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f0150047000 tx=0x7f015004a000 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.816+0000 7f0143fff640 1 -- 172.21.15.32:0/1430608606 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0150004410 con 0x55d173c49580 2024-04-25T00:33:54.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.816+0000 7f0143fff640 1 -- 172.21.15.32:0/1430608606 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0150036070 con 0x55d173c49580 2024-04-25T00:33:54.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.816+0000 7f0143fff640 1 -- 172.21.15.32:0/1430608606 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f015004a450 con 0x55d173c49580 2024-04-25T00:33:54.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.817+0000 7f0162c54740 1 -- 172.21.15.32:0/1430608606 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c49580 msgr2=0x55d173c49960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.817+0000 7f0162c54740 1 --2- 172.21.15.32:0/1430608606 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c49580 0x55d173c49960 secure :-1 s=READY pgs=535 cs=0 l=1 rev1=1 crypto rx=0x7f0150009430 tx=0x7f0150031a70 comp rx=0 tx=0).stop 2024-04-25T00:33:54.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.817+0000 7f0162c54740 1 -- 172.21.15.32:0/1430608606 shutdown_connections 2024-04-25T00:33:54.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.817+0000 7f0162c54740 1 --2- 172.21.15.32:0/1430608606 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d173c54570 0x55d173c56960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.817+0000 7f0162c54740 1 --2- 172.21.15.32:0/1430608606 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d173c49ea0 0x55d173c54030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.817+0000 7f0162c54740 1 --2- 172.21.15.32:0/1430608606 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c49580 0x55d173c49960 unknown :-1 s=CLOSED pgs=535 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.817+0000 7f0162c54740 1 -- 172.21.15.32:0/1430608606 >> 172.21.15.32:0/1430608606 conn(0x55d173c458a0 msgr2=0x55d173c45c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:54.837 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.817+0000 7f0162c54740 1 -- 172.21.15.32:0/1430608606 shutdown_connections 2024-04-25T00:33:54.837 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.817+0000 7f0162c54740 1 -- 172.21.15.32:0/1430608606 wait complete. 2024-04-25T00:33:54.837 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.817+0000 7f0162c54740 1 -- start start 2024-04-25T00:33:54.837 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.817+0000 7f0162c54740 1 Processor -- start 2024-04-25T00:33:54.837 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.818+0000 7f0162c54740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d173c49580 0x55d173c53f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.837 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.818+0000 7f0162c54740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c49ea0 0x55d173c51250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.837 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.818+0000 7f0162c54740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d173c54570 0x55d173c51860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.837 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.818+0000 7f0162c54740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55d173c572a0 con 0x55d173c49ea0 2024-04-25T00:33:54.837 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.818+0000 7f0162c54740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55d173b6a050 con 0x55d173c49580 2024-04-25T00:33:54.837 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.818+0000 7f0162c54740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55d173c57420 con 0x55d173c54570 2024-04-25T00:33:54.837 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.818+0000 7f0162c54740 10 client.0 _ll_register_callbacks cb 0x55d1738c1be0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:33:54.837 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.818+0000 7f015d4dd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c49ea0 0x55d173c51250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.837 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.818+0000 7f015d4dd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c49ea0 0x55d173c51250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53564/0 (socket says 172.21.15.32:53564) 2024-04-25T00:33:54.837 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.818+0000 7f015d4dd640 1 -- 172.21.15.32:0/2959131821 learned_addr learned my addr 172.21.15.32:0/2959131821 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:54.838 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.818+0000 7f015dcde640 1 --2- 172.21.15.32:0/2959131821 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d173c49580 0x55d173c53f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.838 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.818+0000 7f015e4df640 1 --2- 172.21.15.32:0/2959131821 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d173c54570 0x55d173c51860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.838 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.818+0000 7f015d4dd640 1 -- 172.21.15.32:0/2959131821 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d173c54570 msgr2=0x55d173c51860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.838 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.818+0000 7f015d4dd640 1 --2- 172.21.15.32:0/2959131821 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d173c54570 0x55d173c51860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.838 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.818+0000 7f015d4dd640 1 -- 172.21.15.32:0/2959131821 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d173c49580 msgr2=0x55d173c53f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.838 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.818+0000 7f015d4dd640 1 --2- 172.21.15.32:0/2959131821 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d173c49580 0x55d173c53f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.838 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.818+0000 7f015d4dd640 1 -- 172.21.15.32:0/2959131821 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d173b3bdc0 con 0x55d173c49ea0 2024-04-25T00:33:54.838 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.818+0000 7f015e4df640 1 --2- 172.21.15.32:0/2959131821 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d173c54570 0x55d173c51860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:33:54.838 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.818+0000 7f015d4dd640 1 --2- 172.21.15.32:0/2959131821 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c49ea0 0x55d173c51250 secure :-1 s=READY pgs=536 cs=0 l=1 rev1=1 crypto rx=0x7f01480cd7a0 tx=0x7f014801da00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=94e5c0991bfa5b80 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.838 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.818+0000 7f01417fa640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:33:54.838 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.819+0000 7f01417fa640 1 -- 172.21.15.32:0/2959131821 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f01480cd590 con 0x55d173c49ea0 2024-04-25T00:33:54.838 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.819+0000 7f01417fa640 1 -- 172.21.15.32:0/2959131821 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f0148018b70 con 0x55d173c49ea0 2024-04-25T00:33:54.839 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.819+0000 7f0162c54740 1 -- 172.21.15.32:0/2959131821 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f0150036070 con 0x55d173c49ea0 2024-04-25T00:33:54.839 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.819+0000 7f01417fa640 1 -- 172.21.15.32:0/2959131821 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f0148025170 con 0x55d173c49ea0 2024-04-25T00:33:54.839 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.819+0000 7f0162c54740 10 client.15789 fetch_fsmap learned FSMap version 25 2024-04-25T00:33:54.839 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.819+0000 7f0162c54740 1 -- 172.21.15.32:0/2959131821 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x55d1738c23f0 con 0x55d173c49ea0 2024-04-25T00:33:54.839 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.819+0000 7f01417fa640 1 -- 172.21.15.32:0/2959131821 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0150036070 con 0x55d173c49ea0 2024-04-25T00:33:54.839 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.819+0000 7f01417fa640 1 -- 172.21.15.32:0/2959131821 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7f014809b250 con 0x55d173c49ea0 2024-04-25T00:33:54.839 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.819+0000 7f0162c54740 10 client.15789 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:33:54.839 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.819+0000 7f0162c54740 10 client.15789 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-25T00:33:54.839 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.819+0000 7f0162c54740 20 client.15789 populate_metadata read hostname 'smithi032' 2024-04-25T00:33:54.839 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.819+0000 7f0162c54740 1 --2- 172.21.15.32:0/2959131821 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55d173c64520 0x55d173c66930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.839 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.819+0000 7f0162c54740 4 client.15789 mds_command: new command op to 24481 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:33:54.839 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.819+0000 7f0162c54740 1 -- 172.21.15.32:0/2959131821 --> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55d173bbbbb0 con 0x55d173c64520 2024-04-25T00:33:54.839 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.820+0000 7f015dcde640 1 --2- 172.21.15.32:0/2959131821 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55d173c64520 0x55d173c66930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.839 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.820+0000 7f015dcde640 1 --2- 172.21.15.32:0/2959131821 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55d173c64520 0x55d173c66930 crc :-1 s=READY pgs=30 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=3f31270d6b31f685 server_cookie=8be1949c5e4365c4 in_seq=0 out_seq=0 2024-04-25T00:33:54.840 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.820+0000 7f01417fa640 10 client.15789 ms_handle_connect on v2:172.21.15.130:6838/4267125809 2024-04-25T00:33:54.840 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.821+0000 7f01417fa640 1 -- 172.21.15.32:0/2959131821 <== mds.1 v2:172.21.15.130:6838/4267125809 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x55d173bbbbb0 con 0x55d173c64520 2024-04-25T00:33:54.840 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.821+0000 7f01417fa640 10 client.15789 handle_command_reply: tid=0 2024-04-25T00:33:54.840 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.821+0000 7f0162c54740 1 client.15789 shutdown 2024-04-25T00:33:54.840 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.821+0000 7f0162c54740 1 -- 172.21.15.32:0/2959131821 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c49ea0 msgr2=0x55d173c51250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.840 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.821+0000 7f0162c54740 1 --2- 172.21.15.32:0/2959131821 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c49ea0 0x55d173c51250 secure :-1 s=READY pgs=536 cs=0 l=1 rev1=1 crypto rx=0x7f01480cd7a0 tx=0x7f014801da00 comp rx=0 tx=0).stop 2024-04-25T00:33:54.840 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.821+0000 7f0162c54740 1 -- 172.21.15.32:0/2959131821 shutdown_connections 2024-04-25T00:33:54.840 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.821+0000 7f0162c54740 1 --2- 172.21.15.32:0/2959131821 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d173c54570 0x55d173c51860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.840 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.821+0000 7f0162c54740 1 --2- 172.21.15.32:0/2959131821 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55d173c64520 0x55d173c66930 crc :-1 s=READY pgs=30 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.840 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.821+0000 7f0162c54740 1 --2- 172.21.15.32:0/2959131821 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c49ea0 0x55d173c51250 unknown :-1 s=CLOSED pgs=536 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.840 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.821+0000 7f0162c54740 1 --2- 172.21.15.32:0/2959131821 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d173c49580 0x55d173c53f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.840 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.821+0000 7f01417fa640 0 client.15789 ms_handle_reset on v2:172.21.15.130:6838/4267125809 2024-04-25T00:33:54.840 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.821+0000 7f0162c54740 1 -- 172.21.15.32:0/2959131821 >> 172.21.15.32:0/2959131821 conn(0x55d173c458a0 msgr2=0x55d173c4b6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:54.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.821+0000 7f0162c54740 1 -- 172.21.15.32:0/2959131821 shutdown_connections 2024-04-25T00:33:54.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.821+0000 7f0162c54740 1 -- 172.21.15.32:0/2959131821 wait complete. 2024-04-25T00:33:54.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.821+0000 7f0162c54740 20 client.15789 trim_cache size 0 max 16384 2024-04-25T00:33:54.842 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:54 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/4114945454' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:33:54.852 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.835+0000 7f0162c54740 1 Processor -- start 2024-04-25T00:33:54.852 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.835+0000 7f0162c54740 1 -- start start 2024-04-25T00:33:54.853 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.835+0000 7f0162c54740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c49580 0x55d173c60160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.853 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.835+0000 7f0162c54740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d173c49ea0 0x55d173c449c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.853 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.835+0000 7f0162c54740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d173c54570 0x55d173c53a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.853 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.835+0000 7f0162c54740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f01581b3670 con 0x55d173c49580 2024-04-25T00:33:54.853 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.835+0000 7f0162c54740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f01581b0f40 con 0x55d173c49ea0 2024-04-25T00:33:54.853 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.835+0000 7f0162c54740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f01581aed90 con 0x55d173c54570 2024-04-25T00:33:54.853 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.835+0000 7f015dcde640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c49580 0x55d173c60160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.853 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.835+0000 7f015dcde640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c49580 0x55d173c60160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53580/0 (socket says 172.21.15.32:53580) 2024-04-25T00:33:54.853 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.835+0000 7f015dcde640 1 -- 172.21.15.32:0/3774126122 learned_addr learned my addr 172.21.15.32:0/3774126122 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:54.853 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.835+0000 7f015d4dd640 1 --2- 172.21.15.32:0/3774126122 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d173c49ea0 0x55d173c449c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.853 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.835+0000 7f015e4df640 1 --2- 172.21.15.32:0/3774126122 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d173c54570 0x55d173c53a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.853 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.836+0000 7f015d4dd640 1 -- 172.21.15.32:0/3774126122 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d173c54570 msgr2=0x55d173c53a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.853 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.836+0000 7f015d4dd640 1 --2- 172.21.15.32:0/3774126122 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d173c54570 0x55d173c53a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.854 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.836+0000 7f015d4dd640 1 -- 172.21.15.32:0/3774126122 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c49580 msgr2=0x55d173c60160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.854 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.836+0000 7f015d4dd640 1 --2- 172.21.15.32:0/3774126122 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c49580 0x55d173c60160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.854 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.836+0000 7f015d4dd640 1 -- 172.21.15.32:0/3774126122 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d173c51bd0 con 0x55d173c49ea0 2024-04-25T00:33:54.854 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.836+0000 7f015dcde640 1 --2- 172.21.15.32:0/3774126122 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c49580 0x55d173c60160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:54.854 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.836+0000 7f015d4dd640 1 --2- 172.21.15.32:0/3774126122 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d173c49ea0 0x55d173c449c0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x55d173c52820 tx=0x7f014801f000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7bc8faa7a05e33c8 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.854 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.836+0000 7f01437fe640 1 -- 172.21.15.32:0/3774126122 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f014809b760 con 0x55d173c49ea0 2024-04-25T00:33:54.854 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.836+0000 7f01437fe640 1 -- 172.21.15.32:0/3774126122 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0148097ec0 con 0x55d173c49ea0 2024-04-25T00:33:54.854 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.836+0000 7f01437fe640 1 -- 172.21.15.32:0/3774126122 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f01480a4a70 con 0x55d173c49ea0 2024-04-25T00:33:54.854 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.837+0000 7f0162c54740 1 -- 172.21.15.32:0/3774126122 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d173c49ea0 msgr2=0x55d173c449c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.854 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.837+0000 7f0162c54740 1 --2- 172.21.15.32:0/3774126122 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d173c49ea0 0x55d173c449c0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x55d173c52820 tx=0x7f014801f000 comp rx=0 tx=0).stop 2024-04-25T00:33:54.854 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.837+0000 7f0162c54740 1 -- 172.21.15.32:0/3774126122 shutdown_connections 2024-04-25T00:33:54.854 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.837+0000 7f0162c54740 1 --2- 172.21.15.32:0/3774126122 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d173c54570 0x55d173c53a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.855 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.837+0000 7f0162c54740 1 --2- 172.21.15.32:0/3774126122 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d173c49ea0 0x55d173c449c0 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.855 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.837+0000 7f0162c54740 1 --2- 172.21.15.32:0/3774126122 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c49580 0x55d173c60160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.855 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.837+0000 7f0162c54740 1 -- 172.21.15.32:0/3774126122 >> 172.21.15.32:0/3774126122 conn(0x55d173c458a0 msgr2=0x55d173c442b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:54.855 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.837+0000 7f0162c54740 1 -- 172.21.15.32:0/3774126122 shutdown_connections 2024-04-25T00:33:54.855 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.837+0000 7f0162c54740 1 -- 172.21.15.32:0/3774126122 wait complete. 2024-04-25T00:33:54.855 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.837+0000 7f0162c54740 1 -- start start 2024-04-25T00:33:54.855 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.837+0000 7f0162c54740 1 Processor -- start 2024-04-25T00:33:54.855 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.838+0000 7f0162c54740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d173c49580 0x55d173c5cbc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.855 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.838+0000 7f0162c54740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d173c49ea0 0x55d173c57f80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.838+0000 7f0162c54740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c54570 0x55d173c586c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.838+0000 7f0162c54740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f01581af400 con 0x55d173c54570 2024-04-25T00:33:54.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.838+0000 7f0162c54740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f01581b0870 con 0x55d173c49ea0 2024-04-25T00:33:54.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.838+0000 7f0162c54740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f01581afad0 con 0x55d173c49580 2024-04-25T00:33:54.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.838+0000 7f0162c54740 10 client.0 _ll_register_callbacks cb 0x7f01480ccf70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:33:54.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.838+0000 7f015e4df640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c54570 0x55d173c586c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.838+0000 7f015e4df640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c54570 0x55d173c586c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53592/0 (socket says 172.21.15.32:53592) 2024-04-25T00:33:54.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.838+0000 7f015e4df640 1 -- 172.21.15.32:0/2376926680 learned_addr learned my addr 172.21.15.32:0/2376926680 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:54.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.838+0000 7f015d4dd640 1 --2- 172.21.15.32:0/2376926680 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d173c49ea0 0x55d173c57f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.838+0000 7f015dcde640 1 --2- 172.21.15.32:0/2376926680 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d173c49580 0x55d173c5cbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.838+0000 7f015d4dd640 1 -- 172.21.15.32:0/2376926680 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d173c49580 msgr2=0x55d173c5cbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.838+0000 7f015d4dd640 1 --2- 172.21.15.32:0/2376926680 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d173c49580 0x55d173c5cbc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.838+0000 7f015d4dd640 1 -- 172.21.15.32:0/2376926680 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c54570 msgr2=0x55d173c586c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.838+0000 7f015d4dd640 1 --2- 172.21.15.32:0/2376926680 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c54570 0x55d173c586c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.838+0000 7f015d4dd640 1 -- 172.21.15.32:0/2376926680 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d17398c7d0 con 0x55d173c49ea0 2024-04-25T00:33:54.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.838+0000 7f015d4dd640 1 --2- 172.21.15.32:0/2376926680 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d173c49ea0 0x55d173c57f80 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f01480cd1c0 tx=0x7f01480cd090 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7b929e053f404517 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.838+0000 7f01417fa640 10 client.0 ms_handle_connect on v2:172.21.15.121:3300/0 2024-04-25T00:33:54.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.839+0000 7f01417fa640 1 -- 172.21.15.32:0/2376926680 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f014809b760 con 0x55d173c49ea0 2024-04-25T00:33:54.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.839+0000 7f0162c54740 1 -- 172.21.15.32:0/2376926680 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f0148097ec0 con 0x55d173c49ea0 2024-04-25T00:33:54.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.839+0000 7f01417fa640 1 -- 172.21.15.32:0/2376926680 <== mon.2 v2:172.21.15.121:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f0148010630 con 0x55d173c49ea0 2024-04-25T00:33:54.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.839+0000 7f01417fa640 1 -- 172.21.15.32:0/2376926680 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f01480cd5a0 con 0x55d173c49ea0 2024-04-25T00:33:54.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.839+0000 7f0162c54740 10 client.25376 fetch_fsmap learned FSMap version 25 2024-04-25T00:33:54.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.839+0000 7f0162c54740 1 -- 172.21.15.32:0/2376926680 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x55d173ccaa40 con 0x55d173c49ea0 2024-04-25T00:33:54.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.839+0000 7f01417fa640 1 -- 172.21.15.32:0/2376926680 <== mon.2 v2:172.21.15.121:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0148097ec0 con 0x55d173c49ea0 2024-04-25T00:33:54.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.839+0000 7f01417fa640 1 -- 172.21.15.32:0/2376926680 <== mon.2 v2:172.21.15.121:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7f0148018a90 con 0x55d173c49ea0 2024-04-25T00:33:54.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.839+0000 7f0162c54740 10 client.25376 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:33:54.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.839+0000 7f0162c54740 10 client.25376 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-25T00:33:54.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.839+0000 7f0162c54740 20 client.25376 populate_metadata read hostname 'smithi032' 2024-04-25T00:33:54.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.839+0000 7f0162c54740 1 --2- 172.21.15.32:0/2376926680 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55d173ccaf10 0x55d173ccd330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.839+0000 7f0162c54740 4 client.25376 mds_command: new command op to 24481 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-25T00:33:54.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.839+0000 7f0162c54740 1 -- 172.21.15.32:0/2376926680 --> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55d173bb5f00 con 0x55d173ccaf10 2024-04-25T00:33:54.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.840+0000 7f015dcde640 1 --2- 172.21.15.32:0/2376926680 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55d173ccaf10 0x55d173ccd330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.840+0000 7f015dcde640 1 --2- 172.21.15.32:0/2376926680 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55d173ccaf10 0x55d173ccd330 crc :-1 s=READY pgs=31 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=2e5ac5565939015a server_cookie=8a24647b0df255cc in_seq=0 out_seq=0 2024-04-25T00:33:54.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.840+0000 7f01417fa640 10 client.25376 ms_handle_connect on v2:172.21.15.130:6838/4267125809 2024-04-25T00:33:54.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.840+0000 7f01417fa640 1 -- 172.21.15.32:0/2376926680 <== mds.1 v2:172.21.15.130:6838/4267125809 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55d173bb5f00 con 0x55d173ccaf10 2024-04-25T00:33:54.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.840+0000 7f01417fa640 10 client.25376 handle_command_reply: tid=0 2024-04-25T00:33:54.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.840+0000 7f0162c54740 1 client.25376 shutdown 2024-04-25T00:33:54.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.840+0000 7f0162c54740 1 -- 172.21.15.32:0/2376926680 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d173c49ea0 msgr2=0x55d173c57f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.840+0000 7f0162c54740 1 --2- 172.21.15.32:0/2376926680 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d173c49ea0 0x55d173c57f80 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f01480cd1c0 tx=0x7f01480cd090 comp rx=0 tx=0).stop 2024-04-25T00:33:54.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.841+0000 7f0162c54740 1 -- 172.21.15.32:0/2376926680 shutdown_connections 2024-04-25T00:33:54.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.841+0000 7f0162c54740 1 --2- 172.21.15.32:0/2376926680 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55d173ccaf10 0x55d173ccd330 crc :-1 s=READY pgs=31 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.841+0000 7f0162c54740 1 --2- 172.21.15.32:0/2376926680 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d173c54570 0x55d173c586c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.841+0000 7f01417fa640 0 client.25376 ms_handle_reset on v2:172.21.15.130:6838/4267125809 2024-04-25T00:33:54.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.841+0000 7f0162c54740 1 --2- 172.21.15.32:0/2376926680 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d173c49ea0 0x55d173c57f80 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.841+0000 7f0162c54740 1 --2- 172.21.15.32:0/2376926680 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d173c49580 0x55d173c5cbc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.841+0000 7f0162c54740 1 -- 172.21.15.32:0/2376926680 >> 172.21.15.32:0/2376926680 conn(0x55d173c458a0 msgr2=0x55d173c44b40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:54.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.841+0000 7f0162c54740 1 -- 172.21.15.32:0/2376926680 shutdown_connections 2024-04-25T00:33:54.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.841+0000 7f0162c54740 1 -- 172.21.15.32:0/2376926680 wait complete. 2024-04-25T00:33:54.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.841+0000 7f0162c54740 20 client.25376 trim_cache size 0 max 16384 2024-04-25T00:33:54.859 INFO:teuthology.orchestra.run.smithi032.stdout:[] 2024-04-25T00:33:54.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.841+0000 7f015ff42640 1 -- 172.21.15.32:0/2611677667 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f01440a2dc0 msgr2=0x7f01440a5280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.841+0000 7f015ff42640 1 --2- 172.21.15.32:0/2611677667 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f01440a2dc0 0x7f01440a5280 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f0150047000 tx=0x7f015004a000 comp rx=0 tx=0).stop 2024-04-25T00:33:54.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.841+0000 7f015ff42640 1 -- 172.21.15.32:0/2611677667 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0158100fd0 msgr2=0x7f01581027f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.841+0000 7f015ff42640 1 --2- 172.21.15.32:0/2611677667 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0158100fd0 0x7f01581027f0 secure :-1 s=READY pgs=534 cs=0 l=1 rev1=1 crypto rx=0x7f014800e9f0 tx=0x7f014800eeb0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.841+0000 7f015ff42640 1 -- 172.21.15.32:0/2611677667 shutdown_connections 2024-04-25T00:33:54.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.841+0000 7f015ff42640 1 --2- 172.21.15.32:0/2611677667 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f01440a2dc0 0x7f01440a5280 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.841+0000 7f015ff42640 1 --2- 172.21.15.32:0/2611677667 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f01581066b0 0x7f0158102d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.841+0000 7f015ff42640 1 --2- 172.21.15.32:0/2611677667 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0158100fd0 0x7f01581027f0 unknown :-1 s=CLOSED pgs=534 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.841+0000 7f015ff42640 1 --2- 172.21.15.32:0/2611677667 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0158100610 0x7f01581022b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.841+0000 7f015ff42640 1 -- 172.21.15.32:0/2611677667 >> 172.21.15.32:0/2611677667 conn(0x7f0158077740 msgr2=0x7f01580ff840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:54.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.841+0000 7f015ff42640 1 -- 172.21.15.32:0/2611677667 shutdown_connections 2024-04-25T00:33:54.861 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.841+0000 7f015ff42640 1 -- 172.21.15.32:0/2611677667 wait complete. 2024-04-25T00:33:54.861 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:54 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/4114945454' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:33:54.862 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-25T00:33:54.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:54 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/4114945454' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:33:54.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.968+0000 7fe5555f9640 1 -- 172.21.15.32:0/2437430474 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe550100fd0 msgr2=0x7fe55010d4e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.968+0000 7fe5555f9640 1 --2- 172.21.15.32:0/2437430474 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe550100fd0 0x7fe55010d4e0 secure :-1 s=READY pgs=537 cs=0 l=1 rev1=1 crypto rx=0x7fe548009cd0 tx=0x7fe54802a8e0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.969+0000 7fe5555f9640 1 -- 172.21.15.32:0/2437430474 shutdown_connections 2024-04-25T00:33:54.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.969+0000 7fe5555f9640 1 --2- 172.21.15.32:0/2437430474 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe550100fd0 0x7fe55010d4e0 unknown :-1 s=CLOSED pgs=537 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.969+0000 7fe5555f9640 1 --2- 172.21.15.32:0/2437430474 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe550100610 0x7fe550100a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.969+0000 7fe5555f9640 1 --2- 172.21.15.32:0/2437430474 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe5501066b0 0x7fe550106a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.969+0000 7fe5555f9640 1 -- 172.21.15.32:0/2437430474 >> 172.21.15.32:0/2437430474 conn(0x7fe550077740 msgr2=0x7fe550077b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:54.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.969+0000 7fe5555f9640 1 -- 172.21.15.32:0/2437430474 shutdown_connections 2024-04-25T00:33:54.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.969+0000 7fe5555f9640 1 -- 172.21.15.32:0/2437430474 wait complete. 2024-04-25T00:33:54.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.969+0000 7fe5555f9640 1 Processor -- start 2024-04-25T00:33:54.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.969+0000 7fe5555f9640 1 -- start start 2024-04-25T00:33:54.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.970+0000 7fe5555f9640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe550100610 0x7fe550197ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.970+0000 7fe5555f9640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe550100fd0 0x7fe5501983e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.970+0000 7fe5555f9640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe5501066b0 0x7fe55019c760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.970+0000 7fe5555f9640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fe5501109f0 con 0x7fe550100610 2024-04-25T00:33:54.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.970+0000 7fe5555f9640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fe550110870 con 0x7fe5501066b0 2024-04-25T00:33:54.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.970+0000 7fe5555f9640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fe550110b70 con 0x7fe550100fd0 2024-04-25T00:33:54.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.970+0000 7fe54effd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe550100610 0x7fe550197ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.970+0000 7fe54effd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe550100610 0x7fe550197ea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53606/0 (socket says 172.21.15.32:53606) 2024-04-25T00:33:54.973 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.970+0000 7fe54effd640 1 -- 172.21.15.32:0/3047091281 learned_addr learned my addr 172.21.15.32:0/3047091281 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:54.973 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.970+0000 7fe54f7fe640 1 --2- 172.21.15.32:0/3047091281 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe5501066b0 0x7fe55019c760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.973 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.970+0000 7fe5467fc640 1 --2- 172.21.15.32:0/3047091281 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe550100fd0 0x7fe5501983e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.973 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.970+0000 7fe54effd640 1 -- 172.21.15.32:0/3047091281 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe550100fd0 msgr2=0x7fe5501983e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.973 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.970+0000 7fe54effd640 1 --2- 172.21.15.32:0/3047091281 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe550100fd0 0x7fe5501983e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.973 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.970+0000 7fe54effd640 1 -- 172.21.15.32:0/3047091281 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe5501066b0 msgr2=0x7fe55019c760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.973 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.970+0000 7fe54effd640 1 --2- 172.21.15.32:0/3047091281 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe5501066b0 0x7fe55019c760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.973 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.970+0000 7fe54effd640 1 -- 172.21.15.32:0/3047091281 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe55019ce90 con 0x7fe550100610 2024-04-25T00:33:54.973 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.970+0000 7fe54f7fe640 1 --2- 172.21.15.32:0/3047091281 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe5501066b0 0x7fe55019c760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:54.973 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.970+0000 7fe54effd640 1 --2- 172.21.15.32:0/3047091281 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe550100610 0x7fe550197ea0 secure :-1 s=READY pgs=538 cs=0 l=1 rev1=1 crypto rx=0x7fe54000c970 tx=0x7fe54000ce30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.973 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.970+0000 7fe54cff9640 1 -- 172.21.15.32:0/3047091281 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe540007c10 con 0x7fe550100610 2024-04-25T00:33:54.973 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.970+0000 7fe54cff9640 1 -- 172.21.15.32:0/3047091281 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe540007db0 con 0x7fe550100610 2024-04-25T00:33:54.973 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.970+0000 7fe5555f9640 1 -- 172.21.15.32:0/3047091281 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe55019d180 con 0x7fe550100610 2024-04-25T00:33:54.973 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.971+0000 7fe54cff9640 1 -- 172.21.15.32:0/3047091281 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe54000f940 con 0x7fe550100610 2024-04-25T00:33:54.974 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.971+0000 7fe5555f9640 1 -- 172.21.15.32:0/3047091281 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe5501a4a10 con 0x7fe550100610 2024-04-25T00:33:54.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.973+0000 7fe54cff9640 1 -- 172.21.15.32:0/3047091281 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 24) ==== 138009+0+0 (secure 0 0 0) 0x7fe54000fae0 con 0x7fe550100610 2024-04-25T00:33:54.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.974+0000 7fe5583e7740 1 Processor -- start 2024-04-25T00:33:54.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.974+0000 7fe5583e7740 1 -- start start 2024-04-25T00:33:54.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.974+0000 7fe5583e7740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cf0580 0x5566e9cf0960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.974+0000 7fe5583e7740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5566e9cf0ea0 0x5566e9cfb030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.974+0000 7fe5583e7740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e9cfb570 0x5566e9cfd960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.974+0000 7fe5583e7740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5566e9c55670 con 0x5566e9cf0580 2024-04-25T00:33:54.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.974+0000 7fe5583e7740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5566e9bfd970 con 0x5566e9cf0ea0 2024-04-25T00:33:54.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.974+0000 7fe5583e7740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5566e9c51630 con 0x5566e9cfb570 2024-04-25T00:33:54.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.974+0000 7fe54f7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cf0580 0x5566e9cf0960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.974+0000 7fe54f7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cf0580 0x5566e9cf0960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53612/0 (socket says 172.21.15.32:53612) 2024-04-25T00:33:54.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.974+0000 7fe54f7fe640 1 -- 172.21.15.32:0/1792555126 learned_addr learned my addr 172.21.15.32:0/1792555126 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:54.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.974+0000 7fe54effd640 1 --2- 172.21.15.32:0/1792555126 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5566e9cf0ea0 0x5566e9cfb030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.974+0000 7fe54cff9640 1 --2- 172.21.15.32:0/3047091281 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe53c0a2d90 0x7fe53c0a5250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.974+0000 7fe5467fc640 1 --2- 172.21.15.32:0/1792555126 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e9cfb570 0x5566e9cfd960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.975+0000 7fe54f7fe640 1 -- 172.21.15.32:0/1792555126 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e9cfb570 msgr2=0x5566e9cfd960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.975+0000 7fe54f7fe640 1 --2- 172.21.15.32:0/1792555126 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e9cfb570 0x5566e9cfd960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.975+0000 7fe54f7fe640 1 -- 172.21.15.32:0/1792555126 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5566e9cf0ea0 msgr2=0x5566e9cfb030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.975+0000 7fe54f7fe640 1 --2- 172.21.15.32:0/1792555126 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5566e9cf0ea0 0x5566e9cfb030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.975+0000 7fe54f7fe640 1 -- 172.21.15.32:0/1792555126 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5566e9c21400 con 0x5566e9cf0580 2024-04-25T00:33:54.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.975+0000 7fe54effd640 1 --2- 172.21.15.32:0/1792555126 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5566e9cf0ea0 0x5566e9cfb030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:54.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.975+0000 7fe54cff9640 1 -- 172.21.15.32:0/3047091281 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fe5400cfce0 con 0x7fe550100610 2024-04-25T00:33:54.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.975+0000 7fe54f7fe640 1 --2- 172.21.15.32:0/3047091281 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe53c0a2d90 0x7fe53c0a5250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.975+0000 7fe54f7fe640 1 --2- 172.21.15.32:0/1792555126 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cf0580 0x5566e9cf0960 secure :-1 s=READY pgs=539 cs=0 l=1 rev1=1 crypto rx=0x7fe548035000 tx=0x7fe548031990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b2db2b7bda9dca7f server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.975+0000 7fe5457fa640 1 -- 172.21.15.32:0/1792555126 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe548041040 con 0x5566e9cf0580 2024-04-25T00:33:54.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.975+0000 7fe5457fa640 1 -- 172.21.15.32:0/1792555126 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe548043070 con 0x5566e9cf0580 2024-04-25T00:33:54.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.975+0000 7fe5457fa640 1 -- 172.21.15.32:0/1792555126 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe548047630 con 0x5566e9cf0580 2024-04-25T00:33:54.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.976+0000 7fe5583e7740 1 -- 172.21.15.32:0/1792555126 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cf0580 msgr2=0x5566e9cf0960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.976+0000 7fe5583e7740 1 --2- 172.21.15.32:0/1792555126 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cf0580 0x5566e9cf0960 secure :-1 s=READY pgs=539 cs=0 l=1 rev1=1 crypto rx=0x7fe548035000 tx=0x7fe548031990 comp rx=0 tx=0).stop 2024-04-25T00:33:54.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.976+0000 7fe54f7fe640 1 --2- 172.21.15.32:0/3047091281 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe53c0a2d90 0x7fe53c0a5250 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7fe54804c000 tx=0x7fe54804d000 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.976+0000 7fe5583e7740 1 -- 172.21.15.32:0/1792555126 shutdown_connections 2024-04-25T00:33:54.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.976+0000 7fe5583e7740 1 --2- 172.21.15.32:0/1792555126 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e9cfb570 0x5566e9cfd960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.976+0000 7fe5583e7740 1 --2- 172.21.15.32:0/1792555126 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5566e9cf0ea0 0x5566e9cfb030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.976+0000 7fe5583e7740 1 --2- 172.21.15.32:0/1792555126 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cf0580 0x5566e9cf0960 unknown :-1 s=CLOSED pgs=539 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.976+0000 7fe5583e7740 1 -- 172.21.15.32:0/1792555126 >> 172.21.15.32:0/1792555126 conn(0x5566e9cec8a0 msgr2=0x5566e9cecc80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:54.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.976+0000 7fe5583e7740 1 -- 172.21.15.32:0/1792555126 shutdown_connections 2024-04-25T00:33:54.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.976+0000 7fe5583e7740 1 -- 172.21.15.32:0/1792555126 wait complete. 2024-04-25T00:33:54.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.976+0000 7fe5583e7740 1 -- start start 2024-04-25T00:33:54.985 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.977+0000 7fe5583e7740 1 Processor -- start 2024-04-25T00:33:54.985 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.977+0000 7fe5583e7740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5566e9cf0580 0x5566e9cfaec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.985 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.977+0000 7fe5583e7740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cf0ea0 0x5566e9cf81b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.985 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.977+0000 7fe5583e7740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e9cfb570 0x5566e9cf8710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.985 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.977+0000 7fe5583e7740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5566e9cfe360 con 0x5566e9cf0ea0 2024-04-25T00:33:54.985 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.977+0000 7fe5583e7740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5566e9c11050 con 0x5566e9cf0580 2024-04-25T00:33:54.985 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.977+0000 7fe5583e7740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5566e9cfe4e0 con 0x5566e9cfb570 2024-04-25T00:33:54.985 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.977+0000 7fe5583e7740 10 client.0 _ll_register_callbacks cb 0x5566e9968be0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:33:54.985 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.977+0000 7fe5467fc640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cf0ea0 0x5566e9cf81b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.985 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.977+0000 7fe54effd640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5566e9cf0580 0x5566e9cfaec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.985 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.977+0000 7fe5467fc640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cf0ea0 0x5566e9cf81b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53624/0 (socket says 172.21.15.32:53624) 2024-04-25T00:33:54.985 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.977+0000 7fe5467fc640 1 -- 172.21.15.32:0/4077932310 learned_addr learned my addr 172.21.15.32:0/4077932310 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:54.986 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.977+0000 7fe54f7fe640 1 --2- 172.21.15.32:0/4077932310 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e9cfb570 0x5566e9cf8710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.986 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.977+0000 7fe54effd640 1 -- 172.21.15.32:0/4077932310 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e9cfb570 msgr2=0x5566e9cf8710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.986 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.977+0000 7fe54effd640 1 --2- 172.21.15.32:0/4077932310 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e9cfb570 0x5566e9cf8710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.986 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.977+0000 7fe54effd640 1 -- 172.21.15.32:0/4077932310 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cf0ea0 msgr2=0x5566e9cf81b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.986 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.977+0000 7fe54effd640 1 --2- 172.21.15.32:0/4077932310 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cf0ea0 0x5566e9cf81b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.986 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.977+0000 7fe54effd640 1 -- 172.21.15.32:0/4077932310 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5566e9be2dc0 con 0x5566e9cf0580 2024-04-25T00:33:54.986 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.977+0000 7fe5467fc640 1 --2- 172.21.15.32:0/4077932310 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cf0ea0 0x5566e9cf81b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:54.986 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.977+0000 7fe54effd640 1 --2- 172.21.15.32:0/4077932310 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5566e9cf0580 0x5566e9cfaec0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7fe5400ccfc0 tx=0x7fe54009b970 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e880610766c6a34b server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.986 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.977+0000 7fe5367fc640 10 client.0 ms_handle_connect on v2:172.21.15.121:3300/0 2024-04-25T00:33:54.986 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.978+0000 7fe5367fc640 1 -- 172.21.15.32:0/4077932310 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe54009c470 con 0x5566e9cf0580 2024-04-25T00:33:54.986 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.978+0000 7fe5367fc640 1 -- 172.21.15.32:0/4077932310 <== mon.2 v2:172.21.15.121:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fe5400cd270 con 0x5566e9cf0580 2024-04-25T00:33:54.986 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.978+0000 7fe5583e7740 1 -- 172.21.15.32:0/4077932310 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe548043070 con 0x5566e9cf0580 2024-04-25T00:33:54.987 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.978+0000 7fe5367fc640 1 -- 172.21.15.32:0/4077932310 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7fe5400cd470 con 0x5566e9cf0580 2024-04-25T00:33:54.987 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.978+0000 7fe5583e7740 10 client.25388 fetch_fsmap learned FSMap version 25 2024-04-25T00:33:54.987 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.978+0000 7fe5583e7740 1 -- 172.21.15.32:0/4077932310 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x5566e99693f0 con 0x5566e9cf0580 2024-04-25T00:33:54.987 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.978+0000 7fe5367fc640 1 -- 172.21.15.32:0/4077932310 <== mon.2 v2:172.21.15.121:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe548043070 con 0x5566e9cf0580 2024-04-25T00:33:54.987 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.978+0000 7fe5367fc640 1 -- 172.21.15.32:0/4077932310 <== mon.2 v2:172.21.15.121:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7fe54009b1c0 con 0x5566e9cf0580 2024-04-25T00:33:54.987 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.978+0000 7fe5583e7740 10 client.25388 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:33:54.987 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.978+0000 7fe5583e7740 10 client.25388 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:33:54.987 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.978+0000 7fe5583e7740 20 client.25388 populate_metadata read hostname 'smithi032' 2024-04-25T00:33:54.987 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.978+0000 7fe5583e7740 1 --2- 172.21.15.32:0/4077932310 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5566e9d0b670 0x5566e9d0da60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.987 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.978+0000 7fe5583e7740 4 client.25388 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:33:54.987 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.978+0000 7fe5583e7740 1 -- 172.21.15.32:0/4077932310 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5566e9c62bb0 con 0x5566e9d0b670 2024-04-25T00:33:54.987 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.979+0000 7fe5467fc640 1 --2- 172.21.15.32:0/4077932310 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5566e9d0b670 0x5566e9d0da60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.987 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.979+0000 7fe5367fc640 10 client.25388 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:54.987 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.979+0000 7fe5467fc640 1 --2- 172.21.15.32:0/4077932310 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5566e9d0b670 0x5566e9d0da60 crc :-1 s=READY pgs=46 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=de12c3614b6ad2f7 server_cookie=5a7b6bfdcff194d5 in_seq=0 out_seq=0 2024-04-25T00:33:54.987 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.979+0000 7fe5367fc640 1 -- 172.21.15.32:0/4077932310 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x5566e9c62bb0 con 0x5566e9d0b670 2024-04-25T00:33:54.987 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.979+0000 7fe5367fc640 10 client.25388 handle_command_reply: tid=0 2024-04-25T00:33:54.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.980+0000 7fe5583e7740 1 client.25388 shutdown 2024-04-25T00:33:54.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.980+0000 7fe5583e7740 1 -- 172.21.15.32:0/4077932310 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5566e9cf0580 msgr2=0x5566e9cfaec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:54.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.980+0000 7fe5583e7740 1 --2- 172.21.15.32:0/4077932310 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5566e9cf0580 0x5566e9cfaec0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7fe5400ccfc0 tx=0x7fe54009b970 comp rx=0 tx=0).stop 2024-04-25T00:33:54.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.980+0000 7fe5583e7740 1 -- 172.21.15.32:0/4077932310 shutdown_connections 2024-04-25T00:33:54.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.980+0000 7fe5583e7740 1 --2- 172.21.15.32:0/4077932310 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5566e9d0b670 0x5566e9d0da60 crc :-1 s=READY pgs=46 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.980+0000 7fe5583e7740 1 --2- 172.21.15.32:0/4077932310 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e9cfb570 0x5566e9cf8710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.980+0000 7fe5367fc640 0 client.25388 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:54.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.980+0000 7fe5583e7740 1 --2- 172.21.15.32:0/4077932310 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cf0ea0 0x5566e9cf81b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.980+0000 7fe5583e7740 1 --2- 172.21.15.32:0/4077932310 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5566e9cf0580 0x5566e9cfaec0 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.980+0000 7fe5583e7740 1 -- 172.21.15.32:0/4077932310 >> 172.21.15.32:0/4077932310 conn(0x5566e9cec8a0 msgr2=0x5566e9cebb40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:54.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.980+0000 7fe5583e7740 1 -- 172.21.15.32:0/4077932310 shutdown_connections 2024-04-25T00:33:54.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.980+0000 7fe5583e7740 1 -- 172.21.15.32:0/4077932310 wait complete. 2024-04-25T00:33:54.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.980+0000 7fe5583e7740 20 client.25388 trim_cache size 0 max 16384 2024-04-25T00:33:54.995 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.993+0000 7fe5583e7740 1 Processor -- start 2024-04-25T00:33:54.995 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.993+0000 7fe5583e7740 1 -- start start 2024-04-25T00:33:54.995 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.994+0000 7fe5583e7740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5566e9cf0580 0x5566e9cfa9f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.995 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.994+0000 7fe5583e7740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e9cf0ea0 0x5566e9cf23e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.994+0000 7fe5583e7740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cfb570 0x5566e9d13300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:54.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.994+0000 7fe5583e7740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fe5501b13b0 con 0x5566e9cfb570 2024-04-25T00:33:54.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.994+0000 7fe5583e7740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fe5501aec80 con 0x5566e9cf0580 2024-04-25T00:33:54.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.994+0000 7fe5583e7740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fe5501acad0 con 0x5566e9cf0ea0 2024-04-25T00:33:54.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.994+0000 7fe54f7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cfb570 0x5566e9d13300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.994+0000 7fe54f7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cfb570 0x5566e9d13300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53638/0 (socket says 172.21.15.32:53638) 2024-04-25T00:33:54.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.994+0000 7fe54f7fe640 1 -- 172.21.15.32:0/4122786899 learned_addr learned my addr 172.21.15.32:0/4122786899 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:54.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.994+0000 7fe54effd640 1 --2- 172.21.15.32:0/4122786899 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5566e9cf0580 0x5566e9cfa9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.994+0000 7fe5467fc640 1 --2- 172.21.15.32:0/4122786899 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e9cf0ea0 0x5566e9cf23e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:54.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.994+0000 7fe54f7fe640 1 -- 172.21.15.32:0/4122786899 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e9cf0ea0 msgr2=0x5566e9cf23e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.994+0000 7fe54f7fe640 1 --2- 172.21.15.32:0/4122786899 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e9cf0ea0 0x5566e9cf23e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.994+0000 7fe54f7fe640 1 -- 172.21.15.32:0/4122786899 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5566e9cf0580 msgr2=0x5566e9cfa9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:54.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.994+0000 7fe54f7fe640 1 --2- 172.21.15.32:0/4122786899 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5566e9cf0580 0x5566e9cfa9f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:54.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.994+0000 7fe54f7fe640 1 -- 172.21.15.32:0/4122786899 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5566e9a337d0 con 0x5566e9cfb570 2024-04-25T00:33:54.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.994+0000 7fe54f7fe640 1 --2- 172.21.15.32:0/4122786899 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cfb570 0x5566e9d13300 secure :-1 s=READY pgs=540 cs=0 l=1 rev1=1 crypto rx=0x7fe548035000 tx=0x7fe54803e000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=db0a57281b9a5c61 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:54.997 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.995+0000 7fe544ff9640 1 -- 172.21.15.32:0/4122786899 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe54805f070 con 0x5566e9cfb570 2024-04-25T00:33:54.997 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.995+0000 7fe544ff9640 1 -- 172.21.15.32:0/4122786899 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe548036050 con 0x5566e9cfb570 2024-04-25T00:33:54.997 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.995+0000 7fe544ff9640 1 -- 172.21.15.32:0/4122786899 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe548041040 con 0x5566e9cfb570 2024-04-25T00:33:55.001 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.995+0000 7fe5583e7740 1 -- 172.21.15.32:0/4122786899 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cfb570 msgr2=0x5566e9d13300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:55.001 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.995+0000 7fe5583e7740 1 --2- 172.21.15.32:0/4122786899 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cfb570 0x5566e9d13300 secure :-1 s=READY pgs=540 cs=0 l=1 rev1=1 crypto rx=0x7fe548035000 tx=0x7fe54803e000 comp rx=0 tx=0).stop 2024-04-25T00:33:55.002 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.995+0000 7fe5583e7740 1 -- 172.21.15.32:0/4122786899 shutdown_connections 2024-04-25T00:33:55.002 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.995+0000 7fe5583e7740 1 --2- 172.21.15.32:0/4122786899 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cfb570 0x5566e9d13300 unknown :-1 s=CLOSED pgs=540 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:55.002 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.995+0000 7fe5583e7740 1 --2- 172.21.15.32:0/4122786899 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e9cf0ea0 0x5566e9cf23e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:55.002 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.996+0000 7fe5583e7740 1 --2- 172.21.15.32:0/4122786899 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5566e9cf0580 0x5566e9cfa9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:55.002 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.996+0000 7fe5583e7740 1 -- 172.21.15.32:0/4122786899 >> 172.21.15.32:0/4122786899 conn(0x5566e9cec8a0 msgr2=0x5566e9cf1cd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:55.002 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.996+0000 7fe5583e7740 1 -- 172.21.15.32:0/4122786899 shutdown_connections 2024-04-25T00:33:55.002 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.996+0000 7fe5583e7740 1 -- 172.21.15.32:0/4122786899 wait complete. 2024-04-25T00:33:55.002 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.996+0000 7fe5583e7740 1 -- start start 2024-04-25T00:33:55.002 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.996+0000 7fe5583e7740 1 Processor -- start 2024-04-25T00:33:55.002 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.996+0000 7fe5583e7740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e9cf0580 0x5566e9cf4e80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:55.002 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.996+0000 7fe5583e7740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cf0ea0 0x5566e9cf53c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:55.002 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.996+0000 7fe5583e7740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5566e9cfb570 0x5566e9cf5900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:55.002 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.996+0000 7fe5583e7740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fe5501ad140 con 0x5566e9cf0ea0 2024-04-25T00:33:55.003 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.996+0000 7fe5583e7740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fe5501ae5b0 con 0x5566e9cfb570 2024-04-25T00:33:55.003 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.996+0000 7fe5583e7740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fe5501ad810 con 0x5566e9cf0580 2024-04-25T00:33:55.003 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.996+0000 7fe5583e7740 10 client.0 _ll_register_callbacks cb 0x7fe540018e80 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:33:55.003 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.996+0000 7fe5467fc640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cf0ea0 0x5566e9cf53c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:55.003 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.996+0000 7fe5467fc640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cf0ea0 0x5566e9cf53c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53652/0 (socket says 172.21.15.32:53652) 2024-04-25T00:33:55.003 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.996+0000 7fe5467fc640 1 -- 172.21.15.32:0/4018198868 learned_addr learned my addr 172.21.15.32:0/4018198868 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:55.003 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.996+0000 7fe54f7fe640 1 --2- 172.21.15.32:0/4018198868 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5566e9cfb570 0x5566e9cf5900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:55.003 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.996+0000 7fe54effd640 1 --2- 172.21.15.32:0/4018198868 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e9cf0580 0x5566e9cf4e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:55.003 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.997+0000 7fe5467fc640 1 -- 172.21.15.32:0/4018198868 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e9cf0580 msgr2=0x5566e9cf4e80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:55.003 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.997+0000 7fe5467fc640 1 --2- 172.21.15.32:0/4018198868 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e9cf0580 0x5566e9cf4e80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:55.003 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.997+0000 7fe5467fc640 1 -- 172.21.15.32:0/4018198868 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5566e9cfb570 msgr2=0x5566e9cf5900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:55.003 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.997+0000 7fe5467fc640 1 --2- 172.21.15.32:0/4018198868 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5566e9cfb570 0x5566e9cf5900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:55.003 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.997+0000 7fe5467fc640 1 -- 172.21.15.32:0/4018198868 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5566e9d70ba0 con 0x5566e9cf0ea0 2024-04-25T00:33:55.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.997+0000 7fe54f7fe640 1 --2- 172.21.15.32:0/4018198868 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5566e9cfb570 0x5566e9cf5900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:33:55.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.997+0000 7fe5467fc640 1 --2- 172.21.15.32:0/4018198868 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cf0ea0 0x5566e9cf53c0 secure :-1 s=READY pgs=541 cs=0 l=1 rev1=1 crypto rx=0x7fe538009460 tx=0x7fe538012d10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=50eefa392f20d917 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:55.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.997+0000 7fe5367fc640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:33:55.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.997+0000 7fe5367fc640 1 -- 172.21.15.32:0/4018198868 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe538005220 con 0x5566e9cf0ea0 2024-04-25T00:33:55.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.997+0000 7fe5367fc640 1 -- 172.21.15.32:0/4018198868 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fe5380053c0 con 0x5566e9cf0ea0 2024-04-25T00:33:55.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.997+0000 7fe5583e7740 1 -- 172.21.15.32:0/4018198868 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe548036050 con 0x5566e9cf0ea0 2024-04-25T00:33:55.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.997+0000 7fe5367fc640 1 -- 172.21.15.32:0/4018198868 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7fe538031050 con 0x5566e9cf0ea0 2024-04-25T00:33:55.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.997+0000 7fe5583e7740 10 client.15813 fetch_fsmap learned FSMap version 25 2024-04-25T00:33:55.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.997+0000 7fe5583e7740 1 -- 172.21.15.32:0/4018198868 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x5566e9d71ee0 con 0x5566e9cf0ea0 2024-04-25T00:33:55.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.997+0000 7fe5367fc640 1 -- 172.21.15.32:0/4018198868 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe548036050 con 0x5566e9cf0ea0 2024-04-25T00:33:55.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.997+0000 7fe5367fc640 1 -- 172.21.15.32:0/4018198868 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7fe538004a00 con 0x5566e9cf0ea0 2024-04-25T00:33:55.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.998+0000 7fe5583e7740 10 client.15813 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:33:55.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.998+0000 7fe5583e7740 10 client.15813 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:33:55.005 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.998+0000 7fe5583e7740 20 client.15813 populate_metadata read hostname 'smithi032' 2024-04-25T00:33:55.005 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.998+0000 7fe5583e7740 1 --2- 172.21.15.32:0/4018198868 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5566e9d72630 0x5566e9d74a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:55.005 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.998+0000 7fe5583e7740 4 client.15813 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-25T00:33:55.005 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.998+0000 7fe5583e7740 1 -- 172.21.15.32:0/4018198868 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "damage ls"}) -- 0x5566e9c5cf00 con 0x5566e9d72630 2024-04-25T00:33:55.005 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.998+0000 7fe54effd640 1 --2- 172.21.15.32:0/4018198868 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5566e9d72630 0x5566e9d74a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:55.005 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.998+0000 7fe5367fc640 10 client.15813 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:55.005 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.998+0000 7fe54effd640 1 --2- 172.21.15.32:0/4018198868 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5566e9d72630 0x5566e9d74a50 crc :-1 s=READY pgs=47 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9c276cc8d012b8d2 server_cookie=c9158e51a81236cf in_seq=0 out_seq=0 2024-04-25T00:33:55.005 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.998+0000 7fe5367fc640 1 -- 172.21.15.32:0/4018198868 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x5566e9c5cf00 con 0x5566e9d72630 2024-04-25T00:33:55.005 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.998+0000 7fe5367fc640 10 client.15813 handle_command_reply: tid=0 2024-04-25T00:33:55.005 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.998+0000 7fe5583e7740 1 client.15813 shutdown 2024-04-25T00:33:55.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.998+0000 7fe5583e7740 1 -- 172.21.15.32:0/4018198868 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cf0ea0 msgr2=0x5566e9cf53c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:55.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.998+0000 7fe5583e7740 1 --2- 172.21.15.32:0/4018198868 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cf0ea0 0x5566e9cf53c0 secure :-1 s=READY pgs=541 cs=0 l=1 rev1=1 crypto rx=0x7fe538009460 tx=0x7fe538012d10 comp rx=0 tx=0).stop 2024-04-25T00:33:55.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.998+0000 7fe5583e7740 1 -- 172.21.15.32:0/4018198868 shutdown_connections 2024-04-25T00:33:55.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.998+0000 7fe5583e7740 1 --2- 172.21.15.32:0/4018198868 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5566e9d72630 0x5566e9d74a50 crc :-1 s=READY pgs=47 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:55.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.998+0000 7fe5583e7740 1 --2- 172.21.15.32:0/4018198868 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5566e9cfb570 0x5566e9cf5900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:55.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.998+0000 7fe5583e7740 1 --2- 172.21.15.32:0/4018198868 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5566e9cf0ea0 0x5566e9cf53c0 unknown :-1 s=CLOSED pgs=541 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:55.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.998+0000 7fe5367fc640 0 client.15813 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:55.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.998+0000 7fe5583e7740 1 --2- 172.21.15.32:0/4018198868 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e9cf0580 0x5566e9cf4e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:55.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.999+0000 7fe5583e7740 1 -- 172.21.15.32:0/4018198868 >> 172.21.15.32:0/4018198868 conn(0x5566e9cec8a0 msgr2=0x5566e9cf2560 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:55.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.999+0000 7fe5583e7740 1 -- 172.21.15.32:0/4018198868 shutdown_connections 2024-04-25T00:33:55.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.999+0000 7fe5583e7740 1 -- 172.21.15.32:0/4018198868 wait complete. 2024-04-25T00:33:55.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.999+0000 7fe5583e7740 20 client.15813 trim_cache size 0 max 16384 2024-04-25T00:33:55.007 INFO:teuthology.orchestra.run.smithi032.stdout:[] 2024-04-25T00:33:55.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.999+0000 7fe5555f9640 1 -- 172.21.15.32:0/3047091281 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe53c0a2d90 msgr2=0x7fe53c0a5250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:55.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.999+0000 7fe5555f9640 1 --2- 172.21.15.32:0/3047091281 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe53c0a2d90 0x7fe53c0a5250 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7fe54804c000 tx=0x7fe54804d000 comp rx=0 tx=0).stop 2024-04-25T00:33:55.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.999+0000 7fe5555f9640 1 -- 172.21.15.32:0/3047091281 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe550100610 msgr2=0x7fe550197ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:55.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.999+0000 7fe5555f9640 1 --2- 172.21.15.32:0/3047091281 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe550100610 0x7fe550197ea0 secure :-1 s=READY pgs=538 cs=0 l=1 rev1=1 crypto rx=0x7fe54000c970 tx=0x7fe54000ce30 comp rx=0 tx=0).stop 2024-04-25T00:33:55.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.999+0000 7fe5555f9640 1 -- 172.21.15.32:0/3047091281 shutdown_connections 2024-04-25T00:33:55.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.999+0000 7fe5555f9640 1 --2- 172.21.15.32:0/3047091281 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe53c0a2d90 0x7fe53c0a5250 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:55.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.999+0000 7fe5555f9640 1 --2- 172.21.15.32:0/3047091281 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe5501066b0 0x7fe55019c760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:55.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.999+0000 7fe5555f9640 1 --2- 172.21.15.32:0/3047091281 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe550100fd0 0x7fe5501983e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:55.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.999+0000 7fe5555f9640 1 --2- 172.21.15.32:0/3047091281 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe550100610 0x7fe550197ea0 unknown :-1 s=CLOSED pgs=538 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:55.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.999+0000 7fe5555f9640 1 -- 172.21.15.32:0/3047091281 >> 172.21.15.32:0/3047091281 conn(0x7fe550077740 msgr2=0x7fe550104590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:55.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.999+0000 7fe5555f9640 1 -- 172.21.15.32:0/3047091281 shutdown_connections 2024-04-25T00:33:55.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:54.999+0000 7fe5555f9640 1 -- 172.21.15.32:0/3047091281 wait complete. 2024-04-25T00:33:55.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:55 smithi121 ceph-mon[32231]: pgmap v21: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:33:55.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:55 smithi130 ceph-mon[32525]: pgmap v21: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:33:55.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:55 smithi032 ceph-mon[29357]: pgmap v21: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:33:56.011 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-25T00:33:56.011 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:33:56.012 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-25T00:33:56.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.119+0000 7f22e7990640 1 -- 172.21.15.32:0/3261694852 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22e01066b0 msgr2=0x7f22e0106a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:56.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.119+0000 7f22e7990640 1 --2- 172.21.15.32:0/3261694852 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22e01066b0 0x7f22e0106a90 secure :-1 s=READY pgs=542 cs=0 l=1 rev1=1 crypto rx=0x7f22dc009930 tx=0x7f22dc02a420 comp rx=0 tx=0).stop 2024-04-25T00:33:56.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.119+0000 7f22e7990640 1 -- 172.21.15.32:0/3261694852 shutdown_connections 2024-04-25T00:33:56.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.119+0000 7f22e7990640 1 --2- 172.21.15.32:0/3261694852 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22e0100fd0 0x7f22e010d4e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.119+0000 7f22e7990640 1 --2- 172.21.15.32:0/3261694852 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f22e0100610 0x7f22e0100a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.119+0000 7f22e7990640 1 --2- 172.21.15.32:0/3261694852 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22e01066b0 0x7f22e0106a90 unknown :-1 s=CLOSED pgs=542 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.119+0000 7f22e7990640 1 -- 172.21.15.32:0/3261694852 >> 172.21.15.32:0/3261694852 conn(0x7f22e0077740 msgr2=0x7f22e0077b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:56.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.119+0000 7f22e7990640 1 -- 172.21.15.32:0/3261694852 shutdown_connections 2024-04-25T00:33:56.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.119+0000 7f22e7990640 1 -- 172.21.15.32:0/3261694852 wait complete. 2024-04-25T00:33:56.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.120+0000 7f22e7990640 1 Processor -- start 2024-04-25T00:33:56.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.120+0000 7f22e7990640 1 -- start start 2024-04-25T00:33:56.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.120+0000 7f22e7990640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22e0100610 0x7f22e0198df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.120+0000 7f22e7990640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22e0100fd0 0x7f22e0199330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.120+0000 7f22e7990640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f22e01066b0 0x7f22e01a3e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.120+0000 7f22e7990640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f22e0110a30 con 0x7f22e0100610 2024-04-25T00:33:56.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.120+0000 7f22e7990640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f22e01108b0 con 0x7f22e01066b0 2024-04-25T00:33:56.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.120+0000 7f22e7990640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f22e0110bb0 con 0x7f22e0100fd0 2024-04-25T00:33:56.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.120+0000 7f22e572c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22e0100610 0x7f22e0198df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.120+0000 7f22e572c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22e0100610 0x7f22e0198df0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53670/0 (socket says 172.21.15.32:53670) 2024-04-25T00:33:56.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.120+0000 7f22e572c640 1 -- 172.21.15.32:0/1127256971 learned_addr learned my addr 172.21.15.32:0/1127256971 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:56.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.120+0000 7f22e4f2b640 1 --2- 172.21.15.32:0/1127256971 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22e0100fd0 0x7f22e0199330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.120+0000 7f22e5f2d640 1 --2- 172.21.15.32:0/1127256971 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f22e01066b0 0x7f22e01a3e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.120+0000 7f22e572c640 1 -- 172.21.15.32:0/1127256971 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22e0100fd0 msgr2=0x7f22e0199330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:56.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.120+0000 7f22e572c640 1 --2- 172.21.15.32:0/1127256971 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22e0100fd0 0x7f22e0199330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.120+0000 7f22e572c640 1 -- 172.21.15.32:0/1127256971 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f22e01066b0 msgr2=0x7f22e01a3e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:56.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.120+0000 7f22e572c640 1 --2- 172.21.15.32:0/1127256971 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f22e01066b0 0x7f22e01a3e70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.120+0000 7f22e572c640 1 -- 172.21.15.32:0/1127256971 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f22e01a43b0 con 0x7f22e0100610 2024-04-25T00:33:56.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.120+0000 7f22e4f2b640 1 --2- 172.21.15.32:0/1127256971 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22e0100fd0 0x7f22e0199330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:33:56.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.120+0000 7f22e5f2d640 1 --2- 172.21.15.32:0/1127256971 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f22e01066b0 0x7f22e01a3e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:33:56.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.121+0000 7f22e572c640 1 --2- 172.21.15.32:0/1127256971 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22e0100610 0x7f22e0198df0 secure :-1 s=READY pgs=543 cs=0 l=1 rev1=1 crypto rx=0x7f22dc009230 tx=0x7f22dc0042f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:56.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.121+0000 7f22d67fc640 1 -- 172.21.15.32:0/1127256971 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f22dc03f030 con 0x7f22e0100610 2024-04-25T00:33:56.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.121+0000 7f22e7990640 1 -- 172.21.15.32:0/1127256971 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f22e01a4580 con 0x7f22e0100610 2024-04-25T00:33:56.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.121+0000 7f22d67fc640 1 -- 172.21.15.32:0/1127256971 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f22dc003ce0 con 0x7f22e0100610 2024-04-25T00:33:56.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.121+0000 7f22d67fc640 1 -- 172.21.15.32:0/1127256971 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f22dc035070 con 0x7f22e0100610 2024-04-25T00:33:56.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.121+0000 7f22e7990640 1 -- 172.21.15.32:0/1127256971 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f22e01a49e0 con 0x7f22e0100610 2024-04-25T00:33:56.131 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.124+0000 7f22ea77b740 1 Processor -- start 2024-04-25T00:33:56.131 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.124+0000 7f22ea77b740 1 -- start start 2024-04-25T00:33:56.131 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.124+0000 7f22d67fc640 1 -- 172.21.15.32:0/1127256971 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 24) ==== 138009+0+0 (secure 0 0 0) 0x7f22dc042030 con 0x7f22e0100610 2024-04-25T00:33:56.131 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.124+0000 7f22ea77b740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5562e9917980 0x5562e9915cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.131 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.124+0000 7f22ea77b740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5562e9916290 0x5562e9916710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.124+0000 7f22ea77b740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9921010 0x5562e9923400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.124+0000 7f22ea77b740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5562e987a2c0 con 0x5562e9921010 2024-04-25T00:33:56.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.124+0000 7f22ea77b740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5562e9822910 con 0x5562e9916290 2024-04-25T00:33:56.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.124+0000 7f22ea77b740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5562e9876280 con 0x5562e9917980 2024-04-25T00:33:56.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.124+0000 7f22e4f2b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9921010 0x5562e9923400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.124+0000 7f22e4f2b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9921010 0x5562e9923400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53686/0 (socket says 172.21.15.32:53686) 2024-04-25T00:33:56.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.124+0000 7f22e4f2b640 1 -- 172.21.15.32:0/2133783380 learned_addr learned my addr 172.21.15.32:0/2133783380 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:56.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.125+0000 7f22e572c640 1 --2- 172.21.15.32:0/2133783380 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5562e9916290 0x5562e9916710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.125+0000 7f22e5f2d640 1 --2- 172.21.15.32:0/2133783380 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5562e9917980 0x5562e9915cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.125+0000 7f22e4f2b640 1 -- 172.21.15.32:0/2133783380 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5562e9917980 msgr2=0x5562e9915cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:56.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.125+0000 7f22e4f2b640 1 --2- 172.21.15.32:0/2133783380 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5562e9917980 0x5562e9915cc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.125+0000 7f22e4f2b640 1 -- 172.21.15.32:0/2133783380 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5562e9916290 msgr2=0x5562e9916710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:56.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.125+0000 7f22e4f2b640 1 --2- 172.21.15.32:0/2133783380 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5562e9916290 0x5562e9916710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.125+0000 7f22e4f2b640 1 -- 172.21.15.32:0/2133783380 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5562e9833fb0 con 0x5562e9921010 2024-04-25T00:33:56.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.125+0000 7f22e572c640 1 --2- 172.21.15.32:0/2133783380 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5562e9916290 0x5562e9916710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:56.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.125+0000 7f22e4f2b640 1 --2- 172.21.15.32:0/2133783380 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9921010 0x5562e9923400 secure :-1 s=READY pgs=544 cs=0 l=1 rev1=1 crypto rx=0x7f22d000bbe0 tx=0x7f22d000d710 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e5bf5de9d6635fcb server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:56.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.125+0000 7f22d67fc640 1 --2- 172.21.15.32:0/1127256971 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f22cc0a2d90 0x7f22cc0a5250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.125+0000 7f22e5f2d640 1 --2- 172.21.15.32:0/1127256971 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f22cc0a2d90 0x7f22cc0a5250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.125+0000 7f22cb7fe640 1 -- 172.21.15.32:0/2133783380 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f22d0017620 con 0x5562e9921010 2024-04-25T00:33:56.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.125+0000 7f22d67fc640 1 -- 172.21.15.32:0/1127256971 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f22dc0efc40 con 0x7f22e0100610 2024-04-25T00:33:56.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.125+0000 7f22cb7fe640 1 -- 172.21.15.32:0/2133783380 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f22d0017ca0 con 0x5562e9921010 2024-04-25T00:33:56.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.125+0000 7f22cb7fe640 1 -- 172.21.15.32:0/2133783380 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f22d000fd90 con 0x5562e9921010 2024-04-25T00:33:56.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.126+0000 7f22ea77b740 1 -- 172.21.15.32:0/2133783380 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9921010 msgr2=0x5562e9923400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:56.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.126+0000 7f22ea77b740 1 --2- 172.21.15.32:0/2133783380 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9921010 0x5562e9923400 secure :-1 s=READY pgs=544 cs=0 l=1 rev1=1 crypto rx=0x7f22d000bbe0 tx=0x7f22d000d710 comp rx=0 tx=0).stop 2024-04-25T00:33:56.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.126+0000 7f22e5f2d640 1 --2- 172.21.15.32:0/1127256971 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f22cc0a2d90 0x7f22cc0a5250 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f22d8008ea0 tx=0x7f22d80083d0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:56.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.126+0000 7f22ea77b740 1 -- 172.21.15.32:0/2133783380 shutdown_connections 2024-04-25T00:33:56.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.126+0000 7f22ea77b740 1 --2- 172.21.15.32:0/2133783380 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9921010 0x5562e9923400 unknown :-1 s=CLOSED pgs=544 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.126+0000 7f22ea77b740 1 --2- 172.21.15.32:0/2133783380 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5562e9916290 0x5562e9916710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.126+0000 7f22ea77b740 1 --2- 172.21.15.32:0/2133783380 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5562e9917980 0x5562e9915cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.126+0000 7f22ea77b740 1 -- 172.21.15.32:0/2133783380 >> 172.21.15.32:0/2133783380 conn(0x5562e98de950 msgr2=0x5562e99127f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:56.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.126+0000 7f22ea77b740 1 -- 172.21.15.32:0/2133783380 shutdown_connections 2024-04-25T00:33:56.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.126+0000 7f22ea77b740 1 -- 172.21.15.32:0/2133783380 wait complete. 2024-04-25T00:33:56.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.126+0000 7f22ea77b740 1 -- start start 2024-04-25T00:33:56.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.127+0000 7f22ea77b740 1 Processor -- start 2024-04-25T00:33:56.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.127+0000 7f22ea77b740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9916290 0x5562e991deb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.127+0000 7f22ea77b740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5562e9917980 0x5562e991e3f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.127+0000 7f22ea77b740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5562e9921010 0x5562e991e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.127+0000 7f22ea77b740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5562e9923ea0 con 0x5562e9916290 2024-04-25T00:33:56.134 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.127+0000 7f22ea77b740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5562e9835e90 con 0x5562e9921010 2024-04-25T00:33:56.135 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.127+0000 7f22ea77b740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5562e9924020 con 0x5562e9917980 2024-04-25T00:33:56.135 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.127+0000 7f22ea77b740 10 client.0 _ll_register_callbacks cb 0x5562e9871970 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:33:56.135 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.127+0000 7f22e572c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9916290 0x5562e991deb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.135 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.127+0000 7f22e572c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9916290 0x5562e991deb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53698/0 (socket says 172.21.15.32:53698) 2024-04-25T00:33:56.135 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.127+0000 7f22e572c640 1 -- 172.21.15.32:0/4010719273 learned_addr learned my addr 172.21.15.32:0/4010719273 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:56.135 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.127+0000 7f22e5f2d640 1 --2- 172.21.15.32:0/4010719273 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5562e9921010 0x5562e991e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.135 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.127+0000 7f22e572c640 1 -- 172.21.15.32:0/4010719273 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5562e9917980 msgr2=0x5562e991e3f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:56.135 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.127+0000 7f22e572c640 1 --2- 172.21.15.32:0/4010719273 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5562e9917980 0x5562e991e3f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.135 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.127+0000 7f22e572c640 1 -- 172.21.15.32:0/4010719273 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5562e9921010 msgr2=0x5562e991e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:56.135 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.127+0000 7f22e572c640 1 --2- 172.21.15.32:0/4010719273 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5562e9921010 0x5562e991e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.135 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.127+0000 7f22e572c640 1 -- 172.21.15.32:0/4010719273 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5562e96567e0 con 0x5562e9916290 2024-04-25T00:33:56.135 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.128+0000 7f22e572c640 1 --2- 172.21.15.32:0/4010719273 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9916290 0x5562e991deb0 secure :-1 s=READY pgs=545 cs=0 l=1 rev1=1 crypto rx=0x7f22dc004650 tx=0x7f22dc0b7ee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dc1467528ec88795 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:56.135 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.128+0000 7f22c8ff9640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:33:56.136 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.128+0000 7f22c8ff9640 1 -- 172.21.15.32:0/4010719273 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f22dc049070 con 0x5562e9916290 2024-04-25T00:33:56.136 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.128+0000 7f22c8ff9640 1 -- 172.21.15.32:0/4010719273 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f22dc03bc80 con 0x5562e9916290 2024-04-25T00:33:56.136 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.128+0000 7f22ea77b740 1 -- 172.21.15.32:0/4010719273 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f22d0017ca0 con 0x5562e9916290 2024-04-25T00:33:56.136 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.128+0000 7f22c8ff9640 1 -- 172.21.15.32:0/4010719273 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f22dc0bb5a0 con 0x5562e9916290 2024-04-25T00:33:56.136 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.128+0000 7f22ea77b740 10 client.15825 fetch_fsmap learned FSMap version 25 2024-04-25T00:33:56.136 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.128+0000 7f22ea77b740 1 -- 172.21.15.32:0/4010719273 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x5562e96b6f70 con 0x5562e9916290 2024-04-25T00:33:56.136 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.128+0000 7f22c8ff9640 1 -- 172.21.15.32:0/4010719273 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f22d0017ca0 con 0x5562e9916290 2024-04-25T00:33:56.136 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.128+0000 7f22c8ff9640 1 -- 172.21.15.32:0/4010719273 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7f22dc04c060 con 0x5562e9916290 2024-04-25T00:33:56.136 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.128+0000 7f22ea77b740 10 client.15825 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:33:56.136 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.128+0000 7f22ea77b740 10 client.15825 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:33:56.136 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.128+0000 7f22ea77b740 20 client.15825 populate_metadata read hostname 'smithi032' 2024-04-25T00:33:56.136 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.128+0000 7f22ea77b740 1 --2- 172.21.15.32:0/4010719273 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5562e9930f60 0x5562e9933420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.136 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.128+0000 7f22ea77b740 4 client.15825 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:33:56.136 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.128+0000 7f22ea77b740 1 -- 172.21.15.32:0/4010719273 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5562e984e5a0 con 0x5562e9930f60 2024-04-25T00:33:56.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.129+0000 7f22e4f2b640 1 --2- 172.21.15.32:0/4010719273 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5562e9930f60 0x5562e9933420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.129+0000 7f22e4f2b640 1 --2- 172.21.15.32:0/4010719273 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5562e9930f60 0x5562e9933420 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a87972e3d8e85a43 server_cookie=f6adfc6856783ee0 in_seq=0 out_seq=0 2024-04-25T00:33:56.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.129+0000 7f22c8ff9640 10 client.15825 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:56.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.130+0000 7f22c8ff9640 1 -- 172.21.15.32:0/4010719273 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x5562e984e5a0 con 0x5562e9930f60 2024-04-25T00:33:56.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.130+0000 7f22c8ff9640 10 client.15825 handle_command_reply: tid=0 2024-04-25T00:33:56.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.130+0000 7f22ea77b740 1 client.15825 shutdown 2024-04-25T00:33:56.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.130+0000 7f22ea77b740 1 -- 172.21.15.32:0/4010719273 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9916290 msgr2=0x5562e991deb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:56.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.130+0000 7f22ea77b740 1 --2- 172.21.15.32:0/4010719273 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9916290 0x5562e991deb0 secure :-1 s=READY pgs=545 cs=0 l=1 rev1=1 crypto rx=0x7f22dc004650 tx=0x7f22dc0b7ee0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.130+0000 7f22ea77b740 1 -- 172.21.15.32:0/4010719273 shutdown_connections 2024-04-25T00:33:56.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.130+0000 7f22ea77b740 1 --2- 172.21.15.32:0/4010719273 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5562e9930f60 0x5562e9933420 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.130+0000 7f22ea77b740 1 --2- 172.21.15.32:0/4010719273 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5562e9921010 0x5562e991e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.130+0000 7f22c8ff9640 0 client.15825 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:56.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.130+0000 7f22ea77b740 1 --2- 172.21.15.32:0/4010719273 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5562e9917980 0x5562e991e3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.138 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.130+0000 7f22ea77b740 1 --2- 172.21.15.32:0/4010719273 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9916290 0x5562e991deb0 unknown :-1 s=CLOSED pgs=545 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.138 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.130+0000 7f22ea77b740 1 -- 172.21.15.32:0/4010719273 >> 172.21.15.32:0/4010719273 conn(0x5562e98de950 msgr2=0x5562e99185f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:56.138 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.130+0000 7f22ea77b740 1 -- 172.21.15.32:0/4010719273 shutdown_connections 2024-04-25T00:33:56.138 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.130+0000 7f22ea77b740 1 -- 172.21.15.32:0/4010719273 wait complete. 2024-04-25T00:33:56.138 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.130+0000 7f22ea77b740 20 client.15825 trim_cache size 0 max 16384 2024-04-25T00:33:56.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.148+0000 7f22ea77b740 1 Processor -- start 2024-04-25T00:33:56.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.148+0000 7f22ea77b740 1 -- start start 2024-04-25T00:33:56.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.149+0000 7f22ea77b740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5562e9916290 0x5562e992cda0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.149+0000 7f22ea77b740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9917980 0x5562e99117d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.149+0000 7f22ea77b740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5562e9921010 0x5562e9938cf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.149+0000 7f22ea77b740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f22e01b1550 con 0x5562e9917980 2024-04-25T00:33:56.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.149+0000 7f22ea77b740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f22e01aee20 con 0x5562e9916290 2024-04-25T00:33:56.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.149+0000 7f22ea77b740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f22e01acc70 con 0x5562e9921010 2024-04-25T00:33:56.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.149+0000 7f22e4f2b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9917980 0x5562e99117d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.149+0000 7f22e4f2b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9917980 0x5562e99117d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53712/0 (socket says 172.21.15.32:53712) 2024-04-25T00:33:56.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.149+0000 7f22e4f2b640 1 -- 172.21.15.32:0/2619408591 learned_addr learned my addr 172.21.15.32:0/2619408591 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:56.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.149+0000 7f22e5f2d640 1 --2- 172.21.15.32:0/2619408591 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5562e9921010 0x5562e9938cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.149+0000 7f22e4f2b640 1 -- 172.21.15.32:0/2619408591 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5562e9921010 msgr2=0x5562e9938cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:56.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.149+0000 7f22e4f2b640 1 --2- 172.21.15.32:0/2619408591 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5562e9921010 0x5562e9938cf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.149+0000 7f22e572c640 1 --2- 172.21.15.32:0/2619408591 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5562e9916290 0x5562e992cda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.149+0000 7f22e4f2b640 1 -- 172.21.15.32:0/2619408591 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5562e9916290 msgr2=0x5562e992cda0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:56.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.149+0000 7f22e4f2b640 1 --2- 172.21.15.32:0/2619408591 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5562e9916290 0x5562e992cda0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.149+0000 7f22e4f2b640 1 -- 172.21.15.32:0/2619408591 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5562e954e170 con 0x5562e9917980 2024-04-25T00:33:56.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.149+0000 7f22e572c640 1 --2- 172.21.15.32:0/2619408591 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5562e9916290 0x5562e992cda0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:33:56.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.149+0000 7f22e4f2b640 1 --2- 172.21.15.32:0/2619408591 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9917980 0x5562e99117d0 secure :-1 s=READY pgs=546 cs=0 l=1 rev1=1 crypto rx=0x5562e9913980 tx=0x7f22d000f850 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e899f7b30645f378 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:56.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.150+0000 7f22caffd640 1 -- 172.21.15.32:0/2619408591 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f22d000b6d0 con 0x5562e9917980 2024-04-25T00:33:56.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.150+0000 7f22caffd640 1 -- 172.21.15.32:0/2619408591 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f22d000f420 con 0x5562e9917980 2024-04-25T00:33:56.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.150+0000 7f22caffd640 1 -- 172.21.15.32:0/2619408591 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f22d0005550 con 0x5562e9917980 2024-04-25T00:33:56.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.151+0000 7f22ea77b740 1 -- 172.21.15.32:0/2619408591 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9917980 msgr2=0x5562e99117d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:56.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.151+0000 7f22ea77b740 1 --2- 172.21.15.32:0/2619408591 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9917980 0x5562e99117d0 secure :-1 s=READY pgs=546 cs=0 l=1 rev1=1 crypto rx=0x5562e9913980 tx=0x7f22d000f850 comp rx=0 tx=0).stop 2024-04-25T00:33:56.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.151+0000 7f22ea77b740 1 -- 172.21.15.32:0/2619408591 shutdown_connections 2024-04-25T00:33:56.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.151+0000 7f22ea77b740 1 --2- 172.21.15.32:0/2619408591 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5562e9921010 0x5562e9938cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.151+0000 7f22ea77b740 1 --2- 172.21.15.32:0/2619408591 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9917980 0x5562e99117d0 unknown :-1 s=CLOSED pgs=546 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.151+0000 7f22ea77b740 1 --2- 172.21.15.32:0/2619408591 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5562e9916290 0x5562e992cda0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.151+0000 7f22ea77b740 1 -- 172.21.15.32:0/2619408591 >> 172.21.15.32:0/2619408591 conn(0x5562e98de950 msgr2=0x5562e99110c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:56.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.151+0000 7f22ea77b740 1 -- 172.21.15.32:0/2619408591 shutdown_connections 2024-04-25T00:33:56.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.151+0000 7f22ea77b740 1 -- 172.21.15.32:0/2619408591 wait complete. 2024-04-25T00:33:56.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.151+0000 7f22ea77b740 1 -- start start 2024-04-25T00:33:56.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.151+0000 7f22ea77b740 1 Processor -- start 2024-04-25T00:33:56.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.152+0000 7f22ea77b740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5562e9916290 0x5562e9924fa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.152+0000 7f22ea77b740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9917980 0x5562e99254e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.152+0000 7f22ea77b740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5562e9921010 0x5562e9925a40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.152+0000 7f22ea77b740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f22e01ad2e0 con 0x5562e9917980 2024-04-25T00:33:56.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.152+0000 7f22ea77b740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f22e01ae750 con 0x5562e9921010 2024-04-25T00:33:56.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.152+0000 7f22ea77b740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f22e01ad9b0 con 0x5562e9916290 2024-04-25T00:33:56.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.152+0000 7f22ea77b740 10 client.0 _ll_register_callbacks cb 0x7f22dc0ecdb0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:33:56.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.152+0000 7f22e4f2b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9917980 0x5562e99254e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.152+0000 7f22e4f2b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9917980 0x5562e99254e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53728/0 (socket says 172.21.15.32:53728) 2024-04-25T00:33:56.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.152+0000 7f22e4f2b640 1 -- 172.21.15.32:0/2305872933 learned_addr learned my addr 172.21.15.32:0/2305872933 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:56.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.152+0000 7f22e572c640 1 --2- 172.21.15.32:0/2305872933 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5562e9916290 0x5562e9924fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.152+0000 7f22e5f2d640 1 --2- 172.21.15.32:0/2305872933 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5562e9921010 0x5562e9925a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.159 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.152+0000 7f22e4f2b640 1 -- 172.21.15.32:0/2305872933 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5562e9916290 msgr2=0x5562e9924fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:56.159 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.152+0000 7f22e4f2b640 1 --2- 172.21.15.32:0/2305872933 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5562e9916290 0x5562e9924fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.159 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.152+0000 7f22e4f2b640 1 -- 172.21.15.32:0/2305872933 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5562e9921010 msgr2=0x5562e9925a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:56.159 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.152+0000 7f22e4f2b640 1 --2- 172.21.15.32:0/2305872933 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5562e9921010 0x5562e9925a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.159 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.152+0000 7f22e4f2b640 1 -- 172.21.15.32:0/2305872933 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5562e9996830 con 0x5562e9917980 2024-04-25T00:33:56.159 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.152+0000 7f22e4f2b640 1 --2- 172.21.15.32:0/2305872933 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9917980 0x5562e99254e0 secure :-1 s=READY pgs=547 cs=0 l=1 rev1=1 crypto rx=0x7f22d0005c60 tx=0x7f22d000f690 comp rx=0 tx=0).ready entity=mon.0 client_cookie=447dc81ba9c3374d server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:56.159 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.152+0000 7f22c8ff9640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:33:56.159 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.152+0000 7f22c8ff9640 1 -- 172.21.15.32:0/2305872933 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f22d0035020 con 0x5562e9917980 2024-04-25T00:33:56.159 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.153+0000 7f22c8ff9640 1 -- 172.21.15.32:0/2305872933 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f22d0005370 con 0x5562e9917980 2024-04-25T00:33:56.159 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.153+0000 7f22ea77b740 1 -- 172.21.15.32:0/2305872933 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f22d000f420 con 0x5562e9917980 2024-04-25T00:33:56.159 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.153+0000 7f22c8ff9640 1 -- 172.21.15.32:0/2305872933 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f22d0040050 con 0x5562e9917980 2024-04-25T00:33:56.159 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.153+0000 7f22ea77b740 10 client.15831 fetch_fsmap learned FSMap version 25 2024-04-25T00:33:56.159 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.153+0000 7f22ea77b740 1 -- 172.21.15.32:0/2305872933 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x5562e9997ae0 con 0x5562e9917980 2024-04-25T00:33:56.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.153+0000 7f22c8ff9640 1 -- 172.21.15.32:0/2305872933 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f22d000f420 con 0x5562e9917980 2024-04-25T00:33:56.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.153+0000 7f22c8ff9640 1 -- 172.21.15.32:0/2305872933 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7f22d0026770 con 0x5562e9917980 2024-04-25T00:33:56.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.153+0000 7f22ea77b740 10 client.15831 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:33:56.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.153+0000 7f22ea77b740 10 client.15831 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:33:56.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.153+0000 7f22ea77b740 20 client.15831 populate_metadata read hostname 'smithi032' 2024-04-25T00:33:56.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.153+0000 7f22ea77b740 1 --2- 172.21.15.32:0/2305872933 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5562e9998080 0x5562e999a4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.153+0000 7f22ea77b740 4 client.15831 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-25T00:33:56.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.153+0000 7f22ea77b740 1 -- 172.21.15.32:0/2305872933 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) -- 0x5562e9887800 con 0x5562e9998080 2024-04-25T00:33:56.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.153+0000 7f22e572c640 1 --2- 172.21.15.32:0/2305872933 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5562e9998080 0x5562e999a4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.154+0000 7f22c8ff9640 10 client.15831 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:56.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.154+0000 7f22e572c640 1 --2- 172.21.15.32:0/2305872933 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5562e9998080 0x5562e999a4a0 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7e2e7f1344797fd0 server_cookie=ec1b5865be274229 in_seq=0 out_seq=0 2024-04-25T00:33:56.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.155+0000 7f22c8ff9640 1 -- 172.21.15.32:0/2305872933 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+110 (crc 0 0 0) 0x5562e9887800 con 0x5562e9998080 2024-04-25T00:33:56.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.155+0000 7f22c8ff9640 10 client.15831 handle_command_reply: tid=0 2024-04-25T00:33:56.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.155+0000 7f22ea77b740 1 client.15831 shutdown 2024-04-25T00:33:56.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.155+0000 7f22ea77b740 1 -- 172.21.15.32:0/2305872933 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9917980 msgr2=0x5562e99254e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:56.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.155+0000 7f22ea77b740 1 --2- 172.21.15.32:0/2305872933 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9917980 0x5562e99254e0 secure :-1 s=READY pgs=547 cs=0 l=1 rev1=1 crypto rx=0x7f22d0005c60 tx=0x7f22d000f690 comp rx=0 tx=0).stop 2024-04-25T00:33:56.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.155+0000 7f22ea77b740 1 -- 172.21.15.32:0/2305872933 shutdown_connections 2024-04-25T00:33:56.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.155+0000 7f22ea77b740 1 --2- 172.21.15.32:0/2305872933 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5562e9998080 0x5562e999a4a0 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.155+0000 7f22ea77b740 1 --2- 172.21.15.32:0/2305872933 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5562e9921010 0x5562e9925a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.155+0000 7f22ea77b740 1 --2- 172.21.15.32:0/2305872933 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5562e9917980 0x5562e99254e0 unknown :-1 s=CLOSED pgs=547 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.155+0000 7f22ea77b740 1 --2- 172.21.15.32:0/2305872933 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5562e9916290 0x5562e9924fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.155+0000 7f22c8ff9640 0 client.15831 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:56.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.155+0000 7f22ea77b740 1 -- 172.21.15.32:0/2305872933 >> 172.21.15.32:0/2305872933 conn(0x5562e98de950 msgr2=0x5562e9911950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:56.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.155+0000 7f22ea77b740 1 -- 172.21.15.32:0/2305872933 shutdown_connections 2024-04-25T00:33:56.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.155+0000 7f22ea77b740 1 -- 172.21.15.32:0/2305872933 wait complete. 2024-04-25T00:33:56.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.155+0000 7f22ea77b740 20 client.15831 trim_cache size 0 max 16384 2024-04-25T00:33:56.162 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:33:56.162 INFO:teuthology.orchestra.run.smithi032.stdout: "return_code": 0, 2024-04-25T00:33:56.162 INFO:teuthology.orchestra.run.smithi032.stdout: "scrub_tag": "93079411-5a82-4ebf-a044-6a1c24977bc2", 2024-04-25T00:33:56.162 INFO:teuthology.orchestra.run.smithi032.stdout: "mode": "asynchronous" 2024-04-25T00:33:56.162 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:33:56.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.156+0000 7f22e7990640 1 -- 172.21.15.32:0/1127256971 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f22cc0a2d90 msgr2=0x7f22cc0a5250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:56.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.156+0000 7f22e7990640 1 --2- 172.21.15.32:0/1127256971 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f22cc0a2d90 0x7f22cc0a5250 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f22d8008ea0 tx=0x7f22d80083d0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.156+0000 7f22e7990640 1 -- 172.21.15.32:0/1127256971 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22e0100610 msgr2=0x7f22e0198df0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:56.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.156+0000 7f22e7990640 1 --2- 172.21.15.32:0/1127256971 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22e0100610 0x7f22e0198df0 secure :-1 s=READY pgs=543 cs=0 l=1 rev1=1 crypto rx=0x7f22dc009230 tx=0x7f22dc0042f0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.156+0000 7f22e7990640 1 -- 172.21.15.32:0/1127256971 shutdown_connections 2024-04-25T00:33:56.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.156+0000 7f22e7990640 1 --2- 172.21.15.32:0/1127256971 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f22cc0a2d90 0x7f22cc0a5250 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.156+0000 7f22e7990640 1 --2- 172.21.15.32:0/1127256971 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f22e01066b0 0x7f22e01a3e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.156+0000 7f22e7990640 1 --2- 172.21.15.32:0/1127256971 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22e0100fd0 0x7f22e0199330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.156+0000 7f22e7990640 1 --2- 172.21.15.32:0/1127256971 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f22e0100610 0x7f22e0198df0 unknown :-1 s=CLOSED pgs=543 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.156+0000 7f22e7990640 1 -- 172.21.15.32:0/1127256971 >> 172.21.15.32:0/1127256971 conn(0x7f22e0077740 msgr2=0x7f22e00fefd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:56.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.156+0000 7f22e7990640 1 -- 172.21.15.32:0/1127256971 shutdown_connections 2024-04-25T00:33:56.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.156+0000 7f22e7990640 1 -- 172.21.15.32:0/1127256971 wait complete. 2024-04-25T00:33:56.169 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-25T00:33:56.169 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-25T00:33:56.277 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.275+0000 7f6b8ff63640 1 -- 172.21.15.32:0/1150162241 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6b881020e0 msgr2=0x7f6b8810e5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:56.277 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.275+0000 7f6b8ff63640 1 --2- 172.21.15.32:0/1150162241 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6b881020e0 0x7f6b8810e5f0 secure :-1 s=READY pgs=548 cs=0 l=1 rev1=1 crypto rx=0x7f6b80009cd0 tx=0x7f6b8002a8e0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.277 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.275+0000 7f6b8ff63640 1 -- 172.21.15.32:0/1150162241 shutdown_connections 2024-04-25T00:33:56.278 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.275+0000 7f6b8ff63640 1 --2- 172.21.15.32:0/1150162241 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6b881020e0 0x7f6b8810e5f0 unknown :-1 s=CLOSED pgs=548 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.278 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.275+0000 7f6b8ff63640 1 --2- 172.21.15.32:0/1150162241 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6b88101720 0x7f6b88101ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.278 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.275+0000 7f6b8ff63640 1 --2- 172.21.15.32:0/1150162241 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6b88107730 0x7f6b88107b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.278 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.275+0000 7f6b8ff63640 1 -- 172.21.15.32:0/1150162241 >> 172.21.15.32:0/1150162241 conn(0x7f6b88078b40 msgr2=0x7f6b88078f50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:56.278 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.275+0000 7f6b8ff63640 1 -- 172.21.15.32:0/1150162241 shutdown_connections 2024-04-25T00:33:56.278 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.275+0000 7f6b8ff63640 1 -- 172.21.15.32:0/1150162241 wait complete. 2024-04-25T00:33:56.278 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.276+0000 7f6b8ff63640 1 Processor -- start 2024-04-25T00:33:56.278 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.276+0000 7f6b8ff63640 1 -- start start 2024-04-25T00:33:56.278 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.276+0000 7f6b8ff63640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6b88101720 0x7f6b88197fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.278 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.276+0000 7f6b8ff63640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6b881020e0 0x7f6b88198500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.278 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.276+0000 7f6b8ff63640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6b88107730 0x7f6b8819c880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.278 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.276+0000 7f6b8ff63640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f6b88111b00 con 0x7f6b881020e0 2024-04-25T00:33:56.278 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.276+0000 7f6b8ff63640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f6b88111980 con 0x7f6b88107730 2024-04-25T00:33:56.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.276+0000 7f6b8ff63640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f6b88111c80 con 0x7f6b88101720 2024-04-25T00:33:56.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.276+0000 7f6b8d4fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6b881020e0 0x7f6b88198500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.276+0000 7f6b8d4fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6b881020e0 0x7f6b88198500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53740/0 (socket says 172.21.15.32:53740) 2024-04-25T00:33:56.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.276+0000 7f6b8d4fe640 1 -- 172.21.15.32:0/2477626023 learned_addr learned my addr 172.21.15.32:0/2477626023 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:56.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.276+0000 7f6b8dcff640 1 --2- 172.21.15.32:0/2477626023 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6b88101720 0x7f6b88197fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.276+0000 7f6b8e500640 1 --2- 172.21.15.32:0/2477626023 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6b88107730 0x7f6b8819c880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.276+0000 7f6b8d4fe640 1 -- 172.21.15.32:0/2477626023 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6b88101720 msgr2=0x7f6b88197fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:56.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.276+0000 7f6b8d4fe640 1 --2- 172.21.15.32:0/2477626023 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6b88101720 0x7f6b88197fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.276+0000 7f6b8d4fe640 1 -- 172.21.15.32:0/2477626023 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6b88107730 msgr2=0x7f6b8819c880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:56.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.276+0000 7f6b8d4fe640 1 --2- 172.21.15.32:0/2477626023 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6b88107730 0x7f6b8819c880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.276+0000 7f6b8d4fe640 1 -- 172.21.15.32:0/2477626023 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6b8819cfb0 con 0x7f6b881020e0 2024-04-25T00:33:56.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.276+0000 7f6b8d4fe640 1 --2- 172.21.15.32:0/2477626023 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6b881020e0 0x7f6b88198500 secure :-1 s=READY pgs=549 cs=0 l=1 rev1=1 crypto rx=0x7f6b7800e9f0 tx=0x7f6b7800eeb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:56.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.277+0000 7f6b7effd640 1 -- 172.21.15.32:0/2477626023 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b7800cd80 con 0x7f6b881020e0 2024-04-25T00:33:56.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.277+0000 7f6b8ff63640 1 -- 172.21.15.32:0/2477626023 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6b8819d2a0 con 0x7f6b881020e0 2024-04-25T00:33:56.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.277+0000 7f6b7effd640 1 -- 172.21.15.32:0/2477626023 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6b78010910 con 0x7f6b881020e0 2024-04-25T00:33:56.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.277+0000 7f6b7effd640 1 -- 172.21.15.32:0/2477626023 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b780188f0 con 0x7f6b881020e0 2024-04-25T00:33:56.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.277+0000 7f6b8ff63640 1 -- 172.21.15.32:0/2477626023 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f6b881a4b30 con 0x7f6b881020e0 2024-04-25T00:33:56.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.279+0000 7f6b7effd640 1 -- 172.21.15.32:0/2477626023 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 24) ==== 138009+0+0 (secure 0 0 0) 0x7f6b78010ac0 con 0x7f6b881020e0 2024-04-25T00:33:56.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.279+0000 7f6b92c7b740 1 Processor -- start 2024-04-25T00:33:56.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.279+0000 7f6b92c7b740 1 -- start start 2024-04-25T00:33:56.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.280+0000 7f6b92c7b740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f72630 0x561763f72a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.280+0000 7f6b92c7b740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561763f72f50 0x561763f7d0e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.280+0000 7f6b92c7b740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x561763f7d620 0x561763f7fa10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.280+0000 7f6b92c7b740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x561763ed76b0 con 0x561763f72630 2024-04-25T00:33:56.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.280+0000 7f6b92c7b740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x561763e7f880 con 0x561763f7d620 2024-04-25T00:33:56.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.280+0000 7f6b92c7b740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x561763ed3670 con 0x561763f72f50 2024-04-25T00:33:56.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.280+0000 7f6b7effd640 1 --2- 172.21.15.32:0/2477626023 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f6b740a2d90 0x7f6b740a5250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.280+0000 7f6b8e500640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f72630 0x561763f72a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.280+0000 7f6b7effd640 1 -- 172.21.15.32:0/2477626023 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f6b780cfe70 con 0x7f6b881020e0 2024-04-25T00:33:56.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.280+0000 7f6b8e500640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f72630 0x561763f72a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53752/0 (socket says 172.21.15.32:53752) 2024-04-25T00:33:56.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.280+0000 7f6b8e500640 1 -- 172.21.15.32:0/2721379883 learned_addr learned my addr 172.21.15.32:0/2721379883 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:56.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.280+0000 7f6b8d4fe640 1 --2- 172.21.15.32:0/2721379883 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x561763f7d620 0x561763f7fa10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.280+0000 7f6b8e500640 1 --2- 172.21.15.32:0/2477626023 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f6b740a2d90 0x7f6b740a5250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.280+0000 7f6b8dcff640 1 --2- 172.21.15.32:0/2721379883 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561763f72f50 0x561763f7d0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.280+0000 7f6b8d4fe640 1 -- 172.21.15.32:0/2721379883 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561763f72f50 msgr2=0x561763f7d0e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:56.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.280+0000 7f6b8d4fe640 1 --2- 172.21.15.32:0/2721379883 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561763f72f50 0x561763f7d0e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.280+0000 7f6b8d4fe640 1 -- 172.21.15.32:0/2721379883 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f72630 msgr2=0x561763f72a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.280+0000 7f6b8d4fe640 1 --2- 172.21.15.32:0/2721379883 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f72630 0x561763f72a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.280+0000 7f6b8d4fe640 1 -- 172.21.15.32:0/2721379883 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x561763cb57d0 con 0x561763f7d620 2024-04-25T00:33:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.281+0000 7f6b8dcff640 1 --2- 172.21.15.32:0/2721379883 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561763f72f50 0x561763f7d0e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.281+0000 7f6b8d4fe640 1 --2- 172.21.15.32:0/2721379883 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x561763f7d620 0x561763f7fa10 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f6b780cd0f0 tx=0x7f6b78066fa0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e08b4d5c30de290e server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.281+0000 7f6b8e500640 1 --2- 172.21.15.32:0/2477626023 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f6b740a2d90 0x7f6b740a5250 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f6b80004af0 tx=0x7f6b800095c0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.281+0000 7f6b73fff640 1 -- 172.21.15.32:0/2721379883 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b780a01c0 con 0x561763f7d620 2024-04-25T00:33:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.281+0000 7f6b73fff640 1 -- 172.21.15.32:0/2721379883 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6b780a4a80 con 0x561763f7d620 2024-04-25T00:33:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.281+0000 7f6b73fff640 1 -- 172.21.15.32:0/2721379883 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b780cd5a0 con 0x561763f7d620 2024-04-25T00:33:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.282+0000 7f6b92c7b740 1 -- 172.21.15.32:0/2721379883 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x561763f7d620 msgr2=0x561763f7fa10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.282+0000 7f6b92c7b740 1 --2- 172.21.15.32:0/2721379883 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x561763f7d620 0x561763f7fa10 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f6b780cd0f0 tx=0x7f6b78066fa0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.282+0000 7f6b92c7b740 1 -- 172.21.15.32:0/2721379883 shutdown_connections 2024-04-25T00:33:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.282+0000 7f6b92c7b740 1 --2- 172.21.15.32:0/2721379883 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x561763f7d620 0x561763f7fa10 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.282+0000 7f6b92c7b740 1 --2- 172.21.15.32:0/2721379883 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561763f72f50 0x561763f7d0e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.282+0000 7f6b92c7b740 1 --2- 172.21.15.32:0/2721379883 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f72630 0x561763f72a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.282+0000 7f6b92c7b740 1 -- 172.21.15.32:0/2721379883 >> 172.21.15.32:0/2721379883 conn(0x561763f6e950 msgr2=0x561763f6ed30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:56.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.282+0000 7f6b92c7b740 1 -- 172.21.15.32:0/2721379883 shutdown_connections 2024-04-25T00:33:56.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.282+0000 7f6b92c7b740 1 -- 172.21.15.32:0/2721379883 wait complete. 2024-04-25T00:33:56.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.282+0000 7f6b92c7b740 1 -- start start 2024-04-25T00:33:56.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.282+0000 7f6b92c7b740 1 Processor -- start 2024-04-25T00:33:56.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.283+0000 7f6b92c7b740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x561763f72630 0x561763f7a5b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.283+0000 7f6b92c7b740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f72f50 0x561763f7aaf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.283+0000 7f6b92c7b740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561763f7d620 0x561763f7b030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.283+0000 7f6b92c7b740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x561763f803b0 con 0x561763f72f50 2024-04-25T00:33:56.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.283+0000 7f6b92c7b740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x561763e92f50 con 0x561763f72630 2024-04-25T00:33:56.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.283+0000 7f6b92c7b740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x561763f80530 con 0x561763f7d620 2024-04-25T00:33:56.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.283+0000 7f6b92c7b740 10 client.0 _ll_register_callbacks cb 0x561763ece000 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:33:56.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.283+0000 7f6b8d4fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f72f50 0x561763f7aaf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.283+0000 7f6b8d4fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f72f50 0x561763f7aaf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53760/0 (socket says 172.21.15.32:53760) 2024-04-25T00:33:56.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.283+0000 7f6b8d4fe640 1 -- 172.21.15.32:0/2238710602 learned_addr learned my addr 172.21.15.32:0/2238710602 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:56.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.283+0000 7f6b8dcff640 1 --2- 172.21.15.32:0/2238710602 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x561763f72630 0x561763f7a5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.283+0000 7f6b8e500640 1 --2- 172.21.15.32:0/2238710602 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561763f7d620 0x561763f7b030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.283+0000 7f6b8d4fe640 1 -- 172.21.15.32:0/2238710602 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561763f7d620 msgr2=0x561763f7b030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:56.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.283+0000 7f6b8d4fe640 1 --2- 172.21.15.32:0/2238710602 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561763f7d620 0x561763f7b030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.283+0000 7f6b8d4fe640 1 -- 172.21.15.32:0/2238710602 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x561763f72630 msgr2=0x561763f7a5b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:56.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.283+0000 7f6b8d4fe640 1 --2- 172.21.15.32:0/2238710602 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x561763f72630 0x561763f7a5b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.283+0000 7f6b8d4fe640 1 -- 172.21.15.32:0/2238710602 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x561763beb3f0 con 0x561763f72f50 2024-04-25T00:33:56.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.283+0000 7f6b8d4fe640 1 --2- 172.21.15.32:0/2238710602 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f72f50 0x561763f7aaf0 secure :-1 s=READY pgs=550 cs=0 l=1 rev1=1 crypto rx=0x7f6b780cd150 tx=0x7f6b7809bf40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=382cfa6bc1266424 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:56.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.283+0000 7f6b717fa640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:33:56.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.283+0000 7f6b717fa640 1 -- 172.21.15.32:0/2238710602 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b7809b260 con 0x561763f72f50 2024-04-25T00:33:56.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.283+0000 7f6b717fa640 1 -- 172.21.15.32:0/2238710602 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f6b7809b400 con 0x561763f72f50 2024-04-25T00:33:56.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.283+0000 7f6b92c7b740 1 -- 172.21.15.32:0/2238710602 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f6b780a4a80 con 0x561763f72f50 2024-04-25T00:33:56.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.284+0000 7f6b717fa640 1 -- 172.21.15.32:0/2238710602 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f6b78010430 con 0x561763f72f50 2024-04-25T00:33:56.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.284+0000 7f6b92c7b740 10 client.15843 fetch_fsmap learned FSMap version 25 2024-04-25T00:33:56.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.284+0000 7f6b92c7b740 1 -- 172.21.15.32:0/2238710602 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x561763bab0f0 con 0x561763f72f50 2024-04-25T00:33:56.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.284+0000 7f6b717fa640 1 -- 172.21.15.32:0/2238710602 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6b780a4a80 con 0x561763f72f50 2024-04-25T00:33:56.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.284+0000 7f6b717fa640 1 -- 172.21.15.32:0/2238710602 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7f6b7809aed0 con 0x561763f72f50 2024-04-25T00:33:56.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.284+0000 7f6b92c7b740 10 client.15843 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:33:56.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.284+0000 7f6b92c7b740 10 client.15843 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:33:56.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.284+0000 7f6b92c7b740 20 client.15843 populate_metadata read hostname 'smithi032' 2024-04-25T00:33:56.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.284+0000 7f6b92c7b740 1 --2- 172.21.15.32:0/2238710602 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x561763f8d730 0x561763f8fb20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.284+0000 7f6b92c7b740 4 client.15843 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:33:56.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.284+0000 7f6b92c7b740 1 -- 172.21.15.32:0/2238710602 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x561763ee4bf0 con 0x561763f8d730 2024-04-25T00:33:56.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.284+0000 7f6b8dcff640 1 --2- 172.21.15.32:0/2238710602 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x561763f8d730 0x561763f8fb20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.285+0000 7f6b717fa640 10 client.15843 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:56.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.285+0000 7f6b8dcff640 1 --2- 172.21.15.32:0/2238710602 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x561763f8d730 0x561763f8fb20 crc :-1 s=READY pgs=50 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=de432f9020c625bf server_cookie=c97a26ebbf3d21ba in_seq=0 out_seq=0 2024-04-25T00:33:56.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.285+0000 7f6b717fa640 1 -- 172.21.15.32:0/2238710602 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x561763ee4bf0 con 0x561763f8d730 2024-04-25T00:33:56.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.285+0000 7f6b717fa640 10 client.15843 handle_command_reply: tid=0 2024-04-25T00:33:56.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.285+0000 7f6b92c7b740 1 client.15843 shutdown 2024-04-25T00:33:56.297 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.285+0000 7f6b92c7b740 1 -- 172.21.15.32:0/2238710602 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f72f50 msgr2=0x561763f7aaf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:56.297 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.285+0000 7f6b92c7b740 1 --2- 172.21.15.32:0/2238710602 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f72f50 0x561763f7aaf0 secure :-1 s=READY pgs=550 cs=0 l=1 rev1=1 crypto rx=0x7f6b780cd150 tx=0x7f6b7809bf40 comp rx=0 tx=0).stop 2024-04-25T00:33:56.297 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.285+0000 7f6b92c7b740 1 -- 172.21.15.32:0/2238710602 shutdown_connections 2024-04-25T00:33:56.297 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.285+0000 7f6b92c7b740 1 --2- 172.21.15.32:0/2238710602 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x561763f8d730 0x561763f8fb20 crc :-1 s=READY pgs=50 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.297 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.285+0000 7f6b92c7b740 1 --2- 172.21.15.32:0/2238710602 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561763f7d620 0x561763f7b030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.297 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.285+0000 7f6b717fa640 0 client.15843 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:56.297 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.285+0000 7f6b92c7b740 1 --2- 172.21.15.32:0/2238710602 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f72f50 0x561763f7aaf0 unknown :-1 s=CLOSED pgs=550 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.297 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.285+0000 7f6b92c7b740 1 --2- 172.21.15.32:0/2238710602 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x561763f72630 0x561763f7a5b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.297 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.285+0000 7f6b92c7b740 1 -- 172.21.15.32:0/2238710602 >> 172.21.15.32:0/2238710602 conn(0x561763f6e950 msgr2=0x561763f74750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:56.297 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.286+0000 7f6b92c7b740 1 -- 172.21.15.32:0/2238710602 shutdown_connections 2024-04-25T00:33:56.297 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.286+0000 7f6b92c7b740 1 -- 172.21.15.32:0/2238710602 wait complete. 2024-04-25T00:33:56.297 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.286+0000 7f6b92c7b740 20 client.15843 trim_cache size 0 max 16384 2024-04-25T00:33:56.311 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.303+0000 7f6b92c7b740 1 Processor -- start 2024-04-25T00:33:56.311 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.303+0000 7f6b92c7b740 1 -- start start 2024-04-25T00:33:56.311 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.303+0000 7f6b92c7b740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f6e950 0x561763f7cce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.311 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.303+0000 7f6b92c7b740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561763f72630 0x561763f794f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.312 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.303+0000 7f6b92c7b740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x561763f72f50 0x561763f74e50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.312 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.303+0000 7f6b92c7b740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f6b881b1610 con 0x561763f6e950 2024-04-25T00:33:56.312 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.303+0000 7f6b92c7b740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f6b881aeee0 con 0x561763f72f50 2024-04-25T00:33:56.312 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.303+0000 7f6b92c7b740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f6b881acd30 con 0x561763f72630 2024-04-25T00:33:56.312 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.304+0000 7f6b8dcff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f6e950 0x561763f7cce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.312 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.304+0000 7f6b8dcff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f6e950 0x561763f7cce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53776/0 (socket says 172.21.15.32:53776) 2024-04-25T00:33:56.312 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.304+0000 7f6b8dcff640 1 -- 172.21.15.32:0/2607981790 learned_addr learned my addr 172.21.15.32:0/2607981790 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:56.312 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.304+0000 7f6b8e500640 1 --2- 172.21.15.32:0/2607981790 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x561763f72f50 0x561763f74e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.304+0000 7f6b8dcff640 1 -- 172.21.15.32:0/2607981790 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561763f72630 msgr2=0x561763f794f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:56.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.304+0000 7f6b8d4fe640 1 --2- 172.21.15.32:0/2607981790 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561763f72630 0x561763f794f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.304+0000 7f6b8dcff640 1 --2- 172.21.15.32:0/2607981790 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561763f72630 0x561763f794f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.304+0000 7f6b8dcff640 1 -- 172.21.15.32:0/2607981790 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x561763f72f50 msgr2=0x561763f74e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:56.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.304+0000 7f6b8dcff640 1 --2- 172.21.15.32:0/2607981790 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x561763f72f50 0x561763f74e50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.304+0000 7f6b8dcff640 1 -- 172.21.15.32:0/2607981790 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x561763f75390 con 0x561763f6e950 2024-04-25T00:33:56.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.304+0000 7f6b8e500640 1 --2- 172.21.15.32:0/2607981790 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x561763f72f50 0x561763f74e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:56.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.304+0000 7f6b8dcff640 1 --2- 172.21.15.32:0/2607981790 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f6e950 0x561763f7cce0 secure :-1 s=READY pgs=551 cs=0 l=1 rev1=1 crypto rx=0x561763f80550 tx=0x7f6b8400bd30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c32e153a80326b1b server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:56.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.304+0000 7f6b737fe640 1 -- 172.21.15.32:0/2607981790 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b84021020 con 0x561763f6e950 2024-04-25T00:33:56.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.304+0000 7f6b737fe640 1 -- 172.21.15.32:0/2607981790 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6b8401d900 con 0x561763f6e950 2024-04-25T00:33:56.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.304+0000 7f6b737fe640 1 -- 172.21.15.32:0/2607981790 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b84011c10 con 0x561763f6e950 2024-04-25T00:33:56.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.305+0000 7f6b92c7b740 1 -- 172.21.15.32:0/2607981790 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f6e950 msgr2=0x561763f7cce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:56.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.305+0000 7f6b92c7b740 1 --2- 172.21.15.32:0/2607981790 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f6e950 0x561763f7cce0 secure :-1 s=READY pgs=551 cs=0 l=1 rev1=1 crypto rx=0x561763f80550 tx=0x7f6b8400bd30 comp rx=0 tx=0).stop 2024-04-25T00:33:56.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.305+0000 7f6b92c7b740 1 -- 172.21.15.32:0/2607981790 shutdown_connections 2024-04-25T00:33:56.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.305+0000 7f6b92c7b740 1 --2- 172.21.15.32:0/2607981790 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x561763f72f50 0x561763f74e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.314 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.305+0000 7f6b92c7b740 1 --2- 172.21.15.32:0/2607981790 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561763f72630 0x561763f794f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.314 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.305+0000 7f6b92c7b740 1 --2- 172.21.15.32:0/2607981790 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f6e950 0x561763f7cce0 unknown :-1 s=CLOSED pgs=551 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.314 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.305+0000 7f6b92c7b740 1 -- 172.21.15.32:0/2607981790 >> 172.21.15.32:0/2607981790 conn(0x561763f75f80 msgr2=0x561763f89010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:56.314 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.305+0000 7f6b92c7b740 1 -- 172.21.15.32:0/2607981790 shutdown_connections 2024-04-25T00:33:56.314 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.305+0000 7f6b92c7b740 1 -- 172.21.15.32:0/2607981790 wait complete. 2024-04-25T00:33:56.314 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.305+0000 7f6b92c7b740 1 -- start start 2024-04-25T00:33:56.314 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.306+0000 7f6b92c7b740 1 Processor -- start 2024-04-25T00:33:56.314 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.306+0000 7f6b92c7b740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f6e950 0x561763f93040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.314 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.306+0000 7f6b92c7b740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561763f72630 0x561763f903d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.314 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.306+0000 7f6b92c7b740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x561763f72f50 0x561763f909e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.314 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.306+0000 7f6b92c7b740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f6b881ad3a0 con 0x561763f6e950 2024-04-25T00:33:56.314 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.306+0000 7f6b92c7b740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f6b881ae810 con 0x561763f72f50 2024-04-25T00:33:56.314 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.306+0000 7f6b92c7b740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f6b881ada70 con 0x561763f72630 2024-04-25T00:33:56.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.306+0000 7f6b92c7b740 10 client.0 _ll_register_callbacks cb 0x7f6b78005e30 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:33:56.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.306+0000 7f6b8dcff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f6e950 0x561763f93040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.306+0000 7f6b8dcff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f6e950 0x561763f93040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53786/0 (socket says 172.21.15.32:53786) 2024-04-25T00:33:56.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.306+0000 7f6b8dcff640 1 -- 172.21.15.32:0/859145816 learned_addr learned my addr 172.21.15.32:0/859145816 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:33:56.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.306+0000 7f6b8e500640 1 --2- 172.21.15.32:0/859145816 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x561763f72f50 0x561763f909e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.306+0000 7f6b8d4fe640 1 --2- 172.21.15.32:0/859145816 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561763f72630 0x561763f903d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.306+0000 7f6b8dcff640 1 -- 172.21.15.32:0/859145816 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561763f72630 msgr2=0x561763f903d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:56.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.306+0000 7f6b8dcff640 1 --2- 172.21.15.32:0/859145816 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561763f72630 0x561763f903d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.306+0000 7f6b8dcff640 1 -- 172.21.15.32:0/859145816 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x561763f72f50 msgr2=0x561763f909e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:33:56.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.306+0000 7f6b8dcff640 1 --2- 172.21.15.32:0/859145816 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x561763f72f50 0x561763f909e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.306+0000 7f6b8dcff640 1 -- 172.21.15.32:0/859145816 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x561763f91210 con 0x561763f6e950 2024-04-25T00:33:56.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.306+0000 7f6b8e500640 1 --2- 172.21.15.32:0/859145816 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x561763f72f50 0x561763f909e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:33:56.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.306+0000 7f6b8d4fe640 1 --2- 172.21.15.32:0/859145816 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561763f72630 0x561763f903d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:33:56.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.307+0000 7f6b8dcff640 1 --2- 172.21.15.32:0/859145816 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f6e950 0x561763f93040 secure :-1 s=READY pgs=552 cs=0 l=1 rev1=1 crypto rx=0x7f6b84011830 tx=0x7f6b84004450 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9652350204e5be50 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:33:56.316 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.307+0000 7f6b717fa640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:33:56.316 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.307+0000 7f6b717fa640 1 -- 172.21.15.32:0/859145816 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b8401d420 con 0x561763f6e950 2024-04-25T00:33:56.316 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.307+0000 7f6b717fa640 1 -- 172.21.15.32:0/859145816 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f6b8401d5c0 con 0x561763f6e950 2024-04-25T00:33:56.316 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.307+0000 7f6b92c7b740 1 -- 172.21.15.32:0/859145816 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f6b8401d900 con 0x561763f6e950 2024-04-25T00:33:56.316 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.307+0000 7f6b717fa640 1 -- 172.21.15.32:0/859145816 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f6b84011890 con 0x561763f6e950 2024-04-25T00:33:56.316 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.307+0000 7f6b92c7b740 10 client.15849 fetch_fsmap learned FSMap version 25 2024-04-25T00:33:56.316 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.307+0000 7f6b92c7b740 1 -- 172.21.15.32:0/859145816 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x561763ff3a50 con 0x561763f6e950 2024-04-25T00:33:56.316 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.307+0000 7f6b717fa640 1 -- 172.21.15.32:0/859145816 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6b8401d900 con 0x561763f6e950 2024-04-25T00:33:56.316 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.307+0000 7f6b717fa640 1 -- 172.21.15.32:0/859145816 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7f6b840046c0 con 0x561763f6e950 2024-04-25T00:33:56.316 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.307+0000 7f6b92c7b740 10 client.15849 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:33:56.317 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.307+0000 7f6b92c7b740 10 client.15849 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:33:56.317 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.307+0000 7f6b92c7b740 20 client.15849 populate_metadata read hostname 'smithi032' 2024-04-25T00:33:56.317 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.307+0000 7f6b92c7b740 1 --2- 172.21.15.32:0/859145816 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x561763f7d620 0x561763ff6170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:33:56.317 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.307+0000 7f6b92c7b740 4 client.15849 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-25T00:33:56.317 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.307+0000 7f6b92c7b740 1 -- 172.21.15.32:0/859145816 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "scrub status"}) -- 0x561763edef40 con 0x561763f7d620 2024-04-25T00:33:56.317 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.308+0000 7f6b8d4fe640 1 --2- 172.21.15.32:0/859145816 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x561763f7d620 0x561763ff6170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:33:56.317 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.308+0000 7f6b717fa640 10 client.15849 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:56.317 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.308+0000 7f6b8d4fe640 1 --2- 172.21.15.32:0/859145816 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x561763f7d620 0x561763ff6170 crc :-1 s=READY pgs=51 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=24beda5fc8eaa718 server_cookie=38dcd1c5ef2db2dc in_seq=0 out_seq=0 2024-04-25T00:33:56.317 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.308+0000 7f6b717fa640 1 -- 172.21.15.32:0/859145816 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+259 (crc 0 0 0) 0x561763edef40 con 0x561763f7d620 2024-04-25T00:33:56.317 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.308+0000 7f6b717fa640 10 client.15849 handle_command_reply: tid=0 2024-04-25T00:33:56.317 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.308+0000 7f6b92c7b740 1 client.15849 shutdown 2024-04-25T00:33:56.317 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.308+0000 7f6b92c7b740 1 -- 172.21.15.32:0/859145816 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f6e950 msgr2=0x561763f93040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:56.317 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.308+0000 7f6b92c7b740 1 --2- 172.21.15.32:0/859145816 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f6e950 0x561763f93040 secure :-1 s=READY pgs=552 cs=0 l=1 rev1=1 crypto rx=0x7f6b84011830 tx=0x7f6b84004450 comp rx=0 tx=0).stop 2024-04-25T00:33:56.317 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.308+0000 7f6b92c7b740 1 -- 172.21.15.32:0/859145816 shutdown_connections 2024-04-25T00:33:56.318 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.308+0000 7f6b92c7b740 1 --2- 172.21.15.32:0/859145816 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x561763f7d620 0x561763ff6170 crc :-1 s=READY pgs=51 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.318 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.309+0000 7f6b92c7b740 1 --2- 172.21.15.32:0/859145816 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x561763f72f50 0x561763f909e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.318 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.309+0000 7f6b92c7b740 1 --2- 172.21.15.32:0/859145816 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561763f72630 0x561763f903d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.318 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.309+0000 7f6b717fa640 0 client.15849 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:33:56.318 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.309+0000 7f6b92c7b740 1 --2- 172.21.15.32:0/859145816 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x561763f6e950 0x561763f93040 unknown :-1 s=CLOSED pgs=552 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.318 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.309+0000 7f6b92c7b740 1 -- 172.21.15.32:0/859145816 >> 172.21.15.32:0/859145816 conn(0x561763f75f80 msgr2=0x561763f7a3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:56.318 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.309+0000 7f6b92c7b740 1 -- 172.21.15.32:0/859145816 shutdown_connections 2024-04-25T00:33:56.318 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.309+0000 7f6b92c7b740 1 -- 172.21.15.32:0/859145816 wait complete. 2024-04-25T00:33:56.318 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.309+0000 7f6b92c7b740 20 client.15849 trim_cache size 0 max 16384 2024-04-25T00:33:56.318 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:33:56.319 INFO:teuthology.orchestra.run.smithi032.stdout: "status": "IDLE (waiting for more scrubs)", 2024-04-25T00:33:56.319 INFO:teuthology.orchestra.run.smithi032.stdout: "scrubs": { 2024-04-25T00:33:56.319 INFO:teuthology.orchestra.run.smithi032.stdout: "93079411-5a82-4ebf-a044-6a1c24977bc2": { 2024-04-25T00:33:56.319 INFO:teuthology.orchestra.run.smithi032.stdout: "path": "/", 2024-04-25T00:33:56.319 INFO:teuthology.orchestra.run.smithi032.stdout: "tag": "93079411-5a82-4ebf-a044-6a1c24977bc2", 2024-04-25T00:33:56.319 INFO:teuthology.orchestra.run.smithi032.stdout: "options": "recursive,force" 2024-04-25T00:33:56.319 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:33:56.319 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:33:56.319 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:33:56.320 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.309+0000 7f6b8ff63640 1 -- 172.21.15.32:0/2477626023 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f6b740a2d90 msgr2=0x7f6b740a5250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:56.320 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.309+0000 7f6b8ff63640 1 --2- 172.21.15.32:0/2477626023 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f6b740a2d90 0x7f6b740a5250 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f6b80004af0 tx=0x7f6b800095c0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.320 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.309+0000 7f6b8ff63640 1 -- 172.21.15.32:0/2477626023 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6b881020e0 msgr2=0x7f6b88198500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:33:56.320 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.309+0000 7f6b8ff63640 1 --2- 172.21.15.32:0/2477626023 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6b881020e0 0x7f6b88198500 secure :-1 s=READY pgs=549 cs=0 l=1 rev1=1 crypto rx=0x7f6b7800e9f0 tx=0x7f6b7800eeb0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.320 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.309+0000 7f6b8ff63640 1 -- 172.21.15.32:0/2477626023 shutdown_connections 2024-04-25T00:33:56.320 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.309+0000 7f6b8ff63640 1 --2- 172.21.15.32:0/2477626023 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f6b740a2d90 0x7f6b740a5250 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.320 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.309+0000 7f6b8ff63640 1 --2- 172.21.15.32:0/2477626023 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6b88107730 0x7f6b8819c880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.320 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.309+0000 7f6b8ff63640 1 --2- 172.21.15.32:0/2477626023 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6b881020e0 0x7f6b88198500 unknown :-1 s=CLOSED pgs=549 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.320 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.309+0000 7f6b8ff63640 1 --2- 172.21.15.32:0/2477626023 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6b88101720 0x7f6b88197fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:33:56.321 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.309+0000 7f6b8ff63640 1 -- 172.21.15.32:0/2477626023 >> 172.21.15.32:0/2477626023 conn(0x7f6b88078b40 msgr2=0x7f6b88104030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:33:56.321 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.309+0000 7f6b8ff63640 1 -- 172.21.15.32:0/2477626023 shutdown_connections 2024-04-25T00:33:56.321 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:33:56.309+0000 7f6b8ff63640 1 -- 172.21.15.32:0/2477626023 wait complete. 2024-04-25T00:33:56.322 INFO:tasks.cephfs.filesystem:scrub status for tag:93079411-5a82-4ebf-a044-6a1c24977bc2 - {'path': '/', 'tag': '93079411-5a82-4ebf-a044-6a1c24977bc2', 'options': 'recursive,force'} 2024-04-25T00:33:56.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:56 smithi032 ceph-mon[29357]: scrub queued for path: / 2024-04-25T00:33:56.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:56 smithi032 ceph-mon[29357]: scrub summary: idle+waiting paths [/] 2024-04-25T00:33:56.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:56 smithi032 ceph-mon[29357]: scrub summary: active paths [/] 2024-04-25T00:33:56.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:56 smithi032 ceph-mon[29357]: scrub summary: idle+waiting paths [/] 2024-04-25T00:33:56.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:56 smithi121 ceph-mon[32231]: scrub queued for path: / 2024-04-25T00:33:56.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:56 smithi121 ceph-mon[32231]: scrub summary: idle+waiting paths [/] 2024-04-25T00:33:56.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:56 smithi121 ceph-mon[32231]: scrub summary: active paths [/] 2024-04-25T00:33:56.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:56 smithi121 ceph-mon[32231]: scrub summary: idle+waiting paths [/] 2024-04-25T00:33:56.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:56 smithi130 ceph-mon[32525]: scrub queued for path: / 2024-04-25T00:33:56.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:56 smithi130 ceph-mon[32525]: scrub summary: idle+waiting paths [/] 2024-04-25T00:33:56.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:56 smithi130 ceph-mon[32525]: scrub summary: active paths [/] 2024-04-25T00:33:56.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:56 smithi130 ceph-mon[32525]: scrub summary: idle+waiting paths [/] 2024-04-25T00:33:57.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:57 smithi121 ceph-mon[32231]: pgmap v22: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:33:57.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:57 smithi121 ceph-mon[32231]: scrub summary: idle 2024-04-25T00:33:57.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:57 smithi130 ceph-mon[32525]: pgmap v22: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:33:57.895 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:57 smithi130 ceph-mon[32525]: scrub summary: idle 2024-04-25T00:33:57.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:57 smithi032 ceph-mon[29357]: pgmap v22: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:33:57.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:57 smithi032 ceph-mon[29357]: scrub summary: idle 2024-04-25T00:33:59.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:33:59 smithi121 ceph-mon[32231]: pgmap v23: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:33:59.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:33:59 smithi130 ceph-mon[32525]: pgmap v23: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:33:59.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:33:59 smithi032 ceph-mon[29357]: pgmap v23: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:34:00.998 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:34:00 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:34:00.558+0000 7fc49dffe700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-25T00:34:00.998 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:34:00 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:34:00.558+0000 7fc49dffe700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-25T00:34:00.998 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:34:00 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:34:00.558+0000 7fc49dffe700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-25T00:34:00.998 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:34:00 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:34:00.558+0000 7fc49dffe700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-25T00:34:00.999 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:34:00 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:34:00.558+0000 7fc49dffe700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-25T00:34:01.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:01 smithi121 ceph-mon[32231]: pgmap v24: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.9 KiB/s rd, 7 op/s 2024-04-25T00:34:01.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:01 smithi130 ceph-mon[32525]: pgmap v24: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.9 KiB/s rd, 7 op/s 2024-04-25T00:34:01.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:01 smithi032 ceph-mon[29357]: pgmap v24: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.9 KiB/s rd, 7 op/s 2024-04-25T00:34:03.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:03 smithi121 ceph-mon[32231]: pgmap v25: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 5 op/s 2024-04-25T00:34:03.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:03 smithi121 ceph-mon[32231]: mgrmap e25: x(active, since 44s), standbys: z, y 2024-04-25T00:34:03.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:03 smithi130 ceph-mon[32525]: pgmap v25: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 5 op/s 2024-04-25T00:34:03.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:03 smithi130 ceph-mon[32525]: mgrmap e25: x(active, since 44s), standbys: z, y 2024-04-25T00:34:03.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:03 smithi032 ceph-mon[29357]: pgmap v25: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 5 op/s 2024-04-25T00:34:03.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:03 smithi032 ceph-mon[29357]: mgrmap e25: x(active, since 44s), standbys: z, y 2024-04-25T00:34:05.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:05 smithi121 ceph-mon[32231]: pgmap v26: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 12 KiB/s wr, 21 op/s 2024-04-25T00:34:05.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:05 smithi130 ceph-mon[32525]: pgmap v26: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 12 KiB/s wr, 21 op/s 2024-04-25T00:34:05.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:05 smithi032 ceph-mon[29357]: pgmap v26: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 12 KiB/s wr, 21 op/s 2024-04-25T00:34:07.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:07 smithi121 ceph-mon[32231]: pgmap v27: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 23 KiB/s wr, 21 op/s 2024-04-25T00:34:07.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:07 smithi130 ceph-mon[32525]: pgmap v27: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 23 KiB/s wr, 21 op/s 2024-04-25T00:34:07.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:07 smithi032 ceph-mon[29357]: pgmap v27: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 23 KiB/s wr, 21 op/s 2024-04-25T00:34:09.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:09 smithi121 ceph-mon[32231]: pgmap v28: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 23 KiB/s wr, 20 op/s 2024-04-25T00:34:09.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:09 smithi130 ceph-mon[32525]: pgmap v28: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 23 KiB/s wr, 20 op/s 2024-04-25T00:34:09.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:09 smithi032 ceph-mon[29357]: pgmap v28: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 23 KiB/s wr, 20 op/s 2024-04-25T00:34:11.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:11 smithi130 ceph-mon[32525]: pgmap v29: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 23 KiB/s wr, 23 op/s 2024-04-25T00:34:11.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:11 smithi032 ceph-mon[29357]: pgmap v29: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 23 KiB/s wr, 23 op/s 2024-04-25T00:34:12.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:11 smithi121 ceph-mon[32231]: pgmap v29: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 23 KiB/s wr, 23 op/s 2024-04-25T00:34:13.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:13 smithi130 ceph-mon[32525]: pgmap v30: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 23 KiB/s wr, 20 op/s 2024-04-25T00:34:13.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:13 smithi032 ceph-mon[29357]: pgmap v30: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 23 KiB/s wr, 20 op/s 2024-04-25T00:34:14.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:13 smithi121 ceph-mon[32231]: pgmap v30: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 23 KiB/s wr, 20 op/s 2024-04-25T00:34:15.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:15 smithi130 ceph-mon[32525]: pgmap v31: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 23 KiB/s wr, 23 op/s 2024-04-25T00:34:15.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:15 smithi032 ceph-mon[29357]: pgmap v31: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 23 KiB/s wr, 23 op/s 2024-04-25T00:34:16.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:15 smithi121 ceph-mon[32231]: pgmap v31: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 23 KiB/s wr, 23 op/s 2024-04-25T00:34:17.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:17 smithi130 ceph-mon[32525]: pgmap v32: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.5 KiB/s rd, 11 KiB/s wr, 7 op/s 2024-04-25T00:34:17.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:17 smithi032 ceph-mon[29357]: pgmap v32: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.5 KiB/s rd, 11 KiB/s wr, 7 op/s 2024-04-25T00:34:18.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:17 smithi121 ceph-mon[32231]: pgmap v32: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.5 KiB/s rd, 11 KiB/s wr, 7 op/s 2024-04-25T00:34:19.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:19 smithi130 ceph-mon[32525]: pgmap v33: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.6 KiB/s rd, 85 B/s wr, 5 op/s 2024-04-25T00:34:19.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:19 smithi032 ceph-mon[29357]: pgmap v33: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.6 KiB/s rd, 85 B/s wr, 5 op/s 2024-04-25T00:34:20.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:19 smithi121 ceph-mon[32231]: pgmap v33: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.6 KiB/s rd, 85 B/s wr, 5 op/s 2024-04-25T00:34:21.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:21 smithi130 ceph-mon[32525]: pgmap v34: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.8 KiB/s rd, 85 B/s wr, 7 op/s 2024-04-25T00:34:21.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:21 smithi032 ceph-mon[29357]: pgmap v34: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.8 KiB/s rd, 85 B/s wr, 7 op/s 2024-04-25T00:34:22.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:21 smithi121 ceph-mon[32231]: pgmap v34: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.8 KiB/s rd, 85 B/s wr, 7 op/s 2024-04-25T00:34:22.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:22 smithi130 ceph-mon[32525]: pgmap v35: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:34:22.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:22 smithi032 ceph-mon[29357]: pgmap v35: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:34:23.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:22 smithi121 ceph-mon[32231]: pgmap v35: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:34:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:23 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:34:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:23 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:34:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:23 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:34:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:23 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:34:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:23 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:34:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:23 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:34:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:23 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:34:23.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:23 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:34:23.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:23 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:34:23.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:23 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:34:23.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:23 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:34:23.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:23 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:34:23.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:23 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:34:23.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:23 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:34:24.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:23 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:34:24.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:23 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:34:24.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:23 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:34:24.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:23 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:34:24.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:23 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:34:24.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:23 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:34:24.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:23 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:34:24.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:24 smithi130 ceph-mon[32525]: pgmap v36: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 7 op/s 2024-04-25T00:34:24.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:24 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:34:24.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:24 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:34:24.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:24 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:34:24.989 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:24 smithi121 ceph-mon[32231]: pgmap v36: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 7 op/s 2024-04-25T00:34:24.989 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:24 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:34:24.989 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:24 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:34:24.989 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:24 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:34:24.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:24 smithi032 ceph-mon[29357]: pgmap v36: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 7 op/s 2024-04-25T00:34:24.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:24 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:34:24.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:24 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:34:24.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:24 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:34:26.323 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-25T00:34:26.324 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-25T00:34:26.435 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.434+0000 7f5ac5970640 1 -- 172.21.15.32:0/2435833784 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5ac010dfc0 msgr2=0x7f5ac01103d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:26.435 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.434+0000 7f5ac5970640 1 --2- 172.21.15.32:0/2435833784 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5ac010dfc0 0x7f5ac01103d0 secure :-1 s=READY pgs=555 cs=0 l=1 rev1=1 crypto rx=0x7f5ab8009ca0 tx=0x7f5ab802a950 comp rx=0 tx=0).stop 2024-04-25T00:34:26.435 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.434+0000 7f5ac5970640 1 -- 172.21.15.32:0/2435833784 shutdown_connections 2024-04-25T00:34:26.435 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.434+0000 7f5ac5970640 1 --2- 172.21.15.32:0/2435833784 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5ac010dfc0 0x7f5ac01103d0 unknown :-1 s=CLOSED pgs=555 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.435 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.434+0000 7f5ac5970640 1 --2- 172.21.15.32:0/2435833784 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ac0074430 0x7f5ac010da80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.436 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.434+0000 7f5ac5970640 1 --2- 172.21.15.32:0/2435833784 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5ac0073b10 0x7f5ac0073ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.436 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.434+0000 7f5ac5970640 1 -- 172.21.15.32:0/2435833784 >> 172.21.15.32:0/2435833784 conn(0x7f5ac0078b40 msgr2=0x7f5ac0078f50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:26.436 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.434+0000 7f5ac5970640 1 -- 172.21.15.32:0/2435833784 shutdown_connections 2024-04-25T00:34:26.436 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.434+0000 7f5ac5970640 1 -- 172.21.15.32:0/2435833784 wait complete. 2024-04-25T00:34:26.436 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.435+0000 7f5ac5970640 1 Processor -- start 2024-04-25T00:34:26.436 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.435+0000 7f5ac5970640 1 -- start start 2024-04-25T00:34:26.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.435+0000 7f5ac5970640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ac0073b10 0x7f5ac019a240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.435+0000 7f5ac5970640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5ac0074430 0x7f5ac019a780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.435+0000 7f5ac5970640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5ac010dfc0 0x7f5ac019eb00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.435+0000 7f5ac5970640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f5ac0113d20 con 0x7f5ac010dfc0 2024-04-25T00:34:26.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.435+0000 7f5ac5970640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f5ac0113ba0 con 0x7f5ac0074430 2024-04-25T00:34:26.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.435+0000 7f5ac5970640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f5ac0113ea0 con 0x7f5ac0073b10 2024-04-25T00:34:26.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.435+0000 7f5abf7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5ac010dfc0 0x7f5ac019eb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:26.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.435+0000 7f5abf7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5ac010dfc0 0x7f5ac019eb00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39754/0 (socket says 172.21.15.32:39754) 2024-04-25T00:34:26.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.435+0000 7f5abf7fe640 1 -- 172.21.15.32:0/2326115253 learned_addr learned my addr 172.21.15.32:0/2326115253 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:26.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.435+0000 7f5abeffd640 1 --2- 172.21.15.32:0/2326115253 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ac0073b10 0x7f5ac019a240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:26.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.435+0000 7f5abe7fc640 1 --2- 172.21.15.32:0/2326115253 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5ac0074430 0x7f5ac019a780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:26.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.436+0000 7f5abf7fe640 1 -- 172.21.15.32:0/2326115253 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ac0073b10 msgr2=0x7f5ac019a240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:26.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.436+0000 7f5abf7fe640 1 --2- 172.21.15.32:0/2326115253 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ac0073b10 0x7f5ac019a240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.436+0000 7f5abf7fe640 1 -- 172.21.15.32:0/2326115253 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5ac0074430 msgr2=0x7f5ac019a780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:26.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.436+0000 7f5abf7fe640 1 --2- 172.21.15.32:0/2326115253 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5ac0074430 0x7f5ac019a780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.436+0000 7f5abf7fe640 1 -- 172.21.15.32:0/2326115253 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5ac019f230 con 0x7f5ac010dfc0 2024-04-25T00:34:26.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.436+0000 7f5abf7fe640 1 --2- 172.21.15.32:0/2326115253 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5ac010dfc0 0x7f5ac019eb00 secure :-1 s=READY pgs=556 cs=0 l=1 rev1=1 crypto rx=0x7f5ab8005e30 tx=0x7f5ab802f630 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:26.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.436+0000 7f5ac496e640 1 -- 172.21.15.32:0/2326115253 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ab8034040 con 0x7f5ac010dfc0 2024-04-25T00:34:26.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.436+0000 7f5ac5970640 1 -- 172.21.15.32:0/2326115253 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5ac019f4c0 con 0x7f5ac010dfc0 2024-04-25T00:34:26.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.436+0000 7f5ac5970640 1 -- 172.21.15.32:0/2326115253 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5ac01a6db0 con 0x7f5ac010dfc0 2024-04-25T00:34:26.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.436+0000 7f5ac496e640 1 -- 172.21.15.32:0/2326115253 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5ab8030920 con 0x7f5ac010dfc0 2024-04-25T00:34:26.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.436+0000 7f5ac496e640 1 -- 172.21.15.32:0/2326115253 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ab8004780 con 0x7f5ac010dfc0 2024-04-25T00:34:26.441 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.439+0000 7f5ac496e640 1 -- 172.21.15.32:0/2326115253 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 25) ==== 138066+0+0 (secure 0 0 0) 0x7f5ab8037020 con 0x7f5ac010dfc0 2024-04-25T00:34:26.441 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.439+0000 7f5ac8681740 1 Processor -- start 2024-04-25T00:34:26.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.439+0000 7f5ac8681740 1 -- start start 2024-04-25T00:34:26.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.440+0000 7f5ac8681740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555ade78a630 0x555ade78aa10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.440+0000 7f5ac8681740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78af50 0x555ade7950e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.440+0000 7f5ac8681740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555ade795620 0x555ade797a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.440+0000 7f5ac8681740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x555ade6ef6b0 con 0x555ade78af50 2024-04-25T00:34:26.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.440+0000 7f5ac8681740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x555ade697880 con 0x555ade795620 2024-04-25T00:34:26.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.440+0000 7f5ac8681740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x555ade6eb670 con 0x555ade78a630 2024-04-25T00:34:26.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.440+0000 7f5abeffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78af50 0x555ade7950e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:26.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.440+0000 7f5abeffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78af50 0x555ade7950e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39768/0 (socket says 172.21.15.32:39768) 2024-04-25T00:34:26.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.440+0000 7f5abeffd640 1 -- 172.21.15.32:0/3214576022 learned_addr learned my addr 172.21.15.32:0/3214576022 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:26.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.440+0000 7f5abe7fc640 1 --2- 172.21.15.32:0/3214576022 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555ade795620 0x555ade797a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:26.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.440+0000 7f5abe7fc640 1 --2- 172.21.15.32:0/3214576022 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555ade78a630 0x555ade78aa10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:26.443 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.440+0000 7f5ac496e640 1 --2- 172.21.15.32:0/2326115253 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f5aa80a2e20 0x7f5aa80a52e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.443 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.440+0000 7f5abeffd640 1 -- 172.21.15.32:0/3214576022 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555ade78a630 msgr2=0x555ade78aa10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:26.443 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.440+0000 7f5abeffd640 1 --2- 172.21.15.32:0/3214576022 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555ade78a630 0x555ade78aa10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.443 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.440+0000 7f5abeffd640 1 -- 172.21.15.32:0/3214576022 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555ade795620 msgr2=0x555ade797a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:26.443 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.440+0000 7f5abeffd640 1 --2- 172.21.15.32:0/3214576022 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555ade795620 0x555ade797a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.443 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.440+0000 7f5abeffd640 1 -- 172.21.15.32:0/3214576022 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x555ade4cd7d0 con 0x555ade78af50 2024-04-25T00:34:26.443 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.440+0000 7f5ac496e640 1 -- 172.21.15.32:0/2326115253 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f5ab80f0740 con 0x7f5ac010dfc0 2024-04-25T00:34:26.443 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.441+0000 7f5abf7fe640 1 --2- 172.21.15.32:0/2326115253 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f5aa80a2e20 0x7f5aa80a52e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:26.443 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.441+0000 7f5abeffd640 1 --2- 172.21.15.32:0/3214576022 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78af50 0x555ade7950e0 secure :-1 s=READY pgs=557 cs=0 l=1 rev1=1 crypto rx=0x7f5ab000bd10 tx=0x7f5ab00107a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dcb6b2314a27fd server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:26.443 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.441+0000 7f5aa5ffb640 1 -- 172.21.15.32:0/3214576022 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ab0004030 con 0x555ade78af50 2024-04-25T00:34:26.443 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.441+0000 7f5aa5ffb640 1 -- 172.21.15.32:0/3214576022 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5ab00041d0 con 0x555ade78af50 2024-04-25T00:34:26.443 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.441+0000 7f5aa5ffb640 1 -- 172.21.15.32:0/3214576022 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ab0011800 con 0x555ade78af50 2024-04-25T00:34:26.448 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.442+0000 7f5abf7fe640 1 --2- 172.21.15.32:0/2326115253 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f5aa80a2e20 0x7f5aa80a52e0 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f5ab80edad0 tx=0x7f5ab802a950 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:26.448 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.442+0000 7f5ac8681740 1 -- 172.21.15.32:0/3214576022 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78af50 msgr2=0x555ade7950e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:26.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.442+0000 7f5ac8681740 1 --2- 172.21.15.32:0/3214576022 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78af50 0x555ade7950e0 secure :-1 s=READY pgs=557 cs=0 l=1 rev1=1 crypto rx=0x7f5ab000bd10 tx=0x7f5ab00107a0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.442+0000 7f5ac8681740 1 -- 172.21.15.32:0/3214576022 shutdown_connections 2024-04-25T00:34:26.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.442+0000 7f5ac8681740 1 --2- 172.21.15.32:0/3214576022 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555ade795620 0x555ade797a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.442+0000 7f5ac8681740 1 --2- 172.21.15.32:0/3214576022 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78af50 0x555ade7950e0 unknown :-1 s=CLOSED pgs=557 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.442+0000 7f5ac8681740 1 --2- 172.21.15.32:0/3214576022 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555ade78a630 0x555ade78aa10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.442+0000 7f5ac8681740 1 -- 172.21.15.32:0/3214576022 >> 172.21.15.32:0/3214576022 conn(0x555ade786950 msgr2=0x555ade786d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:26.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.442+0000 7f5ac8681740 1 -- 172.21.15.32:0/3214576022 shutdown_connections 2024-04-25T00:34:26.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.442+0000 7f5ac8681740 1 -- 172.21.15.32:0/3214576022 wait complete. 2024-04-25T00:34:26.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.442+0000 7f5ac8681740 1 -- start start 2024-04-25T00:34:26.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.443+0000 7f5ac8681740 1 Processor -- start 2024-04-25T00:34:26.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.443+0000 7f5ac8681740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555ade78a630 0x555ade7924c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.443+0000 7f5ac8681740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78af50 0x555ade792a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.443+0000 7f5ac8681740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555ade795620 0x555ade792f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.443+0000 7f5ac8681740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x555ade798410 con 0x555ade78af50 2024-04-25T00:34:26.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.443+0000 7f5ac8681740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x555ade6aaf50 con 0x555ade795620 2024-04-25T00:34:26.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.443+0000 7f5ac8681740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x555ade798590 con 0x555ade78a630 2024-04-25T00:34:26.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.443+0000 7f5ac8681740 10 client.0 _ll_register_callbacks cb 0x555ade6e6000 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:34:26.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.443+0000 7f5abe7fc640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555ade78a630 0x555ade7924c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:26.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.443+0000 7f5abe7fc640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555ade795620 0x555ade792f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:26.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.443+0000 7f5abe7fc640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555ade78a630 0x555ade7924c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:54392/0 (socket says 172.21.15.32:54392) 2024-04-25T00:34:26.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.443+0000 7f5abe7fc640 1 -- 172.21.15.32:0/3933977233 learned_addr learned my addr 172.21.15.32:0/3933977233 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:26.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.443+0000 7f5abeffd640 1 --2- 172.21.15.32:0/3933977233 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78af50 0x555ade792a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:26.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.443+0000 7f5abe7fc640 1 -- 172.21.15.32:0/3933977233 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555ade795620 msgr2=0x555ade792f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:26.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.443+0000 7f5abe7fc640 1 --2- 172.21.15.32:0/3933977233 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555ade795620 0x555ade792f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.443+0000 7f5abe7fc640 1 -- 172.21.15.32:0/3933977233 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78af50 msgr2=0x555ade792a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:26.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.443+0000 7f5abe7fc640 1 --2- 172.21.15.32:0/3933977233 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78af50 0x555ade792a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.443+0000 7f5abe7fc640 1 -- 172.21.15.32:0/3933977233 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x555ade4033f0 con 0x555ade78a630 2024-04-25T00:34:26.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.444+0000 7f5abeffd640 1 --2- 172.21.15.32:0/3933977233 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78af50 0x555ade792a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:34:26.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.444+0000 7f5abe7fc640 1 --2- 172.21.15.32:0/3933977233 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555ade78a630 0x555ade7924c0 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f5ab40116d0 tx=0x7f5ab400fb40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9d88b341fd410436 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:26.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.444+0000 7f5a9effd640 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:34:26.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.444+0000 7f5a9effd640 1 -- 172.21.15.32:0/3933977233 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ab4014110 con 0x555ade78a630 2024-04-25T00:34:26.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.444+0000 7f5a9effd640 1 -- 172.21.15.32:0/3933977233 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f5ab40142b0 con 0x555ade78a630 2024-04-25T00:34:26.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.444+0000 7f5ac8681740 1 -- 172.21.15.32:0/3933977233 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5ab00041d0 con 0x555ade78a630 2024-04-25T00:34:26.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.444+0000 7f5a9effd640 1 -- 172.21.15.32:0/3933977233 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f5ab4004020 con 0x555ade78a630 2024-04-25T00:34:26.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.444+0000 7f5ac8681740 10 client.25405 fetch_fsmap learned FSMap version 25 2024-04-25T00:34:26.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.444+0000 7f5ac8681740 1 -- 172.21.15.32:0/3933977233 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x555ade3c30f0 con 0x555ade78a630 2024-04-25T00:34:26.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.445+0000 7f5a9effd640 1 -- 172.21.15.32:0/3933977233 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5ab00041d0 con 0x555ade78a630 2024-04-25T00:34:26.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.445+0000 7f5a9effd640 1 -- 172.21.15.32:0/3933977233 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7f5ab40035f0 con 0x555ade78a630 2024-04-25T00:34:26.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.445+0000 7f5ac8681740 10 client.25405 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:34:26.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.445+0000 7f5ac8681740 10 client.25405 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:34:26.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.445+0000 7f5ac8681740 20 client.25405 populate_metadata read hostname 'smithi032' 2024-04-25T00:34:26.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.445+0000 7f5ac8681740 1 --2- 172.21.15.32:0/3933977233 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x555ade7a5cf0 0x555ade7a8170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.445+0000 7f5ac8681740 4 client.25405 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:34:26.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.445+0000 7f5ac8681740 1 -- 172.21.15.32:0/3933977233 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x555ade6fcbf0 con 0x555ade7a5cf0 2024-04-25T00:34:26.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.446+0000 7f5abeffd640 1 --2- 172.21.15.32:0/3933977233 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x555ade7a5cf0 0x555ade7a8170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:26.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.446+0000 7f5a9effd640 10 client.25405 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:34:26.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.446+0000 7f5abeffd640 1 --2- 172.21.15.32:0/3933977233 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x555ade7a5cf0 0x555ade7a8170 crc :-1 s=READY pgs=53 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d0171104f463e062 server_cookie=87b8591e10e6736e in_seq=0 out_seq=0 2024-04-25T00:34:26.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.447+0000 7f5a9effd640 1 -- 172.21.15.32:0/3933977233 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x555ade6fcbf0 con 0x555ade7a5cf0 2024-04-25T00:34:26.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.447+0000 7f5a9effd640 10 client.25405 handle_command_reply: tid=0 2024-04-25T00:34:26.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.447+0000 7f5ac8681740 1 client.25405 shutdown 2024-04-25T00:34:26.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.447+0000 7f5ac8681740 1 -- 172.21.15.32:0/3933977233 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555ade78a630 msgr2=0x555ade7924c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:26.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.447+0000 7f5ac8681740 1 --2- 172.21.15.32:0/3933977233 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555ade78a630 0x555ade7924c0 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f5ab40116d0 tx=0x7f5ab400fb40 comp rx=0 tx=0).stop 2024-04-25T00:34:26.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.447+0000 7f5ac8681740 1 -- 172.21.15.32:0/3933977233 shutdown_connections 2024-04-25T00:34:26.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.447+0000 7f5ac8681740 1 --2- 172.21.15.32:0/3933977233 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x555ade7a5cf0 0x555ade7a8170 crc :-1 s=READY pgs=53 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.447+0000 7f5ac8681740 1 --2- 172.21.15.32:0/3933977233 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555ade795620 0x555ade792f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.447+0000 7f5ac8681740 1 --2- 172.21.15.32:0/3933977233 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78af50 0x555ade792a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.447+0000 7f5ac8681740 1 --2- 172.21.15.32:0/3933977233 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555ade78a630 0x555ade7924c0 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.453 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.447+0000 7f5ac8681740 1 -- 172.21.15.32:0/3933977233 >> 172.21.15.32:0/3933977233 conn(0x555ade786950 msgr2=0x555ade785af0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:26.453 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.447+0000 7f5a9effd640 0 client.25405 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:34:26.453 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.447+0000 7f5ac8681740 1 -- 172.21.15.32:0/3933977233 shutdown_connections 2024-04-25T00:34:26.453 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.447+0000 7f5ac8681740 1 -- 172.21.15.32:0/3933977233 wait complete. 2024-04-25T00:34:26.453 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.447+0000 7f5ac8681740 20 client.25405 trim_cache size 0 max 16384 2024-04-25T00:34:26.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.482+0000 7f5ac8681740 1 Processor -- start 2024-04-25T00:34:26.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.482+0000 7f5ac8681740 1 -- start start 2024-04-25T00:34:26.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.483+0000 7f5ac8681740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555ade78a630 0x555ade7a1360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.483+0000 7f5ac8681740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78af50 0x555ade7914a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.483+0000 7f5ac8681740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555ade795620 0x555ade7ada40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.483+0000 7f5ac8681740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f5aac012820 con 0x555ade78af50 2024-04-25T00:34:26.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.483+0000 7f5ac8681740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f5aac0100f0 con 0x555ade78a630 2024-04-25T00:34:26.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.483+0000 7f5ac8681740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f5aac00df40 con 0x555ade795620 2024-04-25T00:34:26.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.483+0000 7f5abeffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78af50 0x555ade7914a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:26.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.483+0000 7f5abeffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78af50 0x555ade7914a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39780/0 (socket says 172.21.15.32:39780) 2024-04-25T00:34:26.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.483+0000 7f5abeffd640 1 -- 172.21.15.32:0/4173642542 learned_addr learned my addr 172.21.15.32:0/4173642542 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:26.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.483+0000 7f5abe7fc640 1 --2- 172.21.15.32:0/4173642542 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555ade78a630 0x555ade7a1360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:26.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.483+0000 7f5abe7fc640 1 --2- 172.21.15.32:0/4173642542 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555ade795620 0x555ade7ada40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:26.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.483+0000 7f5abeffd640 1 -- 172.21.15.32:0/4173642542 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555ade795620 msgr2=0x555ade7ada40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:26.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.483+0000 7f5abeffd640 1 --2- 172.21.15.32:0/4173642542 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555ade795620 0x555ade7ada40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.483+0000 7f5abeffd640 1 -- 172.21.15.32:0/4173642542 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555ade78a630 msgr2=0x555ade7a1360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:26.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.483+0000 7f5abeffd640 1 --2- 172.21.15.32:0/4173642542 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555ade78a630 0x555ade7a1360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.483+0000 7f5abeffd640 1 -- 172.21.15.32:0/4173642542 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x555ade78ad80 con 0x555ade78af50 2024-04-25T00:34:26.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.483+0000 7f5abe7fc640 1 --2- 172.21.15.32:0/4173642542 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555ade78a630 0x555ade7a1360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:34:26.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.484+0000 7f5abeffd640 1 --2- 172.21.15.32:0/4173642542 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78af50 0x555ade7914a0 secure :-1 s=READY pgs=558 cs=0 l=1 rev1=1 crypto rx=0x7f5ab000bc30 tx=0x7f5ab0003d40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2254044aea7c038b server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:26.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.484+0000 7f5aa57fa640 1 -- 172.21.15.32:0/4173642542 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ab00035f0 con 0x555ade78af50 2024-04-25T00:34:26.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.484+0000 7f5aa57fa640 1 -- 172.21.15.32:0/4173642542 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5ab002e070 con 0x555ade78af50 2024-04-25T00:34:26.486 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.484+0000 7f5aa57fa640 1 -- 172.21.15.32:0/4173642542 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ab0029110 con 0x555ade78af50 2024-04-25T00:34:26.491 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.485+0000 7f5ac8681740 1 -- 172.21.15.32:0/4173642542 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78af50 msgr2=0x555ade7914a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:26.491 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.485+0000 7f5ac8681740 1 --2- 172.21.15.32:0/4173642542 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78af50 0x555ade7914a0 secure :-1 s=READY pgs=558 cs=0 l=1 rev1=1 crypto rx=0x7f5ab000bc30 tx=0x7f5ab0003d40 comp rx=0 tx=0).stop 2024-04-25T00:34:26.491 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.485+0000 7f5ac8681740 1 -- 172.21.15.32:0/4173642542 shutdown_connections 2024-04-25T00:34:26.491 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.485+0000 7f5ac8681740 1 --2- 172.21.15.32:0/4173642542 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555ade795620 0x555ade7ada40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.491 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.485+0000 7f5ac8681740 1 --2- 172.21.15.32:0/4173642542 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78af50 0x555ade7914a0 unknown :-1 s=CLOSED pgs=558 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.491 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.485+0000 7f5ac8681740 1 --2- 172.21.15.32:0/4173642542 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555ade78a630 0x555ade7a1360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.491 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.485+0000 7f5ac8681740 1 -- 172.21.15.32:0/4173642542 >> 172.21.15.32:0/4173642542 conn(0x555ade786950 msgr2=0x555ade790d90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:26.491 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.485+0000 7f5ac8681740 1 -- 172.21.15.32:0/4173642542 shutdown_connections 2024-04-25T00:34:26.491 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.485+0000 7f5ac8681740 1 -- 172.21.15.32:0/4173642542 wait complete. 2024-04-25T00:34:26.491 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.485+0000 7f5ac8681740 1 -- start start 2024-04-25T00:34:26.491 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.486+0000 7f5ac8681740 1 Processor -- start 2024-04-25T00:34:26.491 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.486+0000 7f5ac8681740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78a630 0x555ade79d870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.491 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.486+0000 7f5ac8681740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555ade795620 0x555ade79ddb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.486+0000 7f5ac8681740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555ade78e730 0x555ade78ebb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.486+0000 7f5ac8681740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f5aac00e5b0 con 0x555ade78a630 2024-04-25T00:34:26.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.486+0000 7f5ac8681740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f5aac00fa20 con 0x555ade78e730 2024-04-25T00:34:26.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.486+0000 7f5ac8681740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f5aac00ec80 con 0x555ade795620 2024-04-25T00:34:26.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.486+0000 7f5abe7fc640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78a630 0x555ade79d870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:26.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.486+0000 7f5abe7fc640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78a630 0x555ade79d870 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39786/0 (socket says 172.21.15.32:39786) 2024-04-25T00:34:26.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.486+0000 7f5ac8681740 10 client.0 _ll_register_callbacks cb 0x7f5ab4014f00 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:34:26.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.486+0000 7f5abe7fc640 1 -- 172.21.15.32:0/3404024318 learned_addr learned my addr 172.21.15.32:0/3404024318 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:26.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.486+0000 7f5abeffd640 1 --2- 172.21.15.32:0/3404024318 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555ade795620 0x555ade79ddb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:26.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.487+0000 7f5abeffd640 1 -- 172.21.15.32:0/3404024318 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555ade78e730 msgr2=0x555ade78ebb0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-25T00:34:26.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.487+0000 7f5abeffd640 1 --2- 172.21.15.32:0/3404024318 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555ade78e730 0x555ade78ebb0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.487+0000 7f5abeffd640 1 -- 172.21.15.32:0/3404024318 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78a630 msgr2=0x555ade79d870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:26.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.487+0000 7f5abeffd640 1 --2- 172.21.15.32:0/3404024318 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78a630 0x555ade79d870 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.487+0000 7f5abeffd640 1 -- 172.21.15.32:0/3404024318 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x555ade78f0f0 con 0x555ade795620 2024-04-25T00:34:26.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.487+0000 7f5abe7fc640 1 --2- 172.21.15.32:0/3404024318 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78a630 0x555ade79d870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:34:26.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.487+0000 7f5abeffd640 1 --2- 172.21.15.32:0/3404024318 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555ade795620 0x555ade79ddb0 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x555ade7a4580 tx=0x7f5ab00058d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c094dd83586566e5 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:26.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.487+0000 7f5a9effd640 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:34:26.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.487+0000 7f5a9effd640 1 -- 172.21.15.32:0/3404024318 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ab0036030 con 0x555ade795620 2024-04-25T00:34:26.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.487+0000 7f5a9effd640 1 -- 172.21.15.32:0/3404024318 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f5ab000b770 con 0x555ade795620 2024-04-25T00:34:26.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.487+0000 7f5ac8681740 1 -- 172.21.15.32:0/3404024318 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5ab002e070 con 0x555ade795620 2024-04-25T00:34:26.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.488+0000 7f5a9effd640 1 -- 172.21.15.32:0/3404024318 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f5ab00035f0 con 0x555ade795620 2024-04-25T00:34:26.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.488+0000 7f5ac8681740 10 client.25408 fetch_fsmap learned FSMap version 25 2024-04-25T00:34:26.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.488+0000 7f5ac8681740 1 -- 172.21.15.32:0/3404024318 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x555ade80be40 con 0x555ade795620 2024-04-25T00:34:26.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.488+0000 7f5a9effd640 1 -- 172.21.15.32:0/3404024318 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5ab002e070 con 0x555ade795620 2024-04-25T00:34:26.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.489+0000 7f5a9effd640 1 -- 172.21.15.32:0/3404024318 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7f5ab0029580 con 0x555ade795620 2024-04-25T00:34:26.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.489+0000 7f5ac8681740 10 client.25408 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:34:26.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.489+0000 7f5ac8681740 10 client.25408 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:34:26.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.489+0000 7f5ac8681740 20 client.25408 populate_metadata read hostname 'smithi032' 2024-04-25T00:34:26.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.489+0000 7f5ac8681740 1 --2- 172.21.15.32:0/3404024318 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x555ade80c360 0x555ade80e750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.489+0000 7f5ac8681740 4 client.25408 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-25T00:34:26.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.489+0000 7f5ac8681740 1 -- 172.21.15.32:0/3404024318 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "scrub status"}) -- 0x555ade6f6f40 con 0x555ade80c360 2024-04-25T00:34:26.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.489+0000 7f5abe7fc640 1 --2- 172.21.15.32:0/3404024318 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x555ade80c360 0x555ade80e750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:26.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.489+0000 7f5abe7fc640 1 --2- 172.21.15.32:0/3404024318 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x555ade80c360 0x555ade80e750 crc :-1 s=READY pgs=54 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ed842d96cb8d386b server_cookie=310c5c0180a8128b in_seq=0 out_seq=0 2024-04-25T00:34:26.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.489+0000 7f5a9effd640 10 client.25408 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:34:26.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.490+0000 7f5a9effd640 1 -- 172.21.15.32:0/3404024318 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+63 (crc 0 0 0) 0x555ade6f6f40 con 0x555ade80c360 2024-04-25T00:34:26.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.490+0000 7f5a9effd640 10 client.25408 handle_command_reply: tid=0 2024-04-25T00:34:26.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.490+0000 7f5ac8681740 1 client.25408 shutdown 2024-04-25T00:34:26.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.490+0000 7f5ac8681740 1 -- 172.21.15.32:0/3404024318 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555ade795620 msgr2=0x555ade79ddb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:26.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.490+0000 7f5ac8681740 1 --2- 172.21.15.32:0/3404024318 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555ade795620 0x555ade79ddb0 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x555ade7a4580 tx=0x7f5ab00058d0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.490+0000 7f5ac8681740 1 -- 172.21.15.32:0/3404024318 shutdown_connections 2024-04-25T00:34:26.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.490+0000 7f5ac8681740 1 --2- 172.21.15.32:0/3404024318 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x555ade80c360 0x555ade80e750 crc :-1 s=READY pgs=54 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.490+0000 7f5a9effd640 0 client.25408 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:34:26.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.490+0000 7f5ac8681740 1 --2- 172.21.15.32:0/3404024318 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555ade78e730 0x555ade78ebb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.495 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.490+0000 7f5ac8681740 1 --2- 172.21.15.32:0/3404024318 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555ade795620 0x555ade79ddb0 unknown :-1 s=CLOSED pgs=65 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.495 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.490+0000 7f5ac8681740 1 --2- 172.21.15.32:0/3404024318 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555ade78a630 0x555ade79d870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.495 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.490+0000 7f5ac8681740 1 -- 172.21.15.32:0/3404024318 >> 172.21.15.32:0/3404024318 conn(0x555ade786950 msgr2=0x555ade785af0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:26.495 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.490+0000 7f5ac8681740 1 -- 172.21.15.32:0/3404024318 shutdown_connections 2024-04-25T00:34:26.495 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.490+0000 7f5ac8681740 1 -- 172.21.15.32:0/3404024318 wait complete. 2024-04-25T00:34:26.495 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.490+0000 7f5ac8681740 20 client.25408 trim_cache size 0 max 16384 2024-04-25T00:34:26.495 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:34:26.495 INFO:teuthology.orchestra.run.smithi032.stdout: "status": "no active scrubs running", 2024-04-25T00:34:26.495 INFO:teuthology.orchestra.run.smithi032.stdout: "scrubs": {} 2024-04-25T00:34:26.495 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:34:26.496 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.491+0000 7f5ac5970640 1 -- 172.21.15.32:0/2326115253 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f5aa80a2e20 msgr2=0x7f5aa80a52e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:26.496 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.491+0000 7f5ac5970640 1 --2- 172.21.15.32:0/2326115253 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f5aa80a2e20 0x7f5aa80a52e0 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f5ab80edad0 tx=0x7f5ab802a950 comp rx=0 tx=0).stop 2024-04-25T00:34:26.496 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.491+0000 7f5ac5970640 1 -- 172.21.15.32:0/2326115253 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5ac010dfc0 msgr2=0x7f5ac019eb00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:26.496 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.491+0000 7f5ac5970640 1 --2- 172.21.15.32:0/2326115253 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5ac010dfc0 0x7f5ac019eb00 secure :-1 s=READY pgs=556 cs=0 l=1 rev1=1 crypto rx=0x7f5ab8005e30 tx=0x7f5ab802f630 comp rx=0 tx=0).stop 2024-04-25T00:34:26.496 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.491+0000 7f5ac5970640 1 -- 172.21.15.32:0/2326115253 shutdown_connections 2024-04-25T00:34:26.496 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.491+0000 7f5ac5970640 1 --2- 172.21.15.32:0/2326115253 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f5aa80a2e20 0x7f5aa80a52e0 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.496 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.491+0000 7f5ac5970640 1 --2- 172.21.15.32:0/2326115253 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5ac010dfc0 0x7f5ac019eb00 unknown :-1 s=CLOSED pgs=556 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.497 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.491+0000 7f5ac5970640 1 --2- 172.21.15.32:0/2326115253 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5ac0074430 0x7f5ac019a780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.497 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.491+0000 7f5ac5970640 1 --2- 172.21.15.32:0/2326115253 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ac0073b10 0x7f5ac019a240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.497 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.491+0000 7f5ac5970640 1 -- 172.21.15.32:0/2326115253 >> 172.21.15.32:0/2326115253 conn(0x7f5ac0078b40 msgr2=0x7f5ac00ffb10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:26.497 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.491+0000 7f5ac5970640 1 -- 172.21.15.32:0/2326115253 shutdown_connections 2024-04-25T00:34:26.497 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.491+0000 7f5ac5970640 1 -- 172.21.15.32:0/2326115253 wait complete. 2024-04-25T00:34:26.503 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-25T00:34:26.503 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:34:26.610 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.609+0000 7f8c18412640 1 -- 172.21.15.32:0/804244042 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8c1010bd10 msgr2=0x7f8c1010e1b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:26.610 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.609+0000 7f8c18412640 1 --2- 172.21.15.32:0/804244042 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8c1010bd10 0x7f8c1010e1b0 secure :-1 s=READY pgs=559 cs=0 l=1 rev1=1 crypto rx=0x7f8c08009ca0 tx=0x7f8c0802a9e0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.610 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.609+0000 7f8c18412640 1 -- 172.21.15.32:0/804244042 shutdown_connections 2024-04-25T00:34:26.610 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.609+0000 7f8c18412640 1 --2- 172.21.15.32:0/804244042 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8c1010bd10 0x7f8c1010e1b0 unknown :-1 s=CLOSED pgs=559 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.610 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.609+0000 7f8c18412640 1 --2- 172.21.15.32:0/804244042 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8c101032f0 0x7f8c1010b7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.610 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.609+0000 7f8c18412640 1 --2- 172.21.15.32:0/804244042 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8c101029d0 0x7f8c10102db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.611 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.609+0000 7f8c18412640 1 -- 172.21.15.32:0/804244042 >> 172.21.15.32:0/804244042 conn(0x7f8c10077740 msgr2=0x7f8c10077b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:26.611 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.609+0000 7f8c18412640 1 -- 172.21.15.32:0/804244042 shutdown_connections 2024-04-25T00:34:26.611 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.609+0000 7f8c18412640 1 -- 172.21.15.32:0/804244042 wait complete. 2024-04-25T00:34:26.613 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.610+0000 7f8c18412640 1 Processor -- start 2024-04-25T00:34:26.613 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.610+0000 7f8c18412640 1 -- start start 2024-04-25T00:34:26.613 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.610+0000 7f8c18412640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8c101029d0 0x7f8c10197da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.613 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.610+0000 7f8c18412640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8c101032f0 0x7f8c101982e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.613 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.610+0000 7f8c18412640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8c1010bd10 0x7f8c1019c660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.613 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.610+0000 7f8c18412640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f8c10111b60 con 0x7f8c101032f0 2024-04-25T00:34:26.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.610+0000 7f8c18412640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f8c101119e0 con 0x7f8c1010bd10 2024-04-25T00:34:26.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.610+0000 7f8c18412640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f8c10111ce0 con 0x7f8c101029d0 2024-04-25T00:34:26.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.611+0000 7f8c159ad640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8c101032f0 0x7f8c101982e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:26.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.611+0000 7f8c159ad640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8c101032f0 0x7f8c101982e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39800/0 (socket says 172.21.15.32:39800) 2024-04-25T00:34:26.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.611+0000 7f8c159ad640 1 -- 172.21.15.32:0/3700630824 learned_addr learned my addr 172.21.15.32:0/3700630824 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:26.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.611+0000 7f8c161ae640 1 --2- 172.21.15.32:0/3700630824 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8c101029d0 0x7f8c10197da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:26.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.611+0000 7f8c169af640 1 --2- 172.21.15.32:0/3700630824 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8c1010bd10 0x7f8c1019c660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:26.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.611+0000 7f8c159ad640 1 -- 172.21.15.32:0/3700630824 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8c101029d0 msgr2=0x7f8c10197da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:26.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.611+0000 7f8c159ad640 1 --2- 172.21.15.32:0/3700630824 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8c101029d0 0x7f8c10197da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.611+0000 7f8c159ad640 1 -- 172.21.15.32:0/3700630824 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8c1010bd10 msgr2=0x7f8c1019c660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:26.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.611+0000 7f8c159ad640 1 --2- 172.21.15.32:0/3700630824 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8c1010bd10 0x7f8c1019c660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.611+0000 7f8c159ad640 1 -- 172.21.15.32:0/3700630824 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8c1019cd90 con 0x7f8c101032f0 2024-04-25T00:34:26.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.611+0000 7f8c169af640 1 --2- 172.21.15.32:0/3700630824 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8c1010bd10 0x7f8c1019c660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:34:26.615 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.611+0000 7f8c159ad640 1 --2- 172.21.15.32:0/3700630824 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8c101032f0 0x7f8c101982e0 secure :-1 s=READY pgs=560 cs=0 l=1 rev1=1 crypto rx=0x7f8c0000e9f0 tx=0x7f8c0000edf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:26.615 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.611+0000 7f8c077fe640 1 -- 172.21.15.32:0/3700630824 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8c0000cd80 con 0x7f8c101032f0 2024-04-25T00:34:26.615 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.611+0000 7f8c18412640 1 -- 172.21.15.32:0/3700630824 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8c1019d080 con 0x7f8c101032f0 2024-04-25T00:34:26.615 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.611+0000 7f8c077fe640 1 -- 172.21.15.32:0/3700630824 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8c00005500 con 0x7f8c101032f0 2024-04-25T00:34:26.615 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.611+0000 7f8c077fe640 1 -- 172.21.15.32:0/3700630824 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8c00010690 con 0x7f8c101032f0 2024-04-25T00:34:26.615 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.611+0000 7f8c18412640 1 -- 172.21.15.32:0/3700630824 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8c101a4910 con 0x7f8c101032f0 2024-04-25T00:34:26.615 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.613+0000 7f8c18412640 1 -- 172.21.15.32:0/3700630824 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8c10068560 con 0x7f8c101032f0 2024-04-25T00:34:26.616 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.614+0000 7f8c077fe640 1 -- 172.21.15.32:0/3700630824 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 25) ==== 138066+0+0 (secure 0 0 0) 0x7f8c000046c0 con 0x7f8c101032f0 2024-04-25T00:34:26.616 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.615+0000 7f8c077fe640 1 --2- 172.21.15.32:0/3700630824 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f8bfc0a2ea0 0x7f8bfc0a5360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.616 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.615+0000 7f8c161ae640 1 --2- 172.21.15.32:0/3700630824 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f8bfc0a2ea0 0x7f8bfc0a5360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:26.616 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.615+0000 7f8c077fe640 1 -- 172.21.15.32:0/3700630824 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f8c000cfe50 con 0x7f8c101032f0 2024-04-25T00:34:26.616 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.616+0000 7f8c161ae640 1 --2- 172.21.15.32:0/3700630824 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f8bfc0a2ea0 0x7f8bfc0a5360 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f8c0c00bbe0 tx=0x7f8c0c00b390 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:26.623 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.623+0000 7f8c077fe640 1 -- 172.21.15.32:0/3700630824 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f8c0009c470 con 0x7f8c101032f0 2024-04-25T00:34:26.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.871+0000 7f8c18412640 1 -- 172.21.15.32:0/3700630824 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f8c101a4c70 con 0x7f8c101032f0 2024-04-25T00:34:26.872 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.872+0000 7f8c077fe640 1 -- 172.21.15.32:0/3700630824 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 25 v25) ==== 94+0+11876 (secure 0 0 0) 0x7f8c0009bd70 con 0x7f8c101032f0 2024-04-25T00:34:26.872 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:34:26.873 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":25,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14643,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6839/187492674","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6838","nonce":187492674},{"type":"v1","addr":"172.21.15.32:6839","nonce":187492674}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14655,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6841/1948785545","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6840","nonce":1948785545},{"type":"v1","addr":"172.21.15.32:6841","nonce":1948785545}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":25,"flags":306,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":true},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-25T00:30:47.866875+0000","modified":"2024-04-25T00:33:16.420908+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24506,"mds_1":24481,"mds_2":24476},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24443":{"gid":24443,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24445":{"gid":24445,"name":"f","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24457":{"gid":24457,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24476":{"gid":24476,"name":"h","rank":2,"incarnation":22,"state":"up:active","state_seq":12,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24481":{"gid":24481,"name":"i","rank":1,"incarnation":20,"state":"up:active","state_seq":11,"addr":"172.21.15.130:6839/4267125809","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":4267125809},{"type":"v1","addr":"172.21.15.130:6839","nonce":4267125809}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24506":{"gid":24506,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.121:6839/258444853","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6838","nonce":258444853},{"type":"v1","addr":"172.21.15.121:6839","nonce":258444853}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24506,"qdb_cluster":[24506,24476,24481]},"id":1}]} 2024-04-25T00:34:26.873 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 25 2024-04-25T00:34:26.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.874+0000 7f8c18412640 1 -- 172.21.15.32:0/3700630824 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f8bfc0a2ea0 msgr2=0x7f8bfc0a5360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:26.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.874+0000 7f8c18412640 1 --2- 172.21.15.32:0/3700630824 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f8bfc0a2ea0 0x7f8bfc0a5360 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f8c0c00bbe0 tx=0x7f8c0c00b390 comp rx=0 tx=0).stop 2024-04-25T00:34:26.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.874+0000 7f8c18412640 1 -- 172.21.15.32:0/3700630824 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8c101032f0 msgr2=0x7f8c101982e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:26.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.874+0000 7f8c18412640 1 --2- 172.21.15.32:0/3700630824 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8c101032f0 0x7f8c101982e0 secure :-1 s=READY pgs=560 cs=0 l=1 rev1=1 crypto rx=0x7f8c0000e9f0 tx=0x7f8c0000edf0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.874+0000 7f8c18412640 1 -- 172.21.15.32:0/3700630824 shutdown_connections 2024-04-25T00:34:26.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.874+0000 7f8c18412640 1 --2- 172.21.15.32:0/3700630824 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f8bfc0a2ea0 0x7f8bfc0a5360 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.874+0000 7f8c18412640 1 --2- 172.21.15.32:0/3700630824 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8c1010bd10 0x7f8c1019c660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.874+0000 7f8c18412640 1 --2- 172.21.15.32:0/3700630824 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8c101032f0 0x7f8c101982e0 unknown :-1 s=CLOSED pgs=560 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.874+0000 7f8c18412640 1 --2- 172.21.15.32:0/3700630824 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8c101029d0 0x7f8c10197da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.874+0000 7f8c18412640 1 -- 172.21.15.32:0/3700630824 >> 172.21.15.32:0/3700630824 conn(0x7f8c10077740 msgr2=0x7f8c100fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:26.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.874+0000 7f8c18412640 1 -- 172.21.15.32:0/3700630824 shutdown_connections 2024-04-25T00:34:26.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.874+0000 7f8c18412640 1 -- 172.21.15.32:0/3700630824 wait complete. 2024-04-25T00:34:26.887 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-25T00:34:26.995 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.992+0000 7f0c00fc1640 1 -- 172.21.15.32:0/1653241441 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0bfc1032e0 msgr2=0x7f0bfc10b7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:26.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.992+0000 7f0c00fc1640 1 --2- 172.21.15.32:0/1653241441 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0bfc1032e0 0x7f0bfc10b7c0 secure :-1 s=READY pgs=561 cs=0 l=1 rev1=1 crypto rx=0x7f0bf0009900 tx=0x7f0bf002a510 comp rx=0 tx=0).stop 2024-04-25T00:34:26.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.993+0000 7f0c00fc1640 1 -- 172.21.15.32:0/1653241441 shutdown_connections 2024-04-25T00:34:26.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.993+0000 7f0c00fc1640 1 --2- 172.21.15.32:0/1653241441 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0bfc10bd00 0x7f0bfc10e1a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.993+0000 7f0c00fc1640 1 --2- 172.21.15.32:0/1653241441 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0bfc1032e0 0x7f0bfc10b7c0 unknown :-1 s=CLOSED pgs=561 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.993+0000 7f0c00fc1640 1 --2- 172.21.15.32:0/1653241441 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0bfc1029c0 0x7f0bfc102da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.993+0000 7f0c00fc1640 1 -- 172.21.15.32:0/1653241441 >> 172.21.15.32:0/1653241441 conn(0x7f0bfc077740 msgr2=0x7f0bfc077b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:26.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.993+0000 7f0c00fc1640 1 -- 172.21.15.32:0/1653241441 shutdown_connections 2024-04-25T00:34:26.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.993+0000 7f0c00fc1640 1 -- 172.21.15.32:0/1653241441 wait complete. 2024-04-25T00:34:26.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.993+0000 7f0c00fc1640 1 Processor -- start 2024-04-25T00:34:26.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.993+0000 7f0c00fc1640 1 -- start start 2024-04-25T00:34:26.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.994+0000 7f0c00fc1640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0bfc1029c0 0x7f0bfc198040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.994+0000 7f0c00fc1640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0bfc1032e0 0x7f0bfc198580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.997 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.994+0000 7f0c00fc1640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0bfc10bd00 0x7f0bfc19c900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:26.997 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.994+0000 7f0c00fc1640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f0bfc111b00 con 0x7f0bfc1032e0 2024-04-25T00:34:26.997 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.994+0000 7f0c00fc1640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f0bfc111980 con 0x7f0bfc1029c0 2024-04-25T00:34:26.997 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.994+0000 7f0c00fc1640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f0bfc111c80 con 0x7f0bfc10bd00 2024-04-25T00:34:26.997 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.994+0000 7f0bf9d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0bfc1032e0 0x7f0bfc198580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:26.997 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.994+0000 7f0bf9d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0bfc1032e0 0x7f0bfc198580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39818/0 (socket says 172.21.15.32:39818) 2024-04-25T00:34:26.997 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.994+0000 7f0bf9d9b640 1 -- 172.21.15.32:0/4228849238 learned_addr learned my addr 172.21.15.32:0/4228849238 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:26.997 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.994+0000 7f0bfa59c640 1 --2- 172.21.15.32:0/4228849238 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0bfc1029c0 0x7f0bfc198040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:26.997 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.994+0000 7f0bfad9d640 1 --2- 172.21.15.32:0/4228849238 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0bfc10bd00 0x7f0bfc19c900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:26.997 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.994+0000 7f0bf9d9b640 1 -- 172.21.15.32:0/4228849238 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0bfc10bd00 msgr2=0x7f0bfc19c900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:26.997 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.994+0000 7f0bf9d9b640 1 --2- 172.21.15.32:0/4228849238 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0bfc10bd00 0x7f0bfc19c900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.997 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.994+0000 7f0bf9d9b640 1 -- 172.21.15.32:0/4228849238 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0bfc1029c0 msgr2=0x7f0bfc198040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:26.997 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.994+0000 7f0bf9d9b640 1 --2- 172.21.15.32:0/4228849238 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0bfc1029c0 0x7f0bfc198040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:26.998 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.994+0000 7f0bf9d9b640 1 -- 172.21.15.32:0/4228849238 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0bfc19d030 con 0x7f0bfc1032e0 2024-04-25T00:34:26.998 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.994+0000 7f0bfa59c640 1 --2- 172.21.15.32:0/4228849238 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0bfc1029c0 0x7f0bfc198040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:34:26.998 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.994+0000 7f0bfad9d640 1 --2- 172.21.15.32:0/4228849238 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0bfc10bd00 0x7f0bfc19c900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:34:26.998 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.995+0000 7f0bf9d9b640 1 --2- 172.21.15.32:0/4228849238 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0bfc1032e0 0x7f0bfc198580 secure :-1 s=READY pgs=562 cs=0 l=1 rev1=1 crypto rx=0x7f0bf0006010 tx=0x7f0bf00042f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:26.998 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.995+0000 7f0beb7fe640 1 -- 172.21.15.32:0/4228849238 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0bf003f030 con 0x7f0bfc1032e0 2024-04-25T00:34:26.998 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.995+0000 7f0c00fc1640 1 -- 172.21.15.32:0/4228849238 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0bfc19d2c0 con 0x7f0bfc1032e0 2024-04-25T00:34:26.998 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.995+0000 7f0beb7fe640 1 -- 172.21.15.32:0/4228849238 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0bf0003ce0 con 0x7f0bfc1032e0 2024-04-25T00:34:26.998 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.995+0000 7f0beb7fe640 1 -- 172.21.15.32:0/4228849238 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0bf0035070 con 0x7f0bfc1032e0 2024-04-25T00:34:26.998 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.995+0000 7f0c00fc1640 1 -- 172.21.15.32:0/4228849238 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0bfc100540 con 0x7f0bfc1032e0 2024-04-25T00:34:27.000 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.998+0000 7f0c03dac740 1 Processor -- start 2024-04-25T00:34:27.000 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.998+0000 7f0c03dac740 1 -- start start 2024-04-25T00:34:27.000 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.998+0000 7f0beb7fe640 1 -- 172.21.15.32:0/4228849238 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 25) ==== 138066+0+0 (secure 0 0 0) 0x7f0bf0042030 con 0x7f0bfc1032e0 2024-04-25T00:34:27.001 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.998+0000 7f0c03dac740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b91580 0x55f287b91960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.001 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.998+0000 7f0c03dac740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f287b91ea0 0x55f287b9c030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.001 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.999+0000 7f0c03dac740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55f287b9c570 0x55f287b9e960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.001 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.999+0000 7f0c03dac740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55f287af6670 con 0x55f287b91580 2024-04-25T00:34:27.001 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.999+0000 7f0c03dac740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55f287a9e970 con 0x55f287b9c570 2024-04-25T00:34:27.001 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.999+0000 7f0c03dac740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55f287af2630 con 0x55f287b91ea0 2024-04-25T00:34:27.001 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.999+0000 7f0bfad9d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b91580 0x55f287b91960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.001 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.999+0000 7f0bfad9d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b91580 0x55f287b91960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39830/0 (socket says 172.21.15.32:39830) 2024-04-25T00:34:27.001 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.999+0000 7f0bfad9d640 1 -- 172.21.15.32:0/1124839225 learned_addr learned my addr 172.21.15.32:0/1124839225 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:27.001 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.999+0000 7f0bfa59c640 1 --2- 172.21.15.32:0/1124839225 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f287b91ea0 0x55f287b9c030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.001 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.999+0000 7f0bfad9d640 1 -- 172.21.15.32:0/1124839225 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f287b91ea0 msgr2=0x55f287b9c030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:27.001 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.999+0000 7f0bfad9d640 1 --2- 172.21.15.32:0/1124839225 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f287b91ea0 0x55f287b9c030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.001 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.999+0000 7f0bfad9d640 1 -- 172.21.15.32:0/1124839225 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55f287b9c570 msgr2=0x55f287b9e960 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-25T00:34:27.001 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.999+0000 7f0bfad9d640 1 --2- 172.21.15.32:0/1124839225 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55f287b9c570 0x55f287b9e960 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.002 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.999+0000 7f0bfad9d640 1 -- 172.21.15.32:0/1124839225 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55f287ac2400 con 0x55f287b91580 2024-04-25T00:34:27.002 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:26.999+0000 7f0beb7fe640 1 --2- 172.21.15.32:0/4228849238 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f0be40a2ea0 0x7f0be40a5360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.002 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.000+0000 7f0bfa59c640 1 --2- 172.21.15.32:0/4228849238 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f0be40a2ea0 0x7f0be40a5360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.002 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.000+0000 7f0beb7fe640 1 -- 172.21.15.32:0/4228849238 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f0bf00eff80 con 0x7f0bfc1032e0 2024-04-25T00:34:27.002 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.000+0000 7f0bfad9d640 1 --2- 172.21.15.32:0/1124839225 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b91580 0x55f287b91960 secure :-1 s=READY pgs=563 cs=0 l=1 rev1=1 crypto rx=0x7f0bf4009f30 tx=0x7f0bf4010800 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e6a04ae830ae1dc3 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:27.002 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.000+0000 7f0bfa59c640 1 --2- 172.21.15.32:0/4228849238 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f0be40a2ea0 0x7f0be40a5360 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x55f287b92b40 tx=0x7f0bec00b390 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:27.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.000+0000 7f0be8ff9640 1 -- 172.21.15.32:0/1124839225 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0bf4016070 con 0x55f287b91580 2024-04-25T00:34:27.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.000+0000 7f0be8ff9640 1 -- 172.21.15.32:0/1124839225 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0bf401bb40 con 0x55f287b91580 2024-04-25T00:34:27.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.000+0000 7f0be8ff9640 1 -- 172.21.15.32:0/1124839225 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0bf4012d20 con 0x55f287b91580 2024-04-25T00:34:27.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.001+0000 7f0c03dac740 1 -- 172.21.15.32:0/1124839225 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b91580 msgr2=0x55f287b91960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:27.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.001+0000 7f0c03dac740 1 --2- 172.21.15.32:0/1124839225 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b91580 0x55f287b91960 secure :-1 s=READY pgs=563 cs=0 l=1 rev1=1 crypto rx=0x7f0bf4009f30 tx=0x7f0bf4010800 comp rx=0 tx=0).stop 2024-04-25T00:34:27.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.001+0000 7f0c03dac740 1 -- 172.21.15.32:0/1124839225 shutdown_connections 2024-04-25T00:34:27.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.001+0000 7f0c03dac740 1 --2- 172.21.15.32:0/1124839225 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55f287b9c570 0x55f287b9e960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.001+0000 7f0c03dac740 1 --2- 172.21.15.32:0/1124839225 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f287b91ea0 0x55f287b9c030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.001+0000 7f0c03dac740 1 --2- 172.21.15.32:0/1124839225 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b91580 0x55f287b91960 unknown :-1 s=CLOSED pgs=563 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.001+0000 7f0c03dac740 1 -- 172.21.15.32:0/1124839225 >> 172.21.15.32:0/1124839225 conn(0x55f287b8d8a0 msgr2=0x55f287b8dc80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:27.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.001+0000 7f0c03dac740 1 -- 172.21.15.32:0/1124839225 shutdown_connections 2024-04-25T00:34:27.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.001+0000 7f0c03dac740 1 -- 172.21.15.32:0/1124839225 wait complete. 2024-04-25T00:34:27.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.002+0000 7f0c03dac740 1 -- start start 2024-04-25T00:34:27.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.002+0000 7f0c03dac740 1 Processor -- start 2024-04-25T00:34:27.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.002+0000 7f0c03dac740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b91580 0x55f287b9bee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.002+0000 7f0c03dac740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f287b91ea0 0x55f287b991d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.002+0000 7f0c03dac740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55f287b9c570 0x55f287b99730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.002+0000 7f0c03dac740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55f287b9f360 con 0x55f287b91580 2024-04-25T00:34:27.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.002+0000 7f0c03dac740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55f287ab2050 con 0x55f287b9c570 2024-04-25T00:34:27.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.002+0000 7f0c03dac740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55f287b9f4e0 con 0x55f287b91ea0 2024-04-25T00:34:27.007 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.002+0000 7f0c03dac740 10 client.0 _ll_register_callbacks cb 0x55f287809be0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:34:27.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.003+0000 7f0bfa59c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b91580 0x55f287b9bee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.003+0000 7f0bfa59c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b91580 0x55f287b9bee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39846/0 (socket says 172.21.15.32:39846) 2024-04-25T00:34:27.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.003+0000 7f0bfa59c640 1 -- 172.21.15.32:0/1577325072 learned_addr learned my addr 172.21.15.32:0/1577325072 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:27.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.003+0000 7f0bf9d9b640 1 --2- 172.21.15.32:0/1577325072 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f287b91ea0 0x55f287b991d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.003+0000 7f0bfad9d640 1 --2- 172.21.15.32:0/1577325072 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55f287b9c570 0x55f287b99730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.003+0000 7f0bf9d9b640 1 -- 172.21.15.32:0/1577325072 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55f287b9c570 msgr2=0x55f287b99730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:27.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.003+0000 7f0bf9d9b640 1 --2- 172.21.15.32:0/1577325072 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55f287b9c570 0x55f287b99730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.003+0000 7f0bf9d9b640 1 -- 172.21.15.32:0/1577325072 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b91580 msgr2=0x55f287b9bee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:27.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.003+0000 7f0bf9d9b640 1 --2- 172.21.15.32:0/1577325072 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b91580 0x55f287b9bee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.003+0000 7f0bf9d9b640 1 -- 172.21.15.32:0/1577325072 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55f287a83dc0 con 0x55f287b91ea0 2024-04-25T00:34:27.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.003+0000 7f0bfa59c640 1 --2- 172.21.15.32:0/1577325072 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b91580 0x55f287b9bee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:34:27.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.003+0000 7f0bf9d9b640 1 --2- 172.21.15.32:0/1577325072 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f287b91ea0 0x55f287b991d0 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f0bf00ed1a0 tx=0x7f0bf00b82d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4194b4960d663f3a server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:27.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.003+0000 7f0be1ffb640 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:34:27.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.004+0000 7f0be1ffb640 1 -- 172.21.15.32:0/1577325072 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0bf0049040 con 0x55f287b91ea0 2024-04-25T00:34:27.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.004+0000 7f0be1ffb640 1 -- 172.21.15.32:0/1577325072 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f0bf00c4a10 con 0x55f287b91ea0 2024-04-25T00:34:27.009 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.004+0000 7f0c03dac740 1 -- 172.21.15.32:0/1577325072 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f0bf401bb40 con 0x55f287b91ea0 2024-04-25T00:34:27.009 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.004+0000 7f0be1ffb640 1 -- 172.21.15.32:0/1577325072 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f0bf00bb990 con 0x55f287b91ea0 2024-04-25T00:34:27.009 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.004+0000 7f0c03dac740 10 client.25426 fetch_fsmap learned FSMap version 25 2024-04-25T00:34:27.009 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.004+0000 7f0c03dac740 1 -- 172.21.15.32:0/1577325072 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x55f28780a3f0 con 0x55f287b91ea0 2024-04-25T00:34:27.009 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.004+0000 7f0be1ffb640 1 -- 172.21.15.32:0/1577325072 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0bf401bb40 con 0x55f287b91ea0 2024-04-25T00:34:27.009 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.004+0000 7f0be1ffb640 1 -- 172.21.15.32:0/1577325072 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7f0bf00dabb0 con 0x55f287b91ea0 2024-04-25T00:34:27.009 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.005+0000 7f0c03dac740 10 client.25426 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:34:27.009 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.005+0000 7f0c03dac740 10 client.25426 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-25T00:34:27.009 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.005+0000 7f0c03dac740 20 client.25426 populate_metadata read hostname 'smithi032' 2024-04-25T00:34:27.009 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.005+0000 7f0c03dac740 1 --2- 172.21.15.32:0/1577325072 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x55f287bac5b0 0x55f287bae9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.009 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.005+0000 7f0c03dac740 4 client.25426 mds_command: new command op to 24476 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:34:27.009 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.005+0000 7f0c03dac740 1 -- 172.21.15.32:0/1577325072 --> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55f287b03bb0 con 0x55f287bac5b0 2024-04-25T00:34:27.009 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.005+0000 7f0bfa59c640 1 --2- 172.21.15.32:0/1577325072 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x55f287bac5b0 0x55f287bae9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.010 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.006+0000 7f0be1ffb640 10 client.25426 ms_handle_connect on v2:172.21.15.121:6836/3456022860 2024-04-25T00:34:27.010 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.006+0000 7f0bfa59c640 1 --2- 172.21.15.32:0/1577325072 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x55f287bac5b0 0x55f287bae9a0 crc :-1 s=READY pgs=36 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=5a46036bebfea46a server_cookie=cc32b62b79e44d05 in_seq=0 out_seq=0 2024-04-25T00:34:27.011 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.006+0000 7f0be1ffb640 1 -- 172.21.15.32:0/1577325072 <== mds.2 v2:172.21.15.121:6836/3456022860 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x55f287b03bb0 con 0x55f287bac5b0 2024-04-25T00:34:27.011 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.006+0000 7f0be1ffb640 10 client.25426 handle_command_reply: tid=0 2024-04-25T00:34:27.011 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.006+0000 7f0c03dac740 1 client.25426 shutdown 2024-04-25T00:34:27.011 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.007+0000 7f0c03dac740 1 -- 172.21.15.32:0/1577325072 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f287b91ea0 msgr2=0x55f287b991d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:27.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.007+0000 7f0c03dac740 1 --2- 172.21.15.32:0/1577325072 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f287b91ea0 0x55f287b991d0 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f0bf00ed1a0 tx=0x7f0bf00b82d0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.007+0000 7f0c03dac740 1 -- 172.21.15.32:0/1577325072 shutdown_connections 2024-04-25T00:34:27.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.007+0000 7f0c03dac740 1 --2- 172.21.15.32:0/1577325072 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x55f287bac5b0 0x55f287bae9a0 crc :-1 s=READY pgs=36 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.007+0000 7f0c03dac740 1 --2- 172.21.15.32:0/1577325072 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55f287b9c570 0x55f287b99730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.007+0000 7f0c03dac740 1 --2- 172.21.15.32:0/1577325072 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f287b91ea0 0x55f287b991d0 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.007+0000 7f0c03dac740 1 --2- 172.21.15.32:0/1577325072 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b91580 0x55f287b9bee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.007+0000 7f0c03dac740 1 -- 172.21.15.32:0/1577325072 >> 172.21.15.32:0/1577325072 conn(0x55f287b8d8a0 msgr2=0x55f287b936a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:27.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.007+0000 7f0be1ffb640 0 client.25426 ms_handle_reset on v2:172.21.15.121:6836/3456022860 2024-04-25T00:34:27.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.007+0000 7f0c03dac740 1 -- 172.21.15.32:0/1577325072 shutdown_connections 2024-04-25T00:34:27.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.007+0000 7f0c03dac740 1 -- 172.21.15.32:0/1577325072 wait complete. 2024-04-25T00:34:27.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.007+0000 7f0c03dac740 20 client.25426 trim_cache size 0 max 16384 2024-04-25T00:34:27.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.021+0000 7f0c03dac740 1 Processor -- start 2024-04-25T00:34:27.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.021+0000 7f0c03dac740 1 -- start start 2024-04-25T00:34:27.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.021+0000 7f0c03dac740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55f287b91580 0x55f287ba8220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.021+0000 7f0c03dac740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b91ea0 0x55f287b9ba10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.021+0000 7f0c03dac740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f287b9c570 0x55f287bb4230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.021+0000 7f0c03dac740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f0bfc1b3630 con 0x55f287b91ea0 2024-04-25T00:34:27.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.021+0000 7f0c03dac740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f0bfc1b0f00 con 0x55f287b91580 2024-04-25T00:34:27.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.021+0000 7f0c03dac740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f0bfc1aed50 con 0x55f287b9c570 2024-04-25T00:34:27.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.021+0000 7f0bf9d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b91ea0 0x55f287b9ba10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.021+0000 7f0bf9d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b91ea0 0x55f287b9ba10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39850/0 (socket says 172.21.15.32:39850) 2024-04-25T00:34:27.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.021+0000 7f0bf9d9b640 1 -- 172.21.15.32:0/1982568597 learned_addr learned my addr 172.21.15.32:0/1982568597 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:27.024 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.021+0000 7f0bfa59c640 1 --2- 172.21.15.32:0/1982568597 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55f287b91580 0x55f287ba8220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.024 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.022+0000 7f0bf9d9b640 1 -- 172.21.15.32:0/1982568597 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f287b9c570 msgr2=0x55f287bb4230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:27.024 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.022+0000 7f0bf9d9b640 1 --2- 172.21.15.32:0/1982568597 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f287b9c570 0x55f287bb4230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.024 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.022+0000 7f0bf9d9b640 1 -- 172.21.15.32:0/1982568597 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55f287b91580 msgr2=0x55f287ba8220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:27.024 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.022+0000 7f0bf9d9b640 1 --2- 172.21.15.32:0/1982568597 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55f287b91580 0x55f287ba8220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.024 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.022+0000 7f0bf9d9b640 1 -- 172.21.15.32:0/1982568597 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55f2878d47d0 con 0x55f287b91ea0 2024-04-25T00:34:27.024 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.022+0000 7f0bf9d9b640 1 --2- 172.21.15.32:0/1982568597 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b91ea0 0x55f287b9ba10 secure :-1 s=READY pgs=564 cs=0 l=1 rev1=1 crypto rx=0x7f0bf00b8a00 tx=0x7f0bf00b8400 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c7e2914b37dabe33 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:27.024 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.022+0000 7f0be3fff640 1 -- 172.21.15.32:0/1982568597 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0bf00b6070 con 0x55f287b91ea0 2024-04-25T00:34:27.024 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.022+0000 7f0be3fff640 1 -- 172.21.15.32:0/1982568597 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0bf0003800 con 0x55f287b91ea0 2024-04-25T00:34:27.024 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.022+0000 7f0be3fff640 1 -- 172.21.15.32:0/1982568597 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0bf0049070 con 0x55f287b91ea0 2024-04-25T00:34:27.028 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.023+0000 7f0c03dac740 1 -- 172.21.15.32:0/1982568597 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b91ea0 msgr2=0x55f287b9ba10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:27.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.023+0000 7f0c03dac740 1 --2- 172.21.15.32:0/1982568597 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b91ea0 0x55f287b9ba10 secure :-1 s=READY pgs=564 cs=0 l=1 rev1=1 crypto rx=0x7f0bf00b8a00 tx=0x7f0bf00b8400 comp rx=0 tx=0).stop 2024-04-25T00:34:27.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.023+0000 7f0c03dac740 1 -- 172.21.15.32:0/1982568597 shutdown_connections 2024-04-25T00:34:27.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.023+0000 7f0c03dac740 1 --2- 172.21.15.32:0/1982568597 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f287b9c570 0x55f287bb4230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.024+0000 7f0c03dac740 1 --2- 172.21.15.32:0/1982568597 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b91ea0 0x55f287b9ba10 unknown :-1 s=CLOSED pgs=564 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.024+0000 7f0c03dac740 1 --2- 172.21.15.32:0/1982568597 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55f287b91580 0x55f287ba8220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.024+0000 7f0c03dac740 1 -- 172.21.15.32:0/1982568597 >> 172.21.15.32:0/1982568597 conn(0x55f287b8d8a0 msgr2=0x55f287b94860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:27.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.024+0000 7f0c03dac740 1 -- 172.21.15.32:0/1982568597 shutdown_connections 2024-04-25T00:34:27.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.024+0000 7f0c03dac740 1 -- 172.21.15.32:0/1982568597 wait complete. 2024-04-25T00:34:27.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.024+0000 7f0c03dac740 1 -- start start 2024-04-25T00:34:27.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.024+0000 7f0c03dac740 1 Processor -- start 2024-04-25T00:34:27.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.024+0000 7f0c03dac740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55f287b91580 0x55f287ba01a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.024+0000 7f0c03dac740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b9c570 0x55f287ba06e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.025+0000 7f0bf9d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b9c570 0x55f287ba06e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.025+0000 7f0bf9d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b9c570 0x55f287ba06e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39852/0 (socket says 172.21.15.32:39852) 2024-04-25T00:34:27.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.025+0000 7f0bf9d9b640 1 -- 172.21.15.32:0/4147790525 learned_addr learned my addr 172.21.15.32:0/4147790525 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:27.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.025+0000 7f0c03dac740 1 --2- 172.21.15.32:0/4147790525 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f287ba0c20 0x55f287c11aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.025+0000 7f0c03dac740 1 -- 172.21.15.32:0/4147790525 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f0bfc1af3c0 con 0x55f287b9c570 2024-04-25T00:34:27.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.025+0000 7f0c03dac740 1 -- 172.21.15.32:0/4147790525 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f0bfc1b0830 con 0x55f287b91580 2024-04-25T00:34:27.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.025+0000 7f0c03dac740 1 -- 172.21.15.32:0/4147790525 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f0bfc1afa90 con 0x55f287ba0c20 2024-04-25T00:34:27.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.025+0000 7f0c03dac740 10 client.0 _ll_register_callbacks cb 0x7f0bf003b460 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:34:27.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.025+0000 7f0bfa59c640 1 --2- 172.21.15.32:0/4147790525 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55f287b91580 0x55f287ba01a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.025+0000 7f0bfad9d640 1 --2- 172.21.15.32:0/4147790525 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f287ba0c20 0x55f287c11aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.025+0000 7f0bf9d9b640 1 -- 172.21.15.32:0/4147790525 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f287ba0c20 msgr2=0x55f287c11aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:27.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.025+0000 7f0bf9d9b640 1 --2- 172.21.15.32:0/4147790525 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f287ba0c20 0x55f287c11aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.025+0000 7f0bf9d9b640 1 -- 172.21.15.32:0/4147790525 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55f287b91580 msgr2=0x55f287ba01a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:27.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.025+0000 7f0bf9d9b640 1 --2- 172.21.15.32:0/4147790525 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55f287b91580 0x55f287ba01a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.031 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.025+0000 7f0bf9d9b640 1 -- 172.21.15.32:0/4147790525 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55f287c11fe0 con 0x55f287b9c570 2024-04-25T00:34:27.031 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.025+0000 7f0bfa59c640 1 --2- 172.21.15.32:0/4147790525 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55f287b91580 0x55f287ba01a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:34:27.031 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.025+0000 7f0bf9d9b640 1 --2- 172.21.15.32:0/4147790525 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b9c570 0x55f287ba06e0 secure :-1 s=READY pgs=565 cs=0 l=1 rev1=1 crypto rx=0x7f0bf0005ab0 tx=0x7f0bf00dade0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ec63ea8775e8e648 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:27.031 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.025+0000 7f0be1ffb640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:34:27.031 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.025+0000 7f0be1ffb640 1 -- 172.21.15.32:0/4147790525 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0bf00c8070 con 0x55f287b9c570 2024-04-25T00:34:27.031 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.026+0000 7f0be1ffb640 1 -- 172.21.15.32:0/4147790525 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f0bf00ed480 con 0x55f287b9c570 2024-04-25T00:34:27.031 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.026+0000 7f0c03dac740 1 -- 172.21.15.32:0/4147790525 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x55f287afdf00 con 0x55f287b9c570 2024-04-25T00:34:27.031 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.026+0000 7f0be1ffb640 1 -- 172.21.15.32:0/4147790525 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f0bf0049070 con 0x55f287b9c570 2024-04-25T00:34:27.031 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.026+0000 7f0c03dac740 10 client.15897 fetch_fsmap learned FSMap version 25 2024-04-25T00:34:27.031 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.026+0000 7f0c03dac740 1 -- 172.21.15.32:0/4147790525 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x55f287c11fe0 con 0x55f287b9c570 2024-04-25T00:34:27.031 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.026+0000 7f0be1ffb640 1 -- 172.21.15.32:0/4147790525 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0bf00c03e0 con 0x55f287b9c570 2024-04-25T00:34:27.031 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.026+0000 7f0be1ffb640 1 -- 172.21.15.32:0/4147790525 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7f0bf00cb020 con 0x55f287b9c570 2024-04-25T00:34:27.031 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.026+0000 7f0c03dac740 10 client.15897 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:34:27.031 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.026+0000 7f0c03dac740 10 client.15897 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-25T00:34:27.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.026+0000 7f0c03dac740 20 client.15897 populate_metadata read hostname 'smithi032' 2024-04-25T00:34:27.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.026+0000 7f0c03dac740 1 --2- 172.21.15.32:0/4147790525 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x55f287c134f0 0x55f287c158e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.026+0000 7f0c03dac740 4 client.15897 mds_command: new command op to 24476 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-25T00:34:27.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.026+0000 7f0c03dac740 1 -- 172.21.15.32:0/4147790525 --> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55f287afdf00 con 0x55f287c134f0 2024-04-25T00:34:27.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.027+0000 7f0bfa59c640 1 --2- 172.21.15.32:0/4147790525 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x55f287c134f0 0x55f287c158e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.027+0000 7f0be1ffb640 10 client.15897 ms_handle_connect on v2:172.21.15.121:6836/3456022860 2024-04-25T00:34:27.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.027+0000 7f0bfa59c640 1 --2- 172.21.15.32:0/4147790525 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x55f287c134f0 0x55f287c158e0 crc :-1 s=READY pgs=37 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=877edccc95aacf24 server_cookie=f05b91f9562ea74e in_seq=0 out_seq=0 2024-04-25T00:34:27.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.027+0000 7f0be1ffb640 1 -- 172.21.15.32:0/4147790525 <== mds.2 v2:172.21.15.121:6836/3456022860 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55f287afdf00 con 0x55f287c134f0 2024-04-25T00:34:27.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.027+0000 7f0be1ffb640 10 client.15897 handle_command_reply: tid=0 2024-04-25T00:34:27.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.027+0000 7f0c03dac740 1 client.15897 shutdown 2024-04-25T00:34:27.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.028+0000 7f0c03dac740 1 -- 172.21.15.32:0/4147790525 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b9c570 msgr2=0x55f287ba06e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:27.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.028+0000 7f0c03dac740 1 --2- 172.21.15.32:0/4147790525 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b9c570 0x55f287ba06e0 secure :-1 s=READY pgs=565 cs=0 l=1 rev1=1 crypto rx=0x7f0bf0005ab0 tx=0x7f0bf00dade0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.028+0000 7f0c03dac740 1 -- 172.21.15.32:0/4147790525 shutdown_connections 2024-04-25T00:34:27.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.028+0000 7f0c03dac740 1 --2- 172.21.15.32:0/4147790525 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x55f287c134f0 0x55f287c158e0 crc :-1 s=READY pgs=37 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.028+0000 7f0c03dac740 1 --2- 172.21.15.32:0/4147790525 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f287ba0c20 0x55f287c11aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.028+0000 7f0c03dac740 1 --2- 172.21.15.32:0/4147790525 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55f287b9c570 0x55f287ba06e0 unknown :-1 s=CLOSED pgs=565 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.028+0000 7f0c03dac740 1 --2- 172.21.15.32:0/4147790525 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55f287b91580 0x55f287ba01a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.028+0000 7f0c03dac740 1 -- 172.21.15.32:0/4147790525 >> 172.21.15.32:0/4147790525 conn(0x55f287b8d8a0 msgr2=0x55f287ba7210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:27.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.028+0000 7f0be1ffb640 0 client.15897 ms_handle_reset on v2:172.21.15.121:6836/3456022860 2024-04-25T00:34:27.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.028+0000 7f0c03dac740 1 -- 172.21.15.32:0/4147790525 shutdown_connections 2024-04-25T00:34:27.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.028+0000 7f0c03dac740 1 -- 172.21.15.32:0/4147790525 wait complete. 2024-04-25T00:34:27.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.028+0000 7f0c03dac740 20 client.15897 trim_cache size 0 max 16384 2024-04-25T00:34:27.033 INFO:teuthology.orchestra.run.smithi032.stdout:[] 2024-04-25T00:34:27.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.028+0000 7f0c00fc1640 1 -- 172.21.15.32:0/4228849238 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f0be40a2ea0 msgr2=0x7f0be40a5360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:27.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.028+0000 7f0c00fc1640 1 --2- 172.21.15.32:0/4228849238 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f0be40a2ea0 0x7f0be40a5360 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x55f287b92b40 tx=0x7f0bec00b390 comp rx=0 tx=0).stop 2024-04-25T00:34:27.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.028+0000 7f0c00fc1640 1 -- 172.21.15.32:0/4228849238 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0bfc1032e0 msgr2=0x7f0bfc198580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:27.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.028+0000 7f0c00fc1640 1 --2- 172.21.15.32:0/4228849238 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0bfc1032e0 0x7f0bfc198580 secure :-1 s=READY pgs=562 cs=0 l=1 rev1=1 crypto rx=0x7f0bf0006010 tx=0x7f0bf00042f0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.029+0000 7f0c00fc1640 1 -- 172.21.15.32:0/4228849238 shutdown_connections 2024-04-25T00:34:27.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.029+0000 7f0c00fc1640 1 --2- 172.21.15.32:0/4228849238 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f0be40a2ea0 0x7f0be40a5360 unknown :-1 s=CLOSED pgs=65 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.029+0000 7f0c00fc1640 1 --2- 172.21.15.32:0/4228849238 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0bfc10bd00 0x7f0bfc19c900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.029+0000 7f0c00fc1640 1 --2- 172.21.15.32:0/4228849238 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f0bfc1032e0 0x7f0bfc198580 secure :-1 s=CLOSED pgs=562 cs=0 l=1 rev1=1 crypto rx=0x7f0bf0006010 tx=0x7f0bf00042f0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.029+0000 7f0c00fc1640 1 --2- 172.21.15.32:0/4228849238 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f0bfc1029c0 0x7f0bfc198040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.029+0000 7f0c00fc1640 1 -- 172.21.15.32:0/4228849238 >> 172.21.15.32:0/4228849238 conn(0x7f0bfc077740 msgr2=0x7f0bfc0fd760 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:27.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.029+0000 7f0c00fc1640 1 -- 172.21.15.32:0/4228849238 shutdown_connections 2024-04-25T00:34:27.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.029+0000 7f0c00fc1640 1 -- 172.21.15.32:0/4228849238 wait complete. 2024-04-25T00:34:27.044 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-25T00:34:27.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.150+0000 7f4f99b89640 1 -- 172.21.15.32:0/831259459 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4f9406a780 msgr2=0x7f4f9410e5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:27.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.150+0000 7f4f99b89640 1 --2- 172.21.15.32:0/831259459 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4f9406a780 0x7f4f9410e5f0 secure :-1 s=READY pgs=566 cs=0 l=1 rev1=1 crypto rx=0x7f4f8c009cd0 tx=0x7f4f8c02a8e0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.150+0000 7f4f99b89640 1 -- 172.21.15.32:0/831259459 shutdown_connections 2024-04-25T00:34:27.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.150+0000 7f4f99b89640 1 --2- 172.21.15.32:0/831259459 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4f9406a780 0x7f4f9410e5f0 unknown :-1 s=CLOSED pgs=566 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.150+0000 7f4f99b89640 1 --2- 172.21.15.32:0/831259459 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4f94069dc0 0x7f4f9406a240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.150+0000 7f4f99b89640 1 --2- 172.21.15.32:0/831259459 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4f94107730 0x7f4f94107b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.150+0000 7f4f99b89640 1 -- 172.21.15.32:0/831259459 >> 172.21.15.32:0/831259459 conn(0x7f4f940776a0 msgr2=0x7f4f94077ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:27.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.150+0000 7f4f99b89640 1 -- 172.21.15.32:0/831259459 shutdown_connections 2024-04-25T00:34:27.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.150+0000 7f4f99b89640 1 -- 172.21.15.32:0/831259459 wait complete. 2024-04-25T00:34:27.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.151+0000 7f4f99b89640 1 Processor -- start 2024-04-25T00:34:27.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.151+0000 7f4f99b89640 1 -- start start 2024-04-25T00:34:27.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.151+0000 7f4f99b89640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4f94069dc0 0x7f4f94100670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.151+0000 7f4f99b89640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4f9406a780 0x7f4f94100bb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.152+0000 7f4f99b89640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4f94107730 0x7f4f941010f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.152+0000 7f4f99b89640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f4f94111b40 con 0x7f4f9406a780 2024-04-25T00:34:27.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.152+0000 7f4f92ffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4f9406a780 0x7f4f94100bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.152+0000 7f4f99b89640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f4f941119c0 con 0x7f4f94107730 2024-04-25T00:34:27.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.152+0000 7f4f99b89640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f4f94111cc0 con 0x7f4f94069dc0 2024-04-25T00:34:27.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.152+0000 7f4f92ffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4f9406a780 0x7f4f94100bb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39874/0 (socket says 172.21.15.32:39874) 2024-04-25T00:34:27.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.152+0000 7f4f92ffd640 1 -- 172.21.15.32:0/2219865600 learned_addr learned my addr 172.21.15.32:0/2219865600 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:27.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.152+0000 7f4f93fff640 1 --2- 172.21.15.32:0/2219865600 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4f94107730 0x7f4f941010f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.152+0000 7f4f937fe640 1 --2- 172.21.15.32:0/2219865600 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4f94069dc0 0x7f4f94100670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.155 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.152+0000 7f4f92ffd640 1 -- 172.21.15.32:0/2219865600 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4f94069dc0 msgr2=0x7f4f94100670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:27.155 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.152+0000 7f4f92ffd640 1 --2- 172.21.15.32:0/2219865600 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4f94069dc0 0x7f4f94100670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.155 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.152+0000 7f4f92ffd640 1 -- 172.21.15.32:0/2219865600 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4f94107730 msgr2=0x7f4f941010f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:27.155 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.152+0000 7f4f92ffd640 1 --2- 172.21.15.32:0/2219865600 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4f94107730 0x7f4f941010f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.155 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.152+0000 7f4f92ffd640 1 -- 172.21.15.32:0/2219865600 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4f941a6520 con 0x7f4f9406a780 2024-04-25T00:34:27.155 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.152+0000 7f4f92ffd640 1 --2- 172.21.15.32:0/2219865600 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4f9406a780 0x7f4f94100bb0 secure :-1 s=READY pgs=567 cs=0 l=1 rev1=1 crypto rx=0x7f4f8800e9f0 tx=0x7f4f8800edf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:27.155 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.153+0000 7f4f90ff9640 1 -- 172.21.15.32:0/2219865600 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f8800cd80 con 0x7f4f9406a780 2024-04-25T00:34:27.155 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.153+0000 7f4f99b89640 1 -- 172.21.15.32:0/2219865600 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4f941a6810 con 0x7f4f9406a780 2024-04-25T00:34:27.155 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.153+0000 7f4f90ff9640 1 -- 172.21.15.32:0/2219865600 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4f88005500 con 0x7f4f9406a780 2024-04-25T00:34:27.155 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.153+0000 7f4f99b89640 1 -- 172.21.15.32:0/2219865600 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4f941a6d60 con 0x7f4f9406a780 2024-04-25T00:34:27.157 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.153+0000 7f4f90ff9640 1 -- 172.21.15.32:0/2219865600 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f88010690 con 0x7f4f9406a780 2024-04-25T00:34:27.157 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.155+0000 7f4f9c974740 1 Processor -- start 2024-04-25T00:34:27.157 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.155+0000 7f4f9c974740 1 -- start start 2024-04-25T00:34:27.157 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.155+0000 7f4f9c974740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5632986ce580 0x5632986ce960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.157 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.156+0000 7f4f9c974740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ceea0 0x5632986d9030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.157 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.156+0000 7f4f9c974740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5632986d9570 0x5632986db960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.157 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.156+0000 7f4f9c974740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x563298633670 con 0x5632986ceea0 2024-04-25T00:34:27.157 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.156+0000 7f4f9c974740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5632985db970 con 0x5632986d9570 2024-04-25T00:34:27.157 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.156+0000 7f4f9c974740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x56329862f630 con 0x5632986ce580 2024-04-25T00:34:27.157 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.156+0000 7f4f937fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ceea0 0x5632986d9030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.157 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.156+0000 7f4f93fff640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5632986ce580 0x5632986ce960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.157 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.156+0000 7f4f93fff640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5632986ce580 0x5632986ce960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:54528/0 (socket says 172.21.15.32:54528) 2024-04-25T00:34:27.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.156+0000 7f4f93fff640 1 -- 172.21.15.32:0/3503391497 learned_addr learned my addr 172.21.15.32:0/3503391497 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:27.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.156+0000 7f4f937fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ceea0 0x5632986d9030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39878/0 (socket says 172.21.15.32:39878) 2024-04-25T00:34:27.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.156+0000 7f4f90ff9640 1 -- 172.21.15.32:0/2219865600 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 25) ==== 138066+0+0 (secure 0 0 0) 0x7f4f88004f60 con 0x7f4f9406a780 2024-04-25T00:34:27.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.156+0000 7f4f93fff640 1 -- 172.21.15.32:0/3503391497 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5632986d9570 msgr2=0x5632986db960 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-25T00:34:27.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.156+0000 7f4f93fff640 1 --2- 172.21.15.32:0/3503391497 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5632986d9570 0x5632986db960 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.156+0000 7f4f93fff640 1 -- 172.21.15.32:0/3503391497 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ceea0 msgr2=0x5632986d9030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:27.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.156+0000 7f4f93fff640 1 --2- 172.21.15.32:0/3503391497 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ceea0 0x5632986d9030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.156+0000 7f4f93fff640 1 -- 172.21.15.32:0/3503391497 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5632985ff400 con 0x5632986ce580 2024-04-25T00:34:27.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.157+0000 7f4f90ff9640 1 --2- 172.21.15.32:0/2219865600 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f4f7c0a2e20 0x7f4f7c0a52e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.157+0000 7f4f90ff9640 1 -- 172.21.15.32:0/2219865600 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f4f880cfda0 con 0x7f4f9406a780 2024-04-25T00:34:27.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.157+0000 7f4f937fe640 1 --2- 172.21.15.32:0/2219865600 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f4f7c0a2e20 0x7f4f7c0a52e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.157+0000 7f4f93fff640 1 --2- 172.21.15.32:0/3503391497 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5632986ce580 0x5632986ce960 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f4f8c009cd0 tx=0x7f4f8c009a20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c29f871ae442f548 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:27.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.158+0000 7f4f79ffb640 1 -- 172.21.15.32:0/3503391497 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f8c048070 con 0x5632986ce580 2024-04-25T00:34:27.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.158+0000 7f4f79ffb640 1 -- 172.21.15.32:0/3503391497 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4f8c036070 con 0x5632986ce580 2024-04-25T00:34:27.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.158+0000 7f4f79ffb640 1 -- 172.21.15.32:0/3503391497 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f8c004440 con 0x5632986ce580 2024-04-25T00:34:27.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.158+0000 7f4f9c974740 1 -- 172.21.15.32:0/3503391497 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5632986ce580 msgr2=0x5632986ce960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:27.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.158+0000 7f4f9c974740 1 --2- 172.21.15.32:0/3503391497 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5632986ce580 0x5632986ce960 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f4f8c009cd0 tx=0x7f4f8c009a20 comp rx=0 tx=0).stop 2024-04-25T00:34:27.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.158+0000 7f4f937fe640 1 --2- 172.21.15.32:0/2219865600 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f4f7c0a2e20 0x7f4f7c0a52e0 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x5632986cfb40 tx=0x7f4f8400b410 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:27.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.158+0000 7f4f9c974740 1 -- 172.21.15.32:0/3503391497 shutdown_connections 2024-04-25T00:34:27.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.158+0000 7f4f9c974740 1 --2- 172.21.15.32:0/3503391497 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5632986d9570 0x5632986db960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.158+0000 7f4f9c974740 1 --2- 172.21.15.32:0/3503391497 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ceea0 0x5632986d9030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.158+0000 7f4f9c974740 1 --2- 172.21.15.32:0/3503391497 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5632986ce580 0x5632986ce960 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.159+0000 7f4f9c974740 1 -- 172.21.15.32:0/3503391497 >> 172.21.15.32:0/3503391497 conn(0x5632986ca8a0 msgr2=0x5632986cac80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:27.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.159+0000 7f4f9c974740 1 -- 172.21.15.32:0/3503391497 shutdown_connections 2024-04-25T00:34:27.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.159+0000 7f4f9c974740 1 -- 172.21.15.32:0/3503391497 wait complete. 2024-04-25T00:34:27.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.159+0000 7f4f9c974740 1 -- start start 2024-04-25T00:34:27.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.159+0000 7f4f9c974740 1 Processor -- start 2024-04-25T00:34:27.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.159+0000 7f4f9c974740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ce580 0x5632986d8f80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.159+0000 7f4f9c974740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5632986ceea0 0x5632986d6270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.159+0000 7f4f9c974740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5632986d9570 0x5632986d6880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.159+0000 7f4f9c974740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5632986dc300 con 0x5632986ce580 2024-04-25T00:34:27.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.159+0000 7f4f9c974740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5632985ef050 con 0x5632986ceea0 2024-04-25T00:34:27.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.159+0000 7f4f9c974740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5632986dc480 con 0x5632986d9570 2024-04-25T00:34:27.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.160+0000 7f4f9c974740 10 client.0 _ll_register_callbacks cb 0x563298346be0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:34:27.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.160+0000 7f4f937fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ce580 0x5632986d8f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.160+0000 7f4f937fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ce580 0x5632986d8f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39882/0 (socket says 172.21.15.32:39882) 2024-04-25T00:34:27.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.160+0000 7f4f937fe640 1 -- 172.21.15.32:0/3386125448 learned_addr learned my addr 172.21.15.32:0/3386125448 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:27.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.160+0000 7f4f92ffd640 1 --2- 172.21.15.32:0/3386125448 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5632986ceea0 0x5632986d6270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.160+0000 7f4f93fff640 1 --2- 172.21.15.32:0/3386125448 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5632986d9570 0x5632986d6880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.160+0000 7f4f937fe640 1 -- 172.21.15.32:0/3386125448 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5632986d9570 msgr2=0x5632986d6880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:27.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.160+0000 7f4f937fe640 1 --2- 172.21.15.32:0/3386125448 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5632986d9570 0x5632986d6880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.160+0000 7f4f937fe640 1 -- 172.21.15.32:0/3386125448 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5632986ceea0 msgr2=0x5632986d6270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:27.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.160+0000 7f4f937fe640 1 --2- 172.21.15.32:0/3386125448 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5632986ceea0 0x5632986d6270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.160+0000 7f4f937fe640 1 -- 172.21.15.32:0/3386125448 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5632985c0dc0 con 0x5632986ce580 2024-04-25T00:34:27.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.160+0000 7f4f937fe640 1 --2- 172.21.15.32:0/3386125448 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ce580 0x5632986d8f80 secure :-1 s=READY pgs=568 cs=0 l=1 rev1=1 crypto rx=0x7f4f84005bc0 tx=0x7f4f84004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cf12edb58b1ffbdc server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:27.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.160+0000 7f4f72ffd640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:34:27.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.161+0000 7f4f72ffd640 1 -- 172.21.15.32:0/3386125448 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f8402c060 con 0x5632986ce580 2024-04-25T00:34:27.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.161+0000 7f4f9c974740 1 -- 172.21.15.32:0/3386125448 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4f8c036070 con 0x5632986ce580 2024-04-25T00:34:27.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.161+0000 7f4f72ffd640 1 -- 172.21.15.32:0/3386125448 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f4f84017030 con 0x5632986ce580 2024-04-25T00:34:27.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.161+0000 7f4f72ffd640 1 -- 172.21.15.32:0/3386125448 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f4f8401c6c0 con 0x5632986ce580 2024-04-25T00:34:27.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.161+0000 7f4f9c974740 10 client.15909 fetch_fsmap learned FSMap version 25 2024-04-25T00:34:27.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.161+0000 7f4f9c974740 1 -- 172.21.15.32:0/3386125448 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x5632983473f0 con 0x5632986ce580 2024-04-25T00:34:27.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.161+0000 7f4f72ffd640 1 -- 172.21.15.32:0/3386125448 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4f8c036070 con 0x5632986ce580 2024-04-25T00:34:27.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.161+0000 7f4f72ffd640 1 -- 172.21.15.32:0/3386125448 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7f4f84027940 con 0x5632986ce580 2024-04-25T00:34:27.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.162+0000 7f4f9c974740 10 client.15909 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:34:27.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.162+0000 7f4f9c974740 10 client.15909 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-25T00:34:27.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.162+0000 7f4f9c974740 20 client.15909 populate_metadata read hostname 'smithi032' 2024-04-25T00:34:27.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.162+0000 7f4f9c974740 1 --2- 172.21.15.32:0/3386125448 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x5632986f1dd0 0x5632986f4230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.162+0000 7f4f9c974740 4 client.15909 mds_command: new command op to 24481 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:34:27.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.162+0000 7f4f9c974740 1 -- 172.21.15.32:0/3386125448 --> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x563298640bb0 con 0x5632986f1dd0 2024-04-25T00:34:27.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.162+0000 7f4f92ffd640 1 --2- 172.21.15.32:0/3386125448 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x5632986f1dd0 0x5632986f4230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.163+0000 7f4f92ffd640 1 --2- 172.21.15.32:0/3386125448 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x5632986f1dd0 0x5632986f4230 crc :-1 s=READY pgs=32 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=8670a5288115cca7 server_cookie=3b2d332b3c40cd13 in_seq=0 out_seq=0 2024-04-25T00:34:27.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.163+0000 7f4f72ffd640 10 client.15909 ms_handle_connect on v2:172.21.15.130:6838/4267125809 2024-04-25T00:34:27.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.163+0000 7f4f72ffd640 1 -- 172.21.15.32:0/3386125448 <== mds.1 v2:172.21.15.130:6838/4267125809 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x563298640bb0 con 0x5632986f1dd0 2024-04-25T00:34:27.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.163+0000 7f4f72ffd640 10 client.15909 handle_command_reply: tid=0 2024-04-25T00:34:27.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.163+0000 7f4f9c974740 1 client.15909 shutdown 2024-04-25T00:34:27.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.164+0000 7f4f9c974740 1 -- 172.21.15.32:0/3386125448 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ce580 msgr2=0x5632986d8f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:27.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.164+0000 7f4f9c974740 1 --2- 172.21.15.32:0/3386125448 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ce580 0x5632986d8f80 secure :-1 s=READY pgs=568 cs=0 l=1 rev1=1 crypto rx=0x7f4f84005bc0 tx=0x7f4f84004730 comp rx=0 tx=0).stop 2024-04-25T00:34:27.169 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.164+0000 7f4f9c974740 1 -- 172.21.15.32:0/3386125448 shutdown_connections 2024-04-25T00:34:27.169 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.164+0000 7f4f9c974740 1 --2- 172.21.15.32:0/3386125448 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5632986d9570 0x5632986d6880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.169 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.164+0000 7f4f9c974740 1 --2- 172.21.15.32:0/3386125448 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5632986ceea0 0x5632986d6270 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.169 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.164+0000 7f4f9c974740 1 --2- 172.21.15.32:0/3386125448 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x5632986f1dd0 0x5632986f4230 crc :-1 s=READY pgs=32 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.169 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.164+0000 7f4f72ffd640 0 client.15909 ms_handle_reset on v2:172.21.15.130:6838/4267125809 2024-04-25T00:34:27.169 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.164+0000 7f4f9c974740 1 --2- 172.21.15.32:0/3386125448 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ce580 0x5632986d8f80 unknown :-1 s=CLOSED pgs=568 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.169 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.164+0000 7f4f9c974740 1 -- 172.21.15.32:0/3386125448 >> 172.21.15.32:0/3386125448 conn(0x5632986ca8a0 msgr2=0x5632986c9b40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:27.169 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.164+0000 7f4f9c974740 1 -- 172.21.15.32:0/3386125448 shutdown_connections 2024-04-25T00:34:27.169 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.164+0000 7f4f9c974740 1 -- 172.21.15.32:0/3386125448 wait complete. 2024-04-25T00:34:27.169 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.164+0000 7f4f9c974740 20 client.15909 trim_cache size 0 max 16384 2024-04-25T00:34:27.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.189+0000 7f4f9c974740 1 Processor -- start 2024-04-25T00:34:27.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.190+0000 7f4f9c974740 1 -- start start 2024-04-25T00:34:27.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.190+0000 7f4f9c974740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ce580 0x5632986ca2c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.190+0000 7f4f9c974740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5632986ceea0 0x5632986d8ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.190+0000 7f4f9c974740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5632986d9570 0x5632986f9b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.190+0000 7f4f9c974740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f4f80012920 con 0x5632986ce580 2024-04-25T00:34:27.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.190+0000 7f4f9c974740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f4f800101f0 con 0x5632986ceea0 2024-04-25T00:34:27.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.190+0000 7f4f9c974740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f4f8000dfe0 con 0x5632986d9570 2024-04-25T00:34:27.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.190+0000 7f4f937fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ce580 0x5632986ca2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.190+0000 7f4f937fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ce580 0x5632986ca2c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39890/0 (socket says 172.21.15.32:39890) 2024-04-25T00:34:27.192 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.190+0000 7f4f937fe640 1 -- 172.21.15.32:0/84087040 learned_addr learned my addr 172.21.15.32:0/84087040 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:27.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.190+0000 7f4f93fff640 1 --2- 172.21.15.32:0/84087040 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5632986d9570 0x5632986f9b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.190+0000 7f4f92ffd640 1 --2- 172.21.15.32:0/84087040 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5632986ceea0 0x5632986d8ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.190+0000 7f4f937fe640 1 -- 172.21.15.32:0/84087040 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5632986d9570 msgr2=0x5632986f9b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:27.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.190+0000 7f4f937fe640 1 --2- 172.21.15.32:0/84087040 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5632986d9570 0x5632986f9b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.191+0000 7f4f937fe640 1 -- 172.21.15.32:0/84087040 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5632986ceea0 msgr2=0x5632986d8ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:27.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.191+0000 7f4f937fe640 1 --2- 172.21.15.32:0/84087040 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5632986ceea0 0x5632986d8ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.191+0000 7f4f937fe640 1 -- 172.21.15.32:0/84087040 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5632986dbcd0 con 0x5632986ce580 2024-04-25T00:34:27.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.191+0000 7f4f92ffd640 1 --2- 172.21.15.32:0/84087040 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5632986ceea0 0x5632986d8ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:34:27.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.191+0000 7f4f937fe640 1 --2- 172.21.15.32:0/84087040 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ce580 0x5632986ca2c0 secure :-1 s=READY pgs=569 cs=0 l=1 rev1=1 crypto rx=0x7f4f8400be10 tx=0x7f4f84010640 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3a8b2d4b246979ad server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:27.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.191+0000 7f4f797fa640 1 -- 172.21.15.32:0/84087040 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f8401c6c0 con 0x5632986ce580 2024-04-25T00:34:27.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.191+0000 7f4f797fa640 1 -- 172.21.15.32:0/84087040 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4f8401c860 con 0x5632986ce580 2024-04-25T00:34:27.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.191+0000 7f4f797fa640 1 -- 172.21.15.32:0/84087040 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f84027410 con 0x5632986ce580 2024-04-25T00:34:27.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.192+0000 7f4f9c974740 1 -- 172.21.15.32:0/84087040 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ce580 msgr2=0x5632986ca2c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:27.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.192+0000 7f4f9c974740 1 --2- 172.21.15.32:0/84087040 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ce580 0x5632986ca2c0 secure :-1 s=READY pgs=569 cs=0 l=1 rev1=1 crypto rx=0x7f4f8400be10 tx=0x7f4f84010640 comp rx=0 tx=0).stop 2024-04-25T00:34:27.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.192+0000 7f4f9c974740 1 -- 172.21.15.32:0/84087040 shutdown_connections 2024-04-25T00:34:27.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.192+0000 7f4f9c974740 1 --2- 172.21.15.32:0/84087040 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5632986d9570 0x5632986f9b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.192+0000 7f4f9c974740 1 --2- 172.21.15.32:0/84087040 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5632986ceea0 0x5632986d8ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.192+0000 7f4f9c974740 1 --2- 172.21.15.32:0/84087040 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ce580 0x5632986ca2c0 unknown :-1 s=CLOSED pgs=569 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.192+0000 7f4f9c974740 1 -- 172.21.15.32:0/84087040 >> 172.21.15.32:0/84087040 conn(0x5632986ca8a0 msgr2=0x5632986d1860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:27.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.192+0000 7f4f9c974740 1 -- 172.21.15.32:0/84087040 shutdown_connections 2024-04-25T00:34:27.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.192+0000 7f4f9c974740 1 -- 172.21.15.32:0/84087040 wait complete. 2024-04-25T00:34:27.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.193+0000 7f4f9c974740 1 -- start start 2024-04-25T00:34:27.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.193+0000 7f4f9c974740 1 Processor -- start 2024-04-25T00:34:27.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.193+0000 7f4f9c974740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ce580 0x5632986d3210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.193+0000 7f4f9c974740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5632986ceea0 0x5632986d3750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.193+0000 7f4f9c974740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5632986d9570 0x5632986d3c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.193+0000 7f4f9c974740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f4f8000e6b0 con 0x5632986ce580 2024-04-25T00:34:27.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.193+0000 7f4f9c974740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f4f8000fb20 con 0x5632986ceea0 2024-04-25T00:34:27.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.194+0000 7f4f937fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ce580 0x5632986d3210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.194+0000 7f4f937fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ce580 0x5632986d3210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39906/0 (socket says 172.21.15.32:39906) 2024-04-25T00:34:27.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.194+0000 7f4f937fe640 1 -- 172.21.15.32:0/849487470 learned_addr learned my addr 172.21.15.32:0/849487470 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:27.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.194+0000 7f4f9c974740 1 -- 172.21.15.32:0/849487470 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f4f8000ed80 con 0x5632986d9570 2024-04-25T00:34:27.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.194+0000 7f4f93fff640 1 --2- 172.21.15.32:0/849487470 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5632986d9570 0x5632986d3c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.194+0000 7f4f92ffd640 1 --2- 172.21.15.32:0/849487470 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5632986ceea0 0x5632986d3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.194+0000 7f4f9c974740 10 client.0 _ll_register_callbacks cb 0x7f4f8400ba80 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:34:27.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.194+0000 7f4f937fe640 1 -- 172.21.15.32:0/849487470 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5632986d9570 msgr2=0x5632986d3c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:27.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.194+0000 7f4f937fe640 1 --2- 172.21.15.32:0/849487470 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5632986d9570 0x5632986d3c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.194+0000 7f4f937fe640 1 -- 172.21.15.32:0/849487470 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5632986ceea0 msgr2=0x5632986d3750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:27.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.194+0000 7f4f937fe640 1 --2- 172.21.15.32:0/849487470 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5632986ceea0 0x5632986d3750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.194+0000 7f4f937fe640 1 -- 172.21.15.32:0/849487470 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5632984117d0 con 0x5632986ce580 2024-04-25T00:34:27.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.194+0000 7f4f937fe640 1 --2- 172.21.15.32:0/849487470 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ce580 0x5632986d3210 secure :-1 s=READY pgs=570 cs=0 l=1 rev1=1 crypto rx=0x7f4f84000c00 tx=0x7f4f8401dc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7beb1d3fa1effbdb server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:27.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.194+0000 7f4f72ffd640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:34:27.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.195+0000 7f4f72ffd640 1 -- 172.21.15.32:0/849487470 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f8401c6c0 con 0x5632986ce580 2024-04-25T00:34:27.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.195+0000 7f4f72ffd640 1 -- 172.21.15.32:0/849487470 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f4f84036070 con 0x5632986ce580 2024-04-25T00:34:27.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.195+0000 7f4f9c974740 1 -- 172.21.15.32:0/849487470 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4f8401c860 con 0x5632986ce580 2024-04-25T00:34:27.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.195+0000 7f4f72ffd640 1 -- 172.21.15.32:0/849487470 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f4f84045050 con 0x5632986ce580 2024-04-25T00:34:27.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.195+0000 7f4f9c974740 10 client.15915 fetch_fsmap learned FSMap version 25 2024-04-25T00:34:27.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.195+0000 7f4f9c974740 1 -- 172.21.15.32:0/849487470 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x5632987586b0 con 0x5632986ce580 2024-04-25T00:34:27.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.195+0000 7f4f72ffd640 1 -- 172.21.15.32:0/849487470 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4f8401c860 con 0x5632986ce580 2024-04-25T00:34:27.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.195+0000 7f4f72ffd640 1 -- 172.21.15.32:0/849487470 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7f4f840277f0 con 0x5632986ce580 2024-04-25T00:34:27.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.195+0000 7f4f9c974740 10 client.15915 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:34:27.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.196+0000 7f4f9c974740 10 client.15915 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-25T00:34:27.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.196+0000 7f4f9c974740 20 client.15915 populate_metadata read hostname 'smithi032' 2024-04-25T00:34:27.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.196+0000 7f4f9c974740 1 --2- 172.21.15.32:0/849487470 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x5632986f1dd0 0x56329875ac70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.196+0000 7f4f9c974740 4 client.15915 mds_command: new command op to 24481 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-25T00:34:27.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.196+0000 7f4f9c974740 1 -- 172.21.15.32:0/849487470 --> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] -- command(tid 0: {"prefix": "damage ls"}) -- 0x56329863af00 con 0x5632986f1dd0 2024-04-25T00:34:27.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.196+0000 7f4f92ffd640 1 --2- 172.21.15.32:0/849487470 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x5632986f1dd0 0x56329875ac70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.196+0000 7f4f92ffd640 1 --2- 172.21.15.32:0/849487470 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x5632986f1dd0 0x56329875ac70 crc :-1 s=READY pgs=33 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=fd32aca09f001f65 server_cookie=393897a0fff3f588 in_seq=0 out_seq=0 2024-04-25T00:34:27.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.196+0000 7f4f72ffd640 10 client.15915 ms_handle_connect on v2:172.21.15.130:6838/4267125809 2024-04-25T00:34:27.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.197+0000 7f4f72ffd640 1 -- 172.21.15.32:0/849487470 <== mds.1 v2:172.21.15.130:6838/4267125809 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x56329863af00 con 0x5632986f1dd0 2024-04-25T00:34:27.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.197+0000 7f4f72ffd640 10 client.15915 handle_command_reply: tid=0 2024-04-25T00:34:27.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.197+0000 7f4f9c974740 1 client.15915 shutdown 2024-04-25T00:34:27.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.197+0000 7f4f9c974740 1 -- 172.21.15.32:0/849487470 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ce580 msgr2=0x5632986d3210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:27.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.197+0000 7f4f9c974740 1 --2- 172.21.15.32:0/849487470 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ce580 0x5632986d3210 secure :-1 s=READY pgs=570 cs=0 l=1 rev1=1 crypto rx=0x7f4f84000c00 tx=0x7f4f8401dc30 comp rx=0 tx=0).stop 2024-04-25T00:34:27.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.197+0000 7f4f9c974740 1 -- 172.21.15.32:0/849487470 shutdown_connections 2024-04-25T00:34:27.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.197+0000 7f4f9c974740 1 --2- 172.21.15.32:0/849487470 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5632986d9570 0x5632986d3c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.197+0000 7f4f9c974740 1 --2- 172.21.15.32:0/849487470 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5632986ceea0 0x5632986d3750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.197+0000 7f4f9c974740 1 --2- 172.21.15.32:0/849487470 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x5632986f1dd0 0x56329875ac70 crc :-1 s=READY pgs=33 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.197+0000 7f4f9c974740 1 --2- 172.21.15.32:0/849487470 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5632986ce580 0x5632986d3210 unknown :-1 s=CLOSED pgs=570 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.197+0000 7f4f72ffd640 0 client.15915 ms_handle_reset on v2:172.21.15.130:6838/4267125809 2024-04-25T00:34:27.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.197+0000 7f4f9c974740 1 -- 172.21.15.32:0/849487470 >> 172.21.15.32:0/849487470 conn(0x5632986ca8a0 msgr2=0x5632986cfd10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:27.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.197+0000 7f4f9c974740 1 -- 172.21.15.32:0/849487470 shutdown_connections 2024-04-25T00:34:27.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.198+0000 7f4f9c974740 1 -- 172.21.15.32:0/849487470 wait complete. 2024-04-25T00:34:27.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.198+0000 7f4f9c974740 20 client.15915 trim_cache size 0 max 16384 2024-04-25T00:34:27.205 INFO:teuthology.orchestra.run.smithi032.stdout:[] 2024-04-25T00:34:27.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.198+0000 7f4f99b89640 1 -- 172.21.15.32:0/2219865600 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f4f7c0a2e20 msgr2=0x7f4f7c0a52e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:27.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.198+0000 7f4f99b89640 1 --2- 172.21.15.32:0/2219865600 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f4f7c0a2e20 0x7f4f7c0a52e0 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x5632986cfb40 tx=0x7f4f8400b410 comp rx=0 tx=0).stop 2024-04-25T00:34:27.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.198+0000 7f4f99b89640 1 -- 172.21.15.32:0/2219865600 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4f9406a780 msgr2=0x7f4f94100bb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:27.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.198+0000 7f4f99b89640 1 --2- 172.21.15.32:0/2219865600 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4f9406a780 0x7f4f94100bb0 secure :-1 s=READY pgs=567 cs=0 l=1 rev1=1 crypto rx=0x7f4f8800e9f0 tx=0x7f4f8800edf0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.198+0000 7f4f99b89640 1 -- 172.21.15.32:0/2219865600 shutdown_connections 2024-04-25T00:34:27.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.198+0000 7f4f99b89640 1 --2- 172.21.15.32:0/2219865600 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f4f7c0a2e20 0x7f4f7c0a52e0 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.198+0000 7f4f99b89640 1 --2- 172.21.15.32:0/2219865600 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4f94107730 0x7f4f941010f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.198+0000 7f4f99b89640 1 --2- 172.21.15.32:0/2219865600 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4f9406a780 0x7f4f94100bb0 unknown :-1 s=CLOSED pgs=567 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.198+0000 7f4f99b89640 1 --2- 172.21.15.32:0/2219865600 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4f94069dc0 0x7f4f94100670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.198+0000 7f4f99b89640 1 -- 172.21.15.32:0/2219865600 >> 172.21.15.32:0/2219865600 conn(0x7f4f940776a0 msgr2=0x7f4f940fe650 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:27.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.199+0000 7f4f99b89640 1 -- 172.21.15.32:0/2219865600 shutdown_connections 2024-04-25T00:34:27.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.199+0000 7f4f99b89640 1 -- 172.21.15.32:0/2219865600 wait complete. 2024-04-25T00:34:27.212 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:27 smithi032 ceph-mon[29357]: pgmap v37: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:34:27.212 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:27 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3700630824' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:34:27.213 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-25T00:34:27.321 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.320+0000 7fc9e7b16640 1 -- 172.21.15.32:0/2151184876 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc9e01032e0 msgr2=0x7fc9e010b7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:27.321 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.320+0000 7fc9e7b16640 1 --2- 172.21.15.32:0/2151184876 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc9e01032e0 0x7fc9e010b7c0 secure :-1 s=READY pgs=571 cs=0 l=1 rev1=1 crypto rx=0x7fc9d0009900 tx=0x7fc9d002a510 comp rx=0 tx=0).stop 2024-04-25T00:34:27.321 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.320+0000 7fc9e7b16640 1 -- 172.21.15.32:0/2151184876 shutdown_connections 2024-04-25T00:34:27.321 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.320+0000 7fc9e7b16640 1 --2- 172.21.15.32:0/2151184876 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc9e010bd00 0x7fc9e010e1a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.321 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.320+0000 7fc9e7b16640 1 --2- 172.21.15.32:0/2151184876 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc9e01032e0 0x7fc9e010b7c0 unknown :-1 s=CLOSED pgs=571 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.322 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.320+0000 7fc9e7b16640 1 --2- 172.21.15.32:0/2151184876 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc9e01029c0 0x7fc9e0102da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.322 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.320+0000 7fc9e7b16640 1 -- 172.21.15.32:0/2151184876 >> 172.21.15.32:0/2151184876 conn(0x7fc9e0077740 msgr2=0x7fc9e0077b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:27.322 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.320+0000 7fc9e7b16640 1 -- 172.21.15.32:0/2151184876 shutdown_connections 2024-04-25T00:34:27.322 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.320+0000 7fc9e7b16640 1 -- 172.21.15.32:0/2151184876 wait complete. 2024-04-25T00:34:27.324 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.321+0000 7fc9e7b16640 1 Processor -- start 2024-04-25T00:34:27.324 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.321+0000 7fc9e7b16640 1 -- start start 2024-04-25T00:34:27.324 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.321+0000 7fc9e7b16640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc9e01029c0 0x7fc9e019a220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.324 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.321+0000 7fc9e7b16640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc9e01032e0 0x7fc9e019a760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.325 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.321+0000 7fc9e7b16640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc9e010bd00 0x7fc9e019eae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.325 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.321+0000 7fc9e50b1640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc9e01032e0 0x7fc9e019a760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.325 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.321+0000 7fc9e7b16640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fc9e0111b50 con 0x7fc9e01032e0 2024-04-25T00:34:27.325 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.321+0000 7fc9e7b16640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fc9e01119d0 con 0x7fc9e010bd00 2024-04-25T00:34:27.325 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.321+0000 7fc9e50b1640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc9e01032e0 0x7fc9e019a760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39926/0 (socket says 172.21.15.32:39926) 2024-04-25T00:34:27.325 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.321+0000 7fc9e7b16640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fc9e0111cd0 con 0x7fc9e01029c0 2024-04-25T00:34:27.325 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.322+0000 7fc9e50b1640 1 -- 172.21.15.32:0/3763654274 learned_addr learned my addr 172.21.15.32:0/3763654274 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:27.325 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.322+0000 7fc9e58b2640 1 --2- 172.21.15.32:0/3763654274 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc9e01029c0 0x7fc9e019a220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.325 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.322+0000 7fc9e60b3640 1 --2- 172.21.15.32:0/3763654274 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc9e010bd00 0x7fc9e019eae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.325 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.322+0000 7fc9e50b1640 1 -- 172.21.15.32:0/3763654274 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc9e01029c0 msgr2=0x7fc9e019a220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:27.325 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.322+0000 7fc9e50b1640 1 --2- 172.21.15.32:0/3763654274 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc9e01029c0 0x7fc9e019a220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.325 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.322+0000 7fc9e50b1640 1 -- 172.21.15.32:0/3763654274 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc9e010bd00 msgr2=0x7fc9e019eae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:27.325 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.322+0000 7fc9e50b1640 1 --2- 172.21.15.32:0/3763654274 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc9e010bd00 0x7fc9e019eae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.325 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.322+0000 7fc9e50b1640 1 -- 172.21.15.32:0/3763654274 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc9e019f210 con 0x7fc9e01032e0 2024-04-25T00:34:27.325 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.322+0000 7fc9e50b1640 1 --2- 172.21.15.32:0/3763654274 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc9e01032e0 0x7fc9e019a760 secure :-1 s=READY pgs=572 cs=0 l=1 rev1=1 crypto rx=0x7fc9d0004770 tx=0x7fc9d0003c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:27.326 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.322+0000 7fc9d6ffd640 1 -- 172.21.15.32:0/3763654274 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc9d003f070 con 0x7fc9e01032e0 2024-04-25T00:34:27.326 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.322+0000 7fc9d6ffd640 1 -- 172.21.15.32:0/3763654274 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc9d0031ca0 con 0x7fc9e01032e0 2024-04-25T00:34:27.326 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.322+0000 7fc9e7b16640 1 -- 172.21.15.32:0/3763654274 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc9e019f4a0 con 0x7fc9e01032e0 2024-04-25T00:34:27.326 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.323+0000 7fc9d6ffd640 1 -- 172.21.15.32:0/3763654274 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc9d0035030 con 0x7fc9e01032e0 2024-04-25T00:34:27.326 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.323+0000 7fc9e7b16640 1 -- 172.21.15.32:0/3763654274 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc9e01a6d90 con 0x7fc9e01032e0 2024-04-25T00:34:27.327 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.326+0000 7fc9d6ffd640 1 -- 172.21.15.32:0/3763654274 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 25) ==== 138066+0+0 (secure 0 0 0) 0x7fc9d0042070 con 0x7fc9e01032e0 2024-04-25T00:34:27.327 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.326+0000 7fc9ea81a740 1 Processor -- start 2024-04-25T00:34:27.327 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.326+0000 7fc9ea81a740 1 -- start start 2024-04-25T00:34:27.327 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.327+0000 7fc9ea81a740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55dcd6096580 0x55dcd6096960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.327 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.327+0000 7fc9ea81a740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55dcd6096ea0 0x55dcd60a1030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.327 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.327+0000 7fc9ea81a740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd60a1570 0x55dcd60a3960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.327 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.327+0000 7fc9ea81a740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55dcd5ffb670 con 0x55dcd60a1570 2024-04-25T00:34:27.327 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.327+0000 7fc9ea81a740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55dcd5fa3970 con 0x55dcd6096ea0 2024-04-25T00:34:27.327 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.327+0000 7fc9ea81a740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55dcd5ff7630 con 0x55dcd6096580 2024-04-25T00:34:27.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.327+0000 7fc9e50b1640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd60a1570 0x55dcd60a3960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.327+0000 7fc9e50b1640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd60a1570 0x55dcd60a3960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39934/0 (socket says 172.21.15.32:39934) 2024-04-25T00:34:27.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.327+0000 7fc9e50b1640 1 -- 172.21.15.32:0/1217766305 learned_addr learned my addr 172.21.15.32:0/1217766305 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:27.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.327+0000 7fc9e60b3640 1 --2- 172.21.15.32:0/1217766305 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55dcd6096580 0x55dcd6096960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.327+0000 7fc9e58b2640 1 --2- 172.21.15.32:0/1217766305 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55dcd6096ea0 0x55dcd60a1030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.327+0000 7fc9e60b3640 1 -- 172.21.15.32:0/1217766305 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55dcd6096ea0 msgr2=0x55dcd60a1030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:27.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.327+0000 7fc9e60b3640 1 --2- 172.21.15.32:0/1217766305 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55dcd6096ea0 0x55dcd60a1030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.327+0000 7fc9e60b3640 1 -- 172.21.15.32:0/1217766305 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd60a1570 msgr2=0x55dcd60a3960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:27.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.327+0000 7fc9e60b3640 1 --2- 172.21.15.32:0/1217766305 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd60a1570 0x55dcd60a3960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.327+0000 7fc9e60b3640 1 -- 172.21.15.32:0/1217766305 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55dcd5fc7400 con 0x55dcd6096580 2024-04-25T00:34:27.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.328+0000 7fc9e50b1640 1 --2- 172.21.15.32:0/1217766305 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd60a1570 0x55dcd60a3960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:34:27.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.328+0000 7fc9e60b3640 1 --2- 172.21.15.32:0/1217766305 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55dcd6096580 0x55dcd6096960 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7fc9d8009e40 tx=0x7fc9d800f730 comp rx=0 tx=0).ready entity=mon.1 client_cookie=13ae01b73b2dbb62 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:27.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.328+0000 7fc9d6ffd640 1 --2- 172.21.15.32:0/3763654274 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fc9cc0a2ea0 0x7fc9cc0a5360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.330 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.328+0000 7fc9e58b2640 1 --2- 172.21.15.32:0/3763654274 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fc9cc0a2ea0 0x7fc9cc0a5360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.330 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.328+0000 7fc9cbfff640 1 -- 172.21.15.32:0/1217766305 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc9d8015070 con 0x55dcd6096580 2024-04-25T00:34:27.330 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.328+0000 7fc9d6ffd640 1 -- 172.21.15.32:0/3763654274 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fc9d00efdb0 con 0x7fc9e01032e0 2024-04-25T00:34:27.330 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.328+0000 7fc9cbfff640 1 -- 172.21.15.32:0/1217766305 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc9d801ab40 con 0x55dcd6096580 2024-04-25T00:34:27.330 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.328+0000 7fc9cbfff640 1 -- 172.21.15.32:0/1217766305 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc9d8011bb0 con 0x55dcd6096580 2024-04-25T00:34:27.333 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.329+0000 7fc9ea81a740 1 -- 172.21.15.32:0/1217766305 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55dcd6096580 msgr2=0x55dcd6096960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:27.333 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.329+0000 7fc9e58b2640 1 --2- 172.21.15.32:0/3763654274 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fc9cc0a2ea0 0x7fc9cc0a5360 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x55dcd6097b40 tx=0x7fc9dc00a2d0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:27.333 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.329+0000 7fc9ea81a740 1 --2- 172.21.15.32:0/1217766305 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55dcd6096580 0x55dcd6096960 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7fc9d8009e40 tx=0x7fc9d800f730 comp rx=0 tx=0).stop 2024-04-25T00:34:27.333 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.329+0000 7fc9ea81a740 1 -- 172.21.15.32:0/1217766305 shutdown_connections 2024-04-25T00:34:27.333 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.329+0000 7fc9ea81a740 1 --2- 172.21.15.32:0/1217766305 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd60a1570 0x55dcd60a3960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.333 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.329+0000 7fc9ea81a740 1 --2- 172.21.15.32:0/1217766305 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55dcd6096ea0 0x55dcd60a1030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.333 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.329+0000 7fc9ea81a740 1 --2- 172.21.15.32:0/1217766305 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55dcd6096580 0x55dcd6096960 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.333 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.329+0000 7fc9ea81a740 1 -- 172.21.15.32:0/1217766305 >> 172.21.15.32:0/1217766305 conn(0x55dcd60928a0 msgr2=0x55dcd6092c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:27.333 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.329+0000 7fc9ea81a740 1 -- 172.21.15.32:0/1217766305 shutdown_connections 2024-04-25T00:34:27.333 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.329+0000 7fc9ea81a740 1 -- 172.21.15.32:0/1217766305 wait complete. 2024-04-25T00:34:27.334 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.330+0000 7fc9ea81a740 1 -- start start 2024-04-25T00:34:27.334 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.330+0000 7fc9ea81a740 1 Processor -- start 2024-04-25T00:34:27.334 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.330+0000 7fc9ea81a740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd6096580 0x55dcd60a0ec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.334 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.330+0000 7fc9ea81a740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55dcd6096ea0 0x55dcd609e1b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.334 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.330+0000 7fc9ea81a740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55dcd60a1570 0x55dcd609e710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.334 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.330+0000 7fc9ea81a740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55dcd60a4360 con 0x55dcd6096580 2024-04-25T00:34:27.334 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.330+0000 7fc9ea81a740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55dcd5fb7050 con 0x55dcd6096ea0 2024-04-25T00:34:27.334 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.330+0000 7fc9ea81a740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55dcd60a44e0 con 0x55dcd60a1570 2024-04-25T00:34:27.334 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.330+0000 7fc9e58b2640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd6096580 0x55dcd60a0ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.334 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.330+0000 7fc9ea81a740 10 client.0 _ll_register_callbacks cb 0x55dcd5d0ebe0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:34:27.334 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.330+0000 7fc9e58b2640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd6096580 0x55dcd60a0ec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39948/0 (socket says 172.21.15.32:39948) 2024-04-25T00:34:27.334 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.330+0000 7fc9e58b2640 1 -- 172.21.15.32:0/3632955826 learned_addr learned my addr 172.21.15.32:0/3632955826 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:27.334 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.331+0000 7fc9e60b3640 1 --2- 172.21.15.32:0/3632955826 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55dcd60a1570 0x55dcd609e710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.335 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.331+0000 7fc9e58b2640 1 -- 172.21.15.32:0/3632955826 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55dcd60a1570 msgr2=0x55dcd609e710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:27.335 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.331+0000 7fc9e58b2640 1 --2- 172.21.15.32:0/3632955826 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55dcd60a1570 0x55dcd609e710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.335 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.331+0000 7fc9e58b2640 1 -- 172.21.15.32:0/3632955826 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55dcd6096ea0 msgr2=0x55dcd609e1b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:27.335 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.331+0000 7fc9e50b1640 1 --2- 172.21.15.32:0/3632955826 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55dcd6096ea0 0x55dcd609e1b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.335 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.331+0000 7fc9e58b2640 1 --2- 172.21.15.32:0/3632955826 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55dcd6096ea0 0x55dcd609e1b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.335 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.331+0000 7fc9e58b2640 1 -- 172.21.15.32:0/3632955826 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55dcd5f88dc0 con 0x55dcd6096580 2024-04-25T00:34:27.335 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.331+0000 7fc9e60b3640 1 --2- 172.21.15.32:0/3632955826 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55dcd60a1570 0x55dcd609e710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:34:27.335 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.331+0000 7fc9e50b1640 1 --2- 172.21.15.32:0/3632955826 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55dcd6096ea0 0x55dcd609e1b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:34:27.335 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.331+0000 7fc9e58b2640 1 --2- 172.21.15.32:0/3632955826 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd6096580 0x55dcd60a0ec0 secure :-1 s=READY pgs=573 cs=0 l=1 rev1=1 crypto rx=0x7fc9dc01c8f0 tx=0x7fc9dc004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ddcfbec76cb0b02e server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:27.335 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.331+0000 7fc9c97fa640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:34:27.335 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.331+0000 7fc9c97fa640 1 -- 172.21.15.32:0/3632955826 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc9dc02c060 con 0x55dcd6096580 2024-04-25T00:34:27.335 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.331+0000 7fc9c97fa640 1 -- 172.21.15.32:0/3632955826 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fc9dc016030 con 0x55dcd6096580 2024-04-25T00:34:27.335 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.332+0000 7fc9ea81a740 1 -- 172.21.15.32:0/3632955826 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc9d801ab40 con 0x55dcd6096580 2024-04-25T00:34:27.335 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.332+0000 7fc9c97fa640 1 -- 172.21.15.32:0/3632955826 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7fc9dc027420 con 0x55dcd6096580 2024-04-25T00:34:27.335 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.332+0000 7fc9ea81a740 10 client.15927 fetch_fsmap learned FSMap version 25 2024-04-25T00:34:27.336 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.332+0000 7fc9ea81a740 1 -- 172.21.15.32:0/3632955826 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x55dcd5d0f3f0 con 0x55dcd6096580 2024-04-25T00:34:27.337 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.332+0000 7fc9c97fa640 1 -- 172.21.15.32:0/3632955826 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc9d801ab40 con 0x55dcd6096580 2024-04-25T00:34:27.337 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.333+0000 7fc9c97fa640 1 -- 172.21.15.32:0/3632955826 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7fc9dc0304c0 con 0x55dcd6096580 2024-04-25T00:34:27.337 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.333+0000 7fc9ea81a740 10 client.15927 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:34:27.338 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.333+0000 7fc9ea81a740 10 client.15927 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:34:27.338 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.333+0000 7fc9ea81a740 20 client.15927 populate_metadata read hostname 'smithi032' 2024-04-25T00:34:27.338 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.333+0000 7fc9ea81a740 1 --2- 172.21.15.32:0/3632955826 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55dcd60b1650 0x55dcd60b3a40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.338 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.333+0000 7fc9ea81a740 4 client.15927 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:34:27.338 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.333+0000 7fc9ea81a740 1 -- 172.21.15.32:0/3632955826 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55dcd6008bb0 con 0x55dcd60b1650 2024-04-25T00:34:27.338 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.333+0000 7fc9e50b1640 1 --2- 172.21.15.32:0/3632955826 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55dcd60b1650 0x55dcd60b3a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.338 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.334+0000 7fc9e50b1640 1 --2- 172.21.15.32:0/3632955826 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55dcd60b1650 0x55dcd60b3a40 crc :-1 s=READY pgs=55 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ada997657b81732a server_cookie=7c24d3043aa21463 in_seq=0 out_seq=0 2024-04-25T00:34:27.338 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.334+0000 7fc9c97fa640 10 client.15927 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:34:27.338 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.334+0000 7fc9c97fa640 1 -- 172.21.15.32:0/3632955826 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x55dcd6008bb0 con 0x55dcd60b1650 2024-04-25T00:34:27.338 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.334+0000 7fc9c97fa640 10 client.15927 handle_command_reply: tid=0 2024-04-25T00:34:27.338 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.334+0000 7fc9ea81a740 1 client.15927 shutdown 2024-04-25T00:34:27.338 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.335+0000 7fc9ea81a740 1 -- 172.21.15.32:0/3632955826 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd6096580 msgr2=0x55dcd60a0ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:27.338 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.335+0000 7fc9ea81a740 1 --2- 172.21.15.32:0/3632955826 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd6096580 0x55dcd60a0ec0 secure :-1 s=READY pgs=573 cs=0 l=1 rev1=1 crypto rx=0x7fc9dc01c8f0 tx=0x7fc9dc004730 comp rx=0 tx=0).stop 2024-04-25T00:34:27.339 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.335+0000 7fc9ea81a740 1 -- 172.21.15.32:0/3632955826 shutdown_connections 2024-04-25T00:34:27.339 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.335+0000 7fc9ea81a740 1 --2- 172.21.15.32:0/3632955826 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55dcd60b1650 0x55dcd60b3a40 crc :-1 s=READY pgs=55 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.339 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.335+0000 7fc9ea81a740 1 --2- 172.21.15.32:0/3632955826 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55dcd60a1570 0x55dcd609e710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.339 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.335+0000 7fc9c97fa640 0 client.15927 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:34:27.339 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.335+0000 7fc9ea81a740 1 --2- 172.21.15.32:0/3632955826 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55dcd6096ea0 0x55dcd609e1b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.339 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.335+0000 7fc9ea81a740 1 --2- 172.21.15.32:0/3632955826 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd6096580 0x55dcd60a0ec0 unknown :-1 s=CLOSED pgs=573 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.339 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.335+0000 7fc9ea81a740 1 -- 172.21.15.32:0/3632955826 >> 172.21.15.32:0/3632955826 conn(0x55dcd60928a0 msgr2=0x55dcd60986a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:27.339 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.335+0000 7fc9ea81a740 1 -- 172.21.15.32:0/3632955826 shutdown_connections 2024-04-25T00:34:27.339 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.335+0000 7fc9ea81a740 1 -- 172.21.15.32:0/3632955826 wait complete. 2024-04-25T00:34:27.339 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.335+0000 7fc9ea81a740 20 client.15927 trim_cache size 0 max 16384 2024-04-25T00:34:27.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:27 smithi121 ceph-mon[32231]: pgmap v37: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:34:27.340 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:27 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3700630824' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:34:27.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.362+0000 7fc9ea81a740 1 Processor -- start 2024-04-25T00:34:27.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.362+0000 7fc9ea81a740 1 -- start start 2024-04-25T00:34:27.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.362+0000 7fc9ea81a740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55dcd6096580 0x55dcd60ad250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.362+0000 7fc9ea81a740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55dcd6096ea0 0x55dcd60a09f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.362+0000 7fc9ea81a740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd60a1570 0x55dcd60b92e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.362+0000 7fc9ea81a740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fc9e01b3770 con 0x55dcd60a1570 2024-04-25T00:34:27.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.362+0000 7fc9ea81a740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fc9e01b1040 con 0x55dcd6096ea0 2024-04-25T00:34:27.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.362+0000 7fc9ea81a740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fc9e01aee90 con 0x55dcd6096580 2024-04-25T00:34:27.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.362+0000 7fc9e60b3640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd60a1570 0x55dcd60b92e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.362+0000 7fc9e60b3640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd60a1570 0x55dcd60b92e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39950/0 (socket says 172.21.15.32:39950) 2024-04-25T00:34:27.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.362+0000 7fc9e60b3640 1 -- 172.21.15.32:0/195363542 learned_addr learned my addr 172.21.15.32:0/195363542 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:27.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.362+0000 7fc9e50b1640 1 --2- 172.21.15.32:0/195363542 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55dcd6096ea0 0x55dcd60a09f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.362+0000 7fc9e58b2640 1 --2- 172.21.15.32:0/195363542 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55dcd6096580 0x55dcd60ad250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.363+0000 7fc9e60b3640 1 -- 172.21.15.32:0/195363542 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55dcd6096580 msgr2=0x55dcd60ad250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:27.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.363+0000 7fc9e60b3640 1 --2- 172.21.15.32:0/195363542 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55dcd6096580 0x55dcd60ad250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.363+0000 7fc9e60b3640 1 -- 172.21.15.32:0/195363542 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55dcd6096ea0 msgr2=0x55dcd60a09f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:27.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.363+0000 7fc9e60b3640 1 --2- 172.21.15.32:0/195363542 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55dcd6096ea0 0x55dcd60a09f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.363+0000 7fc9e60b3640 1 -- 172.21.15.32:0/195363542 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55dcd5dd97d0 con 0x55dcd60a1570 2024-04-25T00:34:27.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.363+0000 7fc9e50b1640 1 --2- 172.21.15.32:0/195363542 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55dcd6096ea0 0x55dcd60a09f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:34:27.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.363+0000 7fc9e60b3640 1 --2- 172.21.15.32:0/195363542 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd60a1570 0x55dcd60b92e0 secure :-1 s=READY pgs=574 cs=0 l=1 rev1=1 crypto rx=0x7fc9d8009d90 tx=0x7fc9d8009a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bc0725e60278748b server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:27.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.363+0000 7fc9cb7fe640 1 -- 172.21.15.32:0/195363542 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc9d8028070 con 0x55dcd60a1570 2024-04-25T00:34:27.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.363+0000 7fc9cb7fe640 1 -- 172.21.15.32:0/195363542 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc9d8003e50 con 0x55dcd60a1570 2024-04-25T00:34:27.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.363+0000 7fc9cb7fe640 1 -- 172.21.15.32:0/195363542 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc9d8015040 con 0x55dcd60a1570 2024-04-25T00:34:27.369 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.364+0000 7fc9ea81a740 1 -- 172.21.15.32:0/195363542 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd60a1570 msgr2=0x55dcd60b92e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:27.370 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.364+0000 7fc9ea81a740 1 --2- 172.21.15.32:0/195363542 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd60a1570 0x55dcd60b92e0 secure :-1 s=READY pgs=574 cs=0 l=1 rev1=1 crypto rx=0x7fc9d8009d90 tx=0x7fc9d8009a40 comp rx=0 tx=0).stop 2024-04-25T00:34:27.370 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.364+0000 7fc9ea81a740 1 -- 172.21.15.32:0/195363542 shutdown_connections 2024-04-25T00:34:27.370 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.364+0000 7fc9ea81a740 1 --2- 172.21.15.32:0/195363542 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd60a1570 0x55dcd60b92e0 unknown :-1 s=CLOSED pgs=574 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.370 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.364+0000 7fc9ea81a740 1 --2- 172.21.15.32:0/195363542 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55dcd6096ea0 0x55dcd60a09f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.370 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.364+0000 7fc9ea81a740 1 --2- 172.21.15.32:0/195363542 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55dcd6096580 0x55dcd60ad250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.370 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.364+0000 7fc9ea81a740 1 -- 172.21.15.32:0/195363542 >> 172.21.15.32:0/195363542 conn(0x55dcd60928a0 msgr2=0x55dcd6099740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:27.370 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.364+0000 7fc9ea81a740 1 -- 172.21.15.32:0/195363542 shutdown_connections 2024-04-25T00:34:27.370 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.364+0000 7fc9ea81a740 1 -- 172.21.15.32:0/195363542 wait complete. 2024-04-25T00:34:27.370 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.365+0000 7fc9ea81a740 1 -- start start 2024-04-25T00:34:27.370 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.365+0000 7fc9ea81a740 1 Processor -- start 2024-04-25T00:34:27.370 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.365+0000 7fc9ea81a740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd6096580 0x55dcd60a9a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.371 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.365+0000 7fc9ea81a740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55dcd6096ea0 0x55dcd60a4e60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.371 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.365+0000 7fc9ea81a740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55dcd60a1570 0x55dcd60a53a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.371 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.365+0000 7fc9ea81a740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fc9e01af500 con 0x55dcd6096580 2024-04-25T00:34:27.371 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.365+0000 7fc9ea81a740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fc9e01b0970 con 0x55dcd6096ea0 2024-04-25T00:34:27.371 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.365+0000 7fc9ea81a740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fc9e01afbd0 con 0x55dcd60a1570 2024-04-25T00:34:27.371 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.365+0000 7fc9ea81a740 10 client.0 _ll_register_callbacks cb 0x7fc9dc00a660 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:34:27.371 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.365+0000 7fc9e58b2640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd6096580 0x55dcd60a9a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.371 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.366+0000 7fc9e58b2640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd6096580 0x55dcd60a9a70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39952/0 (socket says 172.21.15.32:39952) 2024-04-25T00:34:27.371 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.366+0000 7fc9e58b2640 1 -- 172.21.15.32:0/2211111562 learned_addr learned my addr 172.21.15.32:0/2211111562 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:27.371 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.366+0000 7fc9e50b1640 1 --2- 172.21.15.32:0/2211111562 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55dcd6096ea0 0x55dcd60a4e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.371 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.366+0000 7fc9e58b2640 1 -- 172.21.15.32:0/2211111562 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55dcd60a1570 msgr2=0x55dcd60a53a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:27.371 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.366+0000 7fc9e58b2640 1 --2- 172.21.15.32:0/2211111562 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55dcd60a1570 0x55dcd60a53a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.371 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.366+0000 7fc9e58b2640 1 -- 172.21.15.32:0/2211111562 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55dcd6096ea0 msgr2=0x55dcd60a4e60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:27.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.366+0000 7fc9e58b2640 1 --2- 172.21.15.32:0/2211111562 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55dcd6096ea0 0x55dcd60a4e60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.366+0000 7fc9e58b2640 1 -- 172.21.15.32:0/2211111562 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55dcd60a5ba0 con 0x55dcd6096580 2024-04-25T00:34:27.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.366+0000 7fc9e58b2640 1 --2- 172.21.15.32:0/2211111562 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd6096580 0x55dcd60a9a70 secure :-1 s=READY pgs=575 cs=0 l=1 rev1=1 crypto rx=0x7fc9dc030ad0 tx=0x7fc9dc0305d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5730f97fc3d5ff09 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:27.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.366+0000 7fc9c97fa640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:34:27.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.366+0000 7fc9c97fa640 1 -- 172.21.15.32:0/2211111562 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc9dc005c20 con 0x55dcd6096580 2024-04-25T00:34:27.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.366+0000 7fc9c97fa640 1 -- 172.21.15.32:0/2211111562 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fc9dc022040 con 0x55dcd6096580 2024-04-25T00:34:27.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.366+0000 7fc9ea81a740 1 -- 172.21.15.32:0/2211111562 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc9d8003e50 con 0x55dcd6096580 2024-04-25T00:34:27.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.366+0000 7fc9c97fa640 1 -- 172.21.15.32:0/2211111562 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7fc9dc02c020 con 0x55dcd6096580 2024-04-25T00:34:27.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.366+0000 7fc9ea81a740 10 client.15933 fetch_fsmap learned FSMap version 25 2024-04-25T00:34:27.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.367+0000 7fc9ea81a740 1 -- 172.21.15.32:0/2211111562 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x55dcd6117aa0 con 0x55dcd6096580 2024-04-25T00:34:27.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.367+0000 7fc9c97fa640 1 -- 172.21.15.32:0/2211111562 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc9d8003e50 con 0x55dcd6096580 2024-04-25T00:34:27.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.367+0000 7fc9c97fa640 1 -- 172.21.15.32:0/2211111562 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7fc9dc03b550 con 0x55dcd6096580 2024-04-25T00:34:27.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.367+0000 7fc9ea81a740 10 client.15933 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:34:27.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.367+0000 7fc9ea81a740 10 client.15933 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:34:27.373 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.367+0000 7fc9ea81a740 20 client.15933 populate_metadata read hostname 'smithi032' 2024-04-25T00:34:27.373 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.367+0000 7fc9ea81a740 1 --2- 172.21.15.32:0/2211111562 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55dcd6118270 0x55dcd611a6f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:27.373 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.367+0000 7fc9ea81a740 4 client.15933 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-25T00:34:27.373 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.367+0000 7fc9ea81a740 1 -- 172.21.15.32:0/2211111562 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55dcd6002f00 con 0x55dcd6118270 2024-04-25T00:34:27.373 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.367+0000 7fc9e50b1640 1 --2- 172.21.15.32:0/2211111562 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55dcd6118270 0x55dcd611a6f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:27.373 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.368+0000 7fc9c97fa640 10 client.15933 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:34:27.373 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.368+0000 7fc9e50b1640 1 --2- 172.21.15.32:0/2211111562 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55dcd6118270 0x55dcd611a6f0 crc :-1 s=READY pgs=56 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=13ed8751cee869cc server_cookie=90ce5779f26b1758 in_seq=0 out_seq=0 2024-04-25T00:34:27.373 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.368+0000 7fc9c97fa640 1 -- 172.21.15.32:0/2211111562 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55dcd6002f00 con 0x55dcd6118270 2024-04-25T00:34:27.373 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.368+0000 7fc9c97fa640 10 client.15933 handle_command_reply: tid=0 2024-04-25T00:34:27.373 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.368+0000 7fc9ea81a740 1 client.15933 shutdown 2024-04-25T00:34:27.373 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.368+0000 7fc9ea81a740 1 -- 172.21.15.32:0/2211111562 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd6096580 msgr2=0x55dcd60a9a70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:27.373 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.368+0000 7fc9ea81a740 1 --2- 172.21.15.32:0/2211111562 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd6096580 0x55dcd60a9a70 secure :-1 s=READY pgs=575 cs=0 l=1 rev1=1 crypto rx=0x7fc9dc030ad0 tx=0x7fc9dc0305d0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.373 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.368+0000 7fc9ea81a740 1 -- 172.21.15.32:0/2211111562 shutdown_connections 2024-04-25T00:34:27.373 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.368+0000 7fc9ea81a740 1 --2- 172.21.15.32:0/2211111562 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55dcd6118270 0x55dcd611a6f0 crc :-1 s=READY pgs=56 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.374 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.368+0000 7fc9ea81a740 1 --2- 172.21.15.32:0/2211111562 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55dcd60a1570 0x55dcd60a53a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.374 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.368+0000 7fc9ea81a740 1 --2- 172.21.15.32:0/2211111562 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55dcd6096ea0 0x55dcd60a4e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.374 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.368+0000 7fc9c97fa640 0 client.15933 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:34:27.374 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.368+0000 7fc9ea81a740 1 --2- 172.21.15.32:0/2211111562 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55dcd6096580 0x55dcd60a9a70 unknown :-1 s=CLOSED pgs=575 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.374 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.368+0000 7fc9ea81a740 1 -- 172.21.15.32:0/2211111562 >> 172.21.15.32:0/2211111562 conn(0x55dcd60928a0 msgr2=0x55dcd6097cd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:27.374 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.369+0000 7fc9ea81a740 1 -- 172.21.15.32:0/2211111562 shutdown_connections 2024-04-25T00:34:27.374 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.369+0000 7fc9ea81a740 1 -- 172.21.15.32:0/2211111562 wait complete. 2024-04-25T00:34:27.374 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.369+0000 7fc9ea81a740 20 client.15933 trim_cache size 0 max 16384 2024-04-25T00:34:27.374 INFO:teuthology.orchestra.run.smithi032.stdout:[] 2024-04-25T00:34:27.375 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.369+0000 7fc9e7b16640 1 -- 172.21.15.32:0/3763654274 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fc9cc0a2ea0 msgr2=0x7fc9cc0a5360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:27.375 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.369+0000 7fc9e7b16640 1 --2- 172.21.15.32:0/3763654274 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fc9cc0a2ea0 0x7fc9cc0a5360 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x55dcd6097b40 tx=0x7fc9dc00a2d0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.375 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.369+0000 7fc9e7b16640 1 -- 172.21.15.32:0/3763654274 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc9e01032e0 msgr2=0x7fc9e019a760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:27.375 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.369+0000 7fc9e7b16640 1 --2- 172.21.15.32:0/3763654274 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc9e01032e0 0x7fc9e019a760 secure :-1 s=READY pgs=572 cs=0 l=1 rev1=1 crypto rx=0x7fc9d0004770 tx=0x7fc9d0003c10 comp rx=0 tx=0).stop 2024-04-25T00:34:27.375 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.369+0000 7fc9e7b16640 1 -- 172.21.15.32:0/3763654274 shutdown_connections 2024-04-25T00:34:27.376 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.369+0000 7fc9e7b16640 1 --2- 172.21.15.32:0/3763654274 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fc9cc0a2ea0 0x7fc9cc0a5360 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.376 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.369+0000 7fc9e7b16640 1 --2- 172.21.15.32:0/3763654274 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc9e010bd00 0x7fc9e019eae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.376 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.369+0000 7fc9e7b16640 1 --2- 172.21.15.32:0/3763654274 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc9e01032e0 0x7fc9e019a760 unknown :-1 s=CLOSED pgs=572 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.376 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.369+0000 7fc9e7b16640 1 --2- 172.21.15.32:0/3763654274 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc9e01029c0 0x7fc9e019a220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:27.376 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.369+0000 7fc9e7b16640 1 -- 172.21.15.32:0/3763654274 >> 172.21.15.32:0/3763654274 conn(0x7fc9e0077740 msgr2=0x7fc9e01019b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:27.376 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.370+0000 7fc9e7b16640 1 -- 172.21.15.32:0/3763654274 shutdown_connections 2024-04-25T00:34:27.376 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:27.370+0000 7fc9e7b16640 1 -- 172.21.15.32:0/3763654274 wait complete. 2024-04-25T00:34:27.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:27 smithi130 ceph-mon[32525]: pgmap v37: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:34:27.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:27 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3700630824' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:34:28.382 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-25T00:34:28.382 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:34:28.382 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-25T00:34:28.525 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.525+0000 7f9075e38640 1 -- 172.21.15.32:0/1195972820 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f90701029a0 msgr2=0x7f9070102d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:28.526 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.525+0000 7f9075e38640 1 --2- 172.21.15.32:0/1195972820 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f90701029a0 0x7f9070102d80 secure :-1 s=READY pgs=576 cs=0 l=1 rev1=1 crypto rx=0x7f9060009900 tx=0x7f906002a480 comp rx=0 tx=0).stop 2024-04-25T00:34:28.526 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.525+0000 7f9075e38640 1 -- 172.21.15.32:0/1195972820 shutdown_connections 2024-04-25T00:34:28.526 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.525+0000 7f9075e38640 1 --2- 172.21.15.32:0/1195972820 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f907010bd70 0x7f907010e1a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.526 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.525+0000 7f9075e38640 1 --2- 172.21.15.32:0/1195972820 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f90701032c0 0x7f907010b7a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.526 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.525+0000 7f9075e38640 1 --2- 172.21.15.32:0/1195972820 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f90701029a0 0x7f9070102d80 unknown :-1 s=CLOSED pgs=576 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.526 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.525+0000 7f9075e38640 1 -- 172.21.15.32:0/1195972820 >> 172.21.15.32:0/1195972820 conn(0x7f90700769c0 msgr2=0x7f9070076dd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:28.526 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.525+0000 7f9075e38640 1 -- 172.21.15.32:0/1195972820 shutdown_connections 2024-04-25T00:34:28.526 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.525+0000 7f9075e38640 1 -- 172.21.15.32:0/1195972820 wait complete. 2024-04-25T00:34:28.528 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.526+0000 7f9075e38640 1 Processor -- start 2024-04-25T00:34:28.528 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.526+0000 7f9075e38640 1 -- start start 2024-04-25T00:34:28.528 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.526+0000 7f9075e38640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f90701029a0 0x7f9070198000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.528 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.526+0000 7f9075e38640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f90701032c0 0x7f9070198540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.528 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.526+0000 7f9075e38640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f907010bd70 0x7f907019c8c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.528 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.526+0000 7f9075e38640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f9070111b00 con 0x7f907010bd70 2024-04-25T00:34:28.529 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.526+0000 7f9075e38640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f9070111980 con 0x7f90701032c0 2024-04-25T00:34:28.529 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.526+0000 7f9075e38640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f9070111c80 con 0x7f90701029a0 2024-04-25T00:34:28.529 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.526+0000 7f906ffff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f907010bd70 0x7f907019c8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.529 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.526+0000 7f906ffff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f907010bd70 0x7f907019c8c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39956/0 (socket says 172.21.15.32:39956) 2024-04-25T00:34:28.529 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.526+0000 7f906ffff640 1 -- 172.21.15.32:0/1574405761 learned_addr learned my addr 172.21.15.32:0/1574405761 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:28.529 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.526+0000 7f906f7fe640 1 --2- 172.21.15.32:0/1574405761 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f90701029a0 0x7f9070198000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.529 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.527+0000 7f906effd640 1 --2- 172.21.15.32:0/1574405761 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f90701032c0 0x7f9070198540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.529 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.527+0000 7f906ffff640 1 -- 172.21.15.32:0/1574405761 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f90701029a0 msgr2=0x7f9070198000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:28.529 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.527+0000 7f906ffff640 1 --2- 172.21.15.32:0/1574405761 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f90701029a0 0x7f9070198000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.529 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.527+0000 7f906ffff640 1 -- 172.21.15.32:0/1574405761 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f90701032c0 msgr2=0x7f9070198540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:28.529 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.527+0000 7f906ffff640 1 --2- 172.21.15.32:0/1574405761 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f90701032c0 0x7f9070198540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.529 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.527+0000 7f906ffff640 1 -- 172.21.15.32:0/1574405761 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f907019cff0 con 0x7f907010bd70 2024-04-25T00:34:28.529 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.527+0000 7f906ffff640 1 --2- 172.21.15.32:0/1574405761 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f907010bd70 0x7f907019c8c0 secure :-1 s=READY pgs=577 cs=0 l=1 rev1=1 crypto rx=0x7f906800eb40 tx=0x7f906800c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:28.529 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.527+0000 7f906cff9640 1 -- 172.21.15.32:0/1574405761 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9068004b20 con 0x7f907010bd70 2024-04-25T00:34:28.530 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.527+0000 7f906cff9640 1 -- 172.21.15.32:0/1574405761 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9068004cc0 con 0x7f907010bd70 2024-04-25T00:34:28.530 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.527+0000 7f9075e38640 1 -- 172.21.15.32:0/1574405761 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f907019d2e0 con 0x7f907010bd70 2024-04-25T00:34:28.530 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.527+0000 7f906cff9640 1 -- 172.21.15.32:0/1574405761 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f90680107f0 con 0x7f907010bd70 2024-04-25T00:34:28.530 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.527+0000 7f9075e38640 1 -- 172.21.15.32:0/1574405761 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9070100520 con 0x7f907010bd70 2024-04-25T00:34:28.531 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.530+0000 7f906cff9640 1 -- 172.21.15.32:0/1574405761 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 25) ==== 138066+0+0 (secure 0 0 0) 0x7f9068005000 con 0x7f907010bd70 2024-04-25T00:34:28.531 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.531+0000 7f9078c26740 1 Processor -- start 2024-04-25T00:34:28.531 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.531+0000 7f9078c26740 1 -- start start 2024-04-25T00:34:28.532 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.531+0000 7f9078c26740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e08145980 0x556e08143cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.533 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.531+0000 7f9078c26740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e08144290 0x556e08144710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.533 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.531+0000 7f9078c26740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556e0814f010 0x556e08151400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.533 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.531+0000 7f9078c26740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x556e080a82c0 con 0x556e08144290 2024-04-25T00:34:28.533 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.531+0000 7f906cff9640 1 --2- 172.21.15.32:0/1574405761 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f90580a2ea0 0x7f90580a5360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.533 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.531+0000 7f9078c26740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x556e08050910 con 0x556e0814f010 2024-04-25T00:34:28.533 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.531+0000 7f9078c26740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x556e080a4280 con 0x556e08145980 2024-04-25T00:34:28.533 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.531+0000 7f906f7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e08144290 0x556e08144710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.533 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.531+0000 7f906cff9640 1 -- 172.21.15.32:0/1574405761 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f90680d02f0 con 0x7f907010bd70 2024-04-25T00:34:28.534 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.531+0000 7f906f7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e08144290 0x556e08144710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39970/0 (socket says 172.21.15.32:39970) 2024-04-25T00:34:28.534 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.531+0000 7f906f7fe640 1 -- 172.21.15.32:0/3313685437 learned_addr learned my addr 172.21.15.32:0/3313685437 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:28.534 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.531+0000 7f906ffff640 1 --2- 172.21.15.32:0/1574405761 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f90580a2ea0 0x7f90580a5360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.534 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.532+0000 7f906effd640 1 --2- 172.21.15.32:0/3313685437 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e08145980 0x556e08143cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.534 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.532+0000 7f906effd640 1 --2- 172.21.15.32:0/3313685437 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556e0814f010 0x556e08151400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.534 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.532+0000 7f906f7fe640 1 -- 172.21.15.32:0/3313685437 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e08145980 msgr2=0x556e08143cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:28.534 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.532+0000 7f906f7fe640 1 --2- 172.21.15.32:0/3313685437 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e08145980 0x556e08143cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.534 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.532+0000 7f906f7fe640 1 -- 172.21.15.32:0/3313685437 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556e0814f010 msgr2=0x556e08151400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:28.534 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.532+0000 7f906f7fe640 1 --2- 172.21.15.32:0/3313685437 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556e0814f010 0x556e08151400 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.534 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.532+0000 7f906f7fe640 1 -- 172.21.15.32:0/3313685437 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x556e08061fb0 con 0x556e08144290 2024-04-25T00:34:28.534 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.532+0000 7f906ffff640 1 --2- 172.21.15.32:0/1574405761 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f90580a2ea0 0x7f90580a5360 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f90680cd510 tx=0x7f90680d0540 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:28.534 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.532+0000 7f906f7fe640 1 --2- 172.21.15.32:0/3313685437 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e08144290 0x556e08144710 secure :-1 s=READY pgs=578 cs=0 l=1 rev1=1 crypto rx=0x7f9060009900 tx=0x7f9060005e00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c7b04debf0d9a78a server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:28.534 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.533+0000 7f9055ffb640 1 -- 172.21.15.32:0/3313685437 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9060040070 con 0x556e08144290 2024-04-25T00:34:28.534 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.533+0000 7f9055ffb640 1 -- 172.21.15.32:0/3313685437 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f90600318f0 con 0x556e08144290 2024-04-25T00:34:28.535 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.533+0000 7f9055ffb640 1 -- 172.21.15.32:0/3313685437 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9060035040 con 0x556e08144290 2024-04-25T00:34:28.538 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.534+0000 7f9078c26740 1 -- 172.21.15.32:0/3313685437 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e08144290 msgr2=0x556e08144710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:28.538 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.534+0000 7f9078c26740 1 --2- 172.21.15.32:0/3313685437 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e08144290 0x556e08144710 secure :-1 s=READY pgs=578 cs=0 l=1 rev1=1 crypto rx=0x7f9060009900 tx=0x7f9060005e00 comp rx=0 tx=0).stop 2024-04-25T00:34:28.538 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.534+0000 7f9078c26740 1 -- 172.21.15.32:0/3313685437 shutdown_connections 2024-04-25T00:34:28.538 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.534+0000 7f9078c26740 1 --2- 172.21.15.32:0/3313685437 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556e0814f010 0x556e08151400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.539 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.534+0000 7f9078c26740 1 --2- 172.21.15.32:0/3313685437 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e08144290 0x556e08144710 unknown :-1 s=CLOSED pgs=578 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.539 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.534+0000 7f9078c26740 1 --2- 172.21.15.32:0/3313685437 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e08145980 0x556e08143cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.539 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.534+0000 7f9078c26740 1 -- 172.21.15.32:0/3313685437 >> 172.21.15.32:0/3313685437 conn(0x556e0810c950 msgr2=0x556e081407f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:28.539 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.534+0000 7f9078c26740 1 -- 172.21.15.32:0/3313685437 shutdown_connections 2024-04-25T00:34:28.539 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.534+0000 7f9078c26740 1 -- 172.21.15.32:0/3313685437 wait complete. 2024-04-25T00:34:28.539 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.534+0000 7f9078c26740 1 -- start start 2024-04-25T00:34:28.539 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.534+0000 7f9078c26740 1 Processor -- start 2024-04-25T00:34:28.539 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.535+0000 7f9078c26740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556e08145980 0x556e0814c1a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.539 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.535+0000 7f9078c26740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e0814f010 0x556e0814c6e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.539 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.535+0000 7f9078c26740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e0814cc70 0x556e0815b2f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.539 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.535+0000 7f906f7fe640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e0814f010 0x556e0814c6e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.539 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.535+0000 7f906f7fe640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e0814f010 0x556e0814c6e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:54624/0 (socket says 172.21.15.32:54624) 2024-04-25T00:34:28.539 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.535+0000 7f906effd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e0814cc70 0x556e0815b2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.539 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.535+0000 7f906effd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e0814cc70 0x556e0815b2f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39976/0 (socket says 172.21.15.32:39976) 2024-04-25T00:34:28.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.535+0000 7f906f7fe640 1 -- 172.21.15.32:0/3708324788 learned_addr learned my addr 172.21.15.32:0/3708324788 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:28.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.535+0000 7f9078c26740 1 -- 172.21.15.32:0/3708324788 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x556e08151e40 con 0x556e0814cc70 2024-04-25T00:34:28.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.535+0000 7f9078c26740 1 -- 172.21.15.32:0/3708324788 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x556e08063e90 con 0x556e08145980 2024-04-25T00:34:28.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.535+0000 7f9078c26740 1 -- 172.21.15.32:0/3708324788 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x556e08151fc0 con 0x556e0814f010 2024-04-25T00:34:28.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.536+0000 7f906f7fe640 1 -- 172.21.15.32:0/3708324788 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556e08145980 msgr2=0x556e0814c1a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:28.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.536+0000 7f906f7fe640 1 --2- 172.21.15.32:0/3708324788 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556e08145980 0x556e0814c1a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.536+0000 7f906f7fe640 1 -- 172.21.15.32:0/3708324788 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e0814cc70 msgr2=0x556e0815b2f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:28.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.536+0000 7f906f7fe640 1 --2- 172.21.15.32:0/3708324788 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e0814cc70 0x556e0815b2f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.536+0000 7f906f7fe640 1 -- 172.21.15.32:0/3708324788 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x556e07e847e0 con 0x556e0814f010 2024-04-25T00:34:28.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.536+0000 7f906effd640 1 --2- 172.21.15.32:0/3708324788 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e0814cc70 0x556e0815b2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:34:28.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.536+0000 7f9078c26740 10 client.0 _ll_register_callbacks cb 0x556e0809f970 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:34:28.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.536+0000 7f9078c26740 1 -- 172.21.15.32:0/3708324788 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f90600318f0 con 0x556e0814f010 2024-04-25T00:34:28.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.536+0000 7f906f7fe640 1 --2- 172.21.15.32:0/3708324788 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e0814f010 0x556e0814c6e0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f906002a9a0 tx=0x7f90600052d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=3f7b60e77c413dd5 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:28.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.536+0000 7f904effd640 10 client.25468 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:34:28.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.536+0000 7f904effd640 1 -- 172.21.15.32:0/3708324788 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9060040070 con 0x556e0814f010 2024-04-25T00:34:28.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.536+0000 7f904effd640 1 -- 172.21.15.32:0/3708324788 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f9060004240 con 0x556e0814f010 2024-04-25T00:34:28.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.537+0000 7f904effd640 1 -- 172.21.15.32:0/3708324788 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f9060035040 con 0x556e0814f010 2024-04-25T00:34:28.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.537+0000 7f9078c26740 10 client.25468 fetch_fsmap learned FSMap version 25 2024-04-25T00:34:28.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.537+0000 7f9078c26740 1 -- 172.21.15.32:0/3708324788 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({fsmap=25}) -- 0x556e07ee4f70 con 0x556e0814f010 2024-04-25T00:34:28.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.537+0000 7f904effd640 1 -- 172.21.15.32:0/3708324788 <== mon.1 v2:172.21.15.130:3300/0 4 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7f90600046c0 con 0x556e0814f010 2024-04-25T00:34:28.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.537+0000 7f9078c26740 10 client.25468 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:34:28.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.537+0000 7f9078c26740 10 client.25468 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:34:28.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.537+0000 7f9078c26740 20 client.25468 populate_metadata read hostname 'smithi032' 2024-04-25T00:34:28.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.537+0000 7f9078c26740 1 --2- 172.21.15.32:0/3708324788 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x556e0815f2a0 0x556e08161690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.537+0000 7f9078c26740 4 client.25468 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:34:28.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.537+0000 7f9078c26740 1 -- 172.21.15.32:0/3708324788 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x556e0807c5a0 con 0x556e0815f2a0 2024-04-25T00:34:28.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.537+0000 7f906effd640 1 --2- 172.21.15.32:0/3708324788 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x556e0815f2a0 0x556e08161690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.538+0000 7f904effd640 10 client.25468 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:34:28.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.538+0000 7f906effd640 1 --2- 172.21.15.32:0/3708324788 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x556e0815f2a0 0x556e08161690 crc :-1 s=READY pgs=57 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=190748e9766aeb52 server_cookie=a9d187874eedfd67 in_seq=0 out_seq=0 2024-04-25T00:34:28.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.539+0000 7f904effd640 1 -- 172.21.15.32:0/3708324788 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x556e0807c5a0 con 0x556e0815f2a0 2024-04-25T00:34:28.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.539+0000 7f904effd640 10 client.25468 handle_command_reply: tid=0 2024-04-25T00:34:28.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.539+0000 7f9078c26740 1 client.25468 shutdown 2024-04-25T00:34:28.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.539+0000 7f9078c26740 1 -- 172.21.15.32:0/3708324788 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e0814f010 msgr2=0x556e0814c6e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:28.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.539+0000 7f9078c26740 1 --2- 172.21.15.32:0/3708324788 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e0814f010 0x556e0814c6e0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f906002a9a0 tx=0x7f90600052d0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.539+0000 7f9078c26740 1 -- 172.21.15.32:0/3708324788 shutdown_connections 2024-04-25T00:34:28.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.539+0000 7f9078c26740 1 --2- 172.21.15.32:0/3708324788 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x556e0815f2a0 0x556e08161690 crc :-1 s=READY pgs=57 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.539+0000 7f9078c26740 1 --2- 172.21.15.32:0/3708324788 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e0814cc70 0x556e0815b2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.539+0000 7f9078c26740 1 --2- 172.21.15.32:0/3708324788 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e0814f010 0x556e0814c6e0 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.544 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.539+0000 7f9078c26740 1 --2- 172.21.15.32:0/3708324788 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556e08145980 0x556e0814c1a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.544 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.539+0000 7f9078c26740 1 -- 172.21.15.32:0/3708324788 >> 172.21.15.32:0/3708324788 conn(0x556e0810c950 msgr2=0x556e08146660 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:28.544 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.539+0000 7f904effd640 0 client.25468 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:34:28.544 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.539+0000 7f9078c26740 1 -- 172.21.15.32:0/3708324788 shutdown_connections 2024-04-25T00:34:28.544 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.539+0000 7f9078c26740 1 -- 172.21.15.32:0/3708324788 wait complete. 2024-04-25T00:34:28.544 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.539+0000 7f9078c26740 20 client.25468 trim_cache size 0 max 16384 2024-04-25T00:34:28.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.557+0000 7f9078c26740 1 Processor -- start 2024-04-25T00:34:28.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.557+0000 7f9078c26740 1 -- start start 2024-04-25T00:34:28.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.557+0000 7f9078c26740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e08145980 0x556e0813f7d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.557+0000 7f9078c26740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556e0814cc70 0x556e08166c60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.557+0000 7f9078c26740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e0814f010 0x556e081671a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.557+0000 7f9078c26740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f90701aae60 con 0x556e08145980 2024-04-25T00:34:28.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.557+0000 7f9078c26740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f9070073110 con 0x556e0814cc70 2024-04-25T00:34:28.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.557+0000 7f9078c26740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f9070070f60 con 0x556e0814f010 2024-04-25T00:34:28.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.558+0000 7f906effd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e08145980 0x556e0813f7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.558+0000 7f906effd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e08145980 0x556e0813f7d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39986/0 (socket says 172.21.15.32:39986) 2024-04-25T00:34:28.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.558+0000 7f906effd640 1 -- 172.21.15.32:0/1946669137 learned_addr learned my addr 172.21.15.32:0/1946669137 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:28.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.558+0000 7f906effd640 1 -- 172.21.15.32:0/1946669137 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e0814f010 msgr2=0x556e081671a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:28.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.558+0000 7f906effd640 1 --2- 172.21.15.32:0/1946669137 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e0814f010 0x556e081671a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.560 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.558+0000 7f906effd640 1 -- 172.21.15.32:0/1946669137 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556e0814cc70 msgr2=0x556e08166c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:28.560 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.558+0000 7f906effd640 1 --2- 172.21.15.32:0/1946669137 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556e0814cc70 0x556e08166c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.560 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.558+0000 7f906effd640 1 -- 172.21.15.32:0/1946669137 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x556e07d7c170 con 0x556e08145980 2024-04-25T00:34:28.560 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.558+0000 7f906effd640 1 --2- 172.21.15.32:0/1946669137 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e08145980 0x556e0813f7d0 secure :-1 s=READY pgs=579 cs=0 l=1 rev1=1 crypto rx=0x7f9064005780 tx=0x7f906400c6e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=54bf29df51b0d6a2 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:28.560 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.558+0000 7f90557fa640 1 -- 172.21.15.32:0/1946669137 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9064034030 con 0x556e08145980 2024-04-25T00:34:28.560 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.558+0000 7f90557fa640 1 -- 172.21.15.32:0/1946669137 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9064025e60 con 0x556e08145980 2024-04-25T00:34:28.560 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.558+0000 7f90557fa640 1 -- 172.21.15.32:0/1946669137 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9064005280 con 0x556e08145980 2024-04-25T00:34:28.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.559+0000 7f9078c26740 1 -- 172.21.15.32:0/1946669137 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e08145980 msgr2=0x556e0813f7d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:28.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.559+0000 7f9078c26740 1 --2- 172.21.15.32:0/1946669137 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e08145980 0x556e0813f7d0 secure :-1 s=READY pgs=579 cs=0 l=1 rev1=1 crypto rx=0x7f9064005780 tx=0x7f906400c6e0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.559+0000 7f9078c26740 1 -- 172.21.15.32:0/1946669137 shutdown_connections 2024-04-25T00:34:28.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.559+0000 7f9078c26740 1 --2- 172.21.15.32:0/1946669137 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e0814f010 0x556e081671a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.559+0000 7f9078c26740 1 --2- 172.21.15.32:0/1946669137 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556e0814cc70 0x556e08166c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.559+0000 7f9078c26740 1 --2- 172.21.15.32:0/1946669137 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e08145980 0x556e0813f7d0 unknown :-1 s=CLOSED pgs=579 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.559+0000 7f9078c26740 1 -- 172.21.15.32:0/1946669137 >> 172.21.15.32:0/1946669137 conn(0x556e0810c950 msgr2=0x556e0813f0c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:28.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.559+0000 7f9078c26740 1 -- 172.21.15.32:0/1946669137 shutdown_connections 2024-04-25T00:34:28.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.559+0000 7f9078c26740 1 -- 172.21.15.32:0/1946669137 wait complete. 2024-04-25T00:34:28.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.560+0000 7f9078c26740 1 -- start start 2024-04-25T00:34:28.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.560+0000 7f9078c26740 1 Processor -- start 2024-04-25T00:34:28.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.560+0000 7f9078c26740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e0814cc70 0x556e08157770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.560+0000 7f9078c26740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e0814f010 0x556e08149610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.560+0000 7f9078c26740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556e08149cb0 0x556e0814fcc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.560+0000 7f9078c26740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f90700715d0 con 0x556e0814cc70 2024-04-25T00:34:28.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.560+0000 7f9078c26740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f9070072a40 con 0x556e08149cb0 2024-04-25T00:34:28.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.560+0000 7f9078c26740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f9070071ca0 con 0x556e0814f010 2024-04-25T00:34:28.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.560+0000 7f9078c26740 10 client.0 _ll_register_callbacks cb 0x7f906400c9d0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:34:28.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.560+0000 7f906effd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e0814cc70 0x556e08157770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.561+0000 7f906effd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e0814cc70 0x556e08157770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39990/0 (socket says 172.21.15.32:39990) 2024-04-25T00:34:28.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.561+0000 7f906effd640 1 -- 172.21.15.32:0/2010861489 learned_addr learned my addr 172.21.15.32:0/2010861489 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:28.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.561+0000 7f906f7fe640 1 --2- 172.21.15.32:0/2010861489 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e0814f010 0x556e08149610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.561+0000 7f906effd640 1 -- 172.21.15.32:0/2010861489 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e0814f010 msgr2=0x556e08149610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:28.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.561+0000 7f906effd640 1 --2- 172.21.15.32:0/2010861489 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e0814f010 0x556e08149610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.561+0000 7f906effd640 1 -- 172.21.15.32:0/2010861489 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556e08149cb0 msgr2=0x556e0814fcc0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-25T00:34:28.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.561+0000 7f906effd640 1 --2- 172.21.15.32:0/2010861489 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556e08149cb0 0x556e0814fcc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.561+0000 7f906effd640 1 -- 172.21.15.32:0/2010861489 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x556e08150200 con 0x556e0814cc70 2024-04-25T00:34:28.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.561+0000 7f906f7fe640 1 --2- 172.21.15.32:0/2010861489 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e0814f010 0x556e08149610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:34:28.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.561+0000 7f906effd640 1 --2- 172.21.15.32:0/2010861489 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e0814cc70 0x556e08157770 secure :-1 s=READY pgs=580 cs=0 l=1 rev1=1 crypto rx=0x7f9064025040 tx=0x7f906400fb20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8da6edbd5deb0e1f server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:28.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.561+0000 7f904effd640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:34:28.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.561+0000 7f904effd640 1 -- 172.21.15.32:0/2010861489 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f906403e070 con 0x556e0814cc70 2024-04-25T00:34:28.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.561+0000 7f904effd640 1 -- 172.21.15.32:0/2010861489 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f90640252d0 con 0x556e0814cc70 2024-04-25T00:34:28.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.561+0000 7f9078c26740 1 -- 172.21.15.32:0/2010861489 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f9064025e60 con 0x556e0814cc70 2024-04-25T00:34:28.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.562+0000 7f904effd640 1 -- 172.21.15.32:0/2010861489 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f906404c050 con 0x556e0814cc70 2024-04-25T00:34:28.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.562+0000 7f9078c26740 10 client.15951 fetch_fsmap learned FSMap version 25 2024-04-25T00:34:28.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.562+0000 7f9078c26740 1 -- 172.21.15.32:0/2010861489 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x556e081c5c10 con 0x556e0814cc70 2024-04-25T00:34:28.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.562+0000 7f904effd640 1 -- 172.21.15.32:0/2010861489 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9064025e60 con 0x556e0814cc70 2024-04-25T00:34:28.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.562+0000 7f904effd640 1 -- 172.21.15.32:0/2010861489 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7f9064030320 con 0x556e0814cc70 2024-04-25T00:34:28.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.562+0000 7f9078c26740 10 client.15951 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:34:28.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.562+0000 7f9078c26740 10 client.15951 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:34:28.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.562+0000 7f9078c26740 20 client.15951 populate_metadata read hostname 'smithi032' 2024-04-25T00:34:28.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.562+0000 7f9078c26740 1 --2- 172.21.15.32:0/2010861489 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x556e081c61e0 0x556e081c85d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.562+0000 7f9078c26740 4 client.15951 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-25T00:34:28.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.562+0000 7f9078c26740 1 -- 172.21.15.32:0/2010861489 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) -- 0x556e08140d50 con 0x556e081c61e0 2024-04-25T00:34:28.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.562+0000 7f906f7fe640 1 --2- 172.21.15.32:0/2010861489 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x556e081c61e0 0x556e081c85d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.563+0000 7f906f7fe640 1 --2- 172.21.15.32:0/2010861489 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x556e081c61e0 0x556e081c85d0 crc :-1 s=READY pgs=58 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1adc7be017200d9a server_cookie=fe5b1d55a65d53e7 in_seq=0 out_seq=0 2024-04-25T00:34:28.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.563+0000 7f904effd640 10 client.15951 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:34:28.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.564+0000 7f904effd640 1 -- 172.21.15.32:0/2010861489 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+110 (crc 0 0 0) 0x556e08140d50 con 0x556e081c61e0 2024-04-25T00:34:28.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.564+0000 7f904effd640 10 client.15951 handle_command_reply: tid=0 2024-04-25T00:34:28.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.564+0000 7f9078c26740 1 client.15951 shutdown 2024-04-25T00:34:28.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.564+0000 7f9078c26740 1 -- 172.21.15.32:0/2010861489 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e0814cc70 msgr2=0x556e08157770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:28.569 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.564+0000 7f9078c26740 1 --2- 172.21.15.32:0/2010861489 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e0814cc70 0x556e08157770 secure :-1 s=READY pgs=580 cs=0 l=1 rev1=1 crypto rx=0x7f9064025040 tx=0x7f906400fb20 comp rx=0 tx=0).stop 2024-04-25T00:34:28.569 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.565+0000 7f9078c26740 1 -- 172.21.15.32:0/2010861489 shutdown_connections 2024-04-25T00:34:28.569 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.565+0000 7f9078c26740 1 --2- 172.21.15.32:0/2010861489 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x556e081c61e0 0x556e081c85d0 crc :-1 s=READY pgs=58 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.569 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.565+0000 7f9078c26740 1 --2- 172.21.15.32:0/2010861489 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556e08149cb0 0x556e0814fcc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.569 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.565+0000 7f904effd640 0 client.15951 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:34:28.569 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.565+0000 7f9078c26740 1 --2- 172.21.15.32:0/2010861489 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e0814f010 0x556e08149610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.569 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.565+0000 7f9078c26740 1 --2- 172.21.15.32:0/2010861489 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556e0814cc70 0x556e08157770 unknown :-1 s=CLOSED pgs=580 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.569 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.565+0000 7f9078c26740 1 -- 172.21.15.32:0/2010861489 >> 172.21.15.32:0/2010861489 conn(0x556e0810c950 msgr2=0x556e0813f950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:28.569 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.565+0000 7f9078c26740 1 -- 172.21.15.32:0/2010861489 shutdown_connections 2024-04-25T00:34:28.569 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.565+0000 7f9078c26740 1 -- 172.21.15.32:0/2010861489 wait complete. 2024-04-25T00:34:28.569 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.565+0000 7f9078c26740 20 client.15951 trim_cache size 0 max 16384 2024-04-25T00:34:28.569 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:34:28.570 INFO:teuthology.orchestra.run.smithi032.stdout: "return_code": 0, 2024-04-25T00:34:28.570 INFO:teuthology.orchestra.run.smithi032.stdout: "scrub_tag": "8cf2bbe8-8d6d-43b8-9cc6-1e35e7ad526b", 2024-04-25T00:34:28.570 INFO:teuthology.orchestra.run.smithi032.stdout: "mode": "asynchronous" 2024-04-25T00:34:28.570 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:34:28.571 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.565+0000 7f9075e38640 1 -- 172.21.15.32:0/1574405761 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f90580a2ea0 msgr2=0x7f90580a5360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:28.571 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.565+0000 7f9075e38640 1 --2- 172.21.15.32:0/1574405761 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f90580a2ea0 0x7f90580a5360 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f90680cd510 tx=0x7f90680d0540 comp rx=0 tx=0).stop 2024-04-25T00:34:28.571 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.566+0000 7f9075e38640 1 -- 172.21.15.32:0/1574405761 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f907010bd70 msgr2=0x7f907019c8c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:28.571 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.566+0000 7f9075e38640 1 --2- 172.21.15.32:0/1574405761 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f907010bd70 0x7f907019c8c0 secure :-1 s=READY pgs=577 cs=0 l=1 rev1=1 crypto rx=0x7f906800eb40 tx=0x7f906800c2d0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.571 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.566+0000 7f9075e38640 1 -- 172.21.15.32:0/1574405761 shutdown_connections 2024-04-25T00:34:28.571 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.566+0000 7f9075e38640 1 --2- 172.21.15.32:0/1574405761 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f90580a2ea0 0x7f90580a5360 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.571 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.566+0000 7f9075e38640 1 --2- 172.21.15.32:0/1574405761 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f907010bd70 0x7f907019c8c0 unknown :-1 s=CLOSED pgs=577 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.571 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.566+0000 7f9075e38640 1 --2- 172.21.15.32:0/1574405761 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f90701032c0 0x7f9070198540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.571 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.566+0000 7f9075e38640 1 --2- 172.21.15.32:0/1574405761 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f90701029a0 0x7f9070198000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.572 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.566+0000 7f9075e38640 1 -- 172.21.15.32:0/1574405761 >> 172.21.15.32:0/1574405761 conn(0x7f90700769c0 msgr2=0x7f90700fd560 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:28.572 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.566+0000 7f9075e38640 1 -- 172.21.15.32:0/1574405761 shutdown_connections 2024-04-25T00:34:28.572 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.566+0000 7f9075e38640 1 -- 172.21.15.32:0/1574405761 wait complete. 2024-04-25T00:34:28.578 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-25T00:34:28.578 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-25T00:34:28.683 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.682+0000 7f823c87d640 1 -- 172.21.15.32:0/1797466151 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8238073af0 msgr2=0x7f8238073ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:28.683 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.682+0000 7f823c87d640 1 --2- 172.21.15.32:0/1797466151 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8238073af0 0x7f8238073ed0 secure :-1 s=READY pgs=581 cs=0 l=1 rev1=1 crypto rx=0x7f8228009900 tx=0x7f822802a480 comp rx=0 tx=0).stop 2024-04-25T00:34:28.683 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.682+0000 7f823c87d640 1 -- 172.21.15.32:0/1797466151 shutdown_connections 2024-04-25T00:34:28.684 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.682+0000 7f823c87d640 1 --2- 172.21.15.32:0/1797466151 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f823810dfd0 0x7f82381103c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.684 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.682+0000 7f823c87d640 1 --2- 172.21.15.32:0/1797466151 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8238074410 0x7f823810da90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.684 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.682+0000 7f823c87d640 1 --2- 172.21.15.32:0/1797466151 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8238073af0 0x7f8238073ed0 unknown :-1 s=CLOSED pgs=581 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.684 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.682+0000 7f823c87d640 1 -- 172.21.15.32:0/1797466151 >> 172.21.15.32:0/1797466151 conn(0x7f82380fd180 msgr2=0x7f82380ff5a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:28.684 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.682+0000 7f823c87d640 1 -- 172.21.15.32:0/1797466151 shutdown_connections 2024-04-25T00:34:28.684 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.682+0000 7f823c87d640 1 -- 172.21.15.32:0/1797466151 wait complete. 2024-04-25T00:34:28.684 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.683+0000 7f823c87d640 1 Processor -- start 2024-04-25T00:34:28.684 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.683+0000 7f823c87d640 1 -- start start 2024-04-25T00:34:28.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.683+0000 7f823c87d640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8238073af0 0x7f823819a230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.683+0000 7f823c87d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8238074410 0x7f823819a770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.683+0000 7f823c87d640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f823810dfd0 0x7f823819eaf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.683+0000 7f823c87d640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f8238113cf0 con 0x7f8238074410 2024-04-25T00:34:28.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.683+0000 7f823c87d640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f8238113b70 con 0x7f8238073af0 2024-04-25T00:34:28.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.683+0000 7f823c87d640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f8238113e70 con 0x7f823810dfd0 2024-04-25T00:34:28.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.683+0000 7f8235d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8238074410 0x7f823819a770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.683+0000 7f8235d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8238074410 0x7f823819a770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40002/0 (socket says 172.21.15.32:40002) 2024-04-25T00:34:28.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.683+0000 7f8235d9b640 1 -- 172.21.15.32:0/1823992528 learned_addr learned my addr 172.21.15.32:0/1823992528 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:28.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.684+0000 7f823659c640 1 --2- 172.21.15.32:0/1823992528 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8238073af0 0x7f823819a230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.684+0000 7f8235d9b640 1 -- 172.21.15.32:0/1823992528 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f823810dfd0 msgr2=0x7f823819eaf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:28.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.684+0000 7f8235d9b640 1 --2- 172.21.15.32:0/1823992528 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f823810dfd0 0x7f823819eaf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.684+0000 7f8235d9b640 1 -- 172.21.15.32:0/1823992528 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8238073af0 msgr2=0x7f823819a230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:28.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.684+0000 7f8235d9b640 1 --2- 172.21.15.32:0/1823992528 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8238073af0 0x7f823819a230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.684+0000 7f8235d9b640 1 -- 172.21.15.32:0/1823992528 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f823819f220 con 0x7f8238074410 2024-04-25T00:34:28.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.684+0000 7f823659c640 1 --2- 172.21.15.32:0/1823992528 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8238073af0 0x7f823819a230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:34:28.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.684+0000 7f8235d9b640 1 --2- 172.21.15.32:0/1823992528 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8238074410 0x7f823819a770 secure :-1 s=READY pgs=582 cs=0 l=1 rev1=1 crypto rx=0x7f822c00d950 tx=0x7f822c00de10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:28.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.684+0000 7f82277fe640 1 -- 172.21.15.32:0/1823992528 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f822c017070 con 0x7f8238074410 2024-04-25T00:34:28.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.684+0000 7f823c87d640 1 -- 172.21.15.32:0/1823992528 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f823819f510 con 0x7f8238074410 2024-04-25T00:34:28.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.684+0000 7f823c87d640 1 -- 172.21.15.32:0/1823992528 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f82381a6da0 con 0x7f8238074410 2024-04-25T00:34:28.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.684+0000 7f82277fe640 1 -- 172.21.15.32:0/1823992528 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f822c009c30 con 0x7f8238074410 2024-04-25T00:34:28.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.685+0000 7f82277fe640 1 -- 172.21.15.32:0/1823992528 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f822c00f6b0 con 0x7f8238074410 2024-04-25T00:34:28.689 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.687+0000 7f82277fe640 1 -- 172.21.15.32:0/1823992528 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 25) ==== 138066+0+0 (secure 0 0 0) 0x7f822c0109d0 con 0x7f8238074410 2024-04-25T00:34:28.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.687+0000 7f823f58c740 1 Processor -- start 2024-04-25T00:34:28.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.687+0000 7f823f58c740 1 -- start start 2024-04-25T00:34:28.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.687+0000 7f823f58c740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x562e3714f630 0x562e3714fa10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.687+0000 7f823f58c740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562e3714ff50 0x562e3715a0e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.687+0000 7f823f58c740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3715a620 0x562e3715ca10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.687+0000 7f823f58c740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x562e370b46b0 con 0x562e3715a620 2024-04-25T00:34:28.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.687+0000 7f823f58c740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x562e3705c880 con 0x562e3714f630 2024-04-25T00:34:28.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.687+0000 7f823f58c740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x562e370b0670 con 0x562e3714ff50 2024-04-25T00:34:28.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.688+0000 7f8235d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3715a620 0x562e3715ca10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.688+0000 7f8235d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3715a620 0x562e3715ca10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40016/0 (socket says 172.21.15.32:40016) 2024-04-25T00:34:28.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.688+0000 7f8235d9b640 1 -- 172.21.15.32:0/1668128043 learned_addr learned my addr 172.21.15.32:0/1668128043 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:28.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.688+0000 7f8236d9d640 1 --2- 172.21.15.32:0/1668128043 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x562e3714f630 0x562e3714fa10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.691 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.688+0000 7f823659c640 1 --2- 172.21.15.32:0/1668128043 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562e3714ff50 0x562e3715a0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.691 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.688+0000 7f82277fe640 1 --2- 172.21.15.32:0/1823992528 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f82200a2e20 0x7f82200a52e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.691 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.688+0000 7f8235d9b640 1 -- 172.21.15.32:0/1668128043 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562e3714ff50 msgr2=0x562e3715a0e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:28.691 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.688+0000 7f8235d9b640 1 --2- 172.21.15.32:0/1668128043 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562e3714ff50 0x562e3715a0e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.691 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.688+0000 7f8235d9b640 1 -- 172.21.15.32:0/1668128043 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x562e3714f630 msgr2=0x562e3714fa10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:28.691 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.688+0000 7f8235d9b640 1 --2- 172.21.15.32:0/1668128043 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x562e3714f630 0x562e3714fa10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.691 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.688+0000 7f8235d9b640 1 -- 172.21.15.32:0/1668128043 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562e36e927d0 con 0x562e3715a620 2024-04-25T00:34:28.691 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.688+0000 7f8236d9d640 1 --2- 172.21.15.32:0/1668128043 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x562e3714f630 0x562e3714fa10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:34:28.691 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.688+0000 7f8236d9d640 1 --2- 172.21.15.32:0/1823992528 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f82200a2e20 0x7f82200a52e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.691 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.688+0000 7f82277fe640 1 -- 172.21.15.32:0/1823992528 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f822c0cfaf0 con 0x7f8238074410 2024-04-25T00:34:28.691 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.689+0000 7f8235d9b640 1 --2- 172.21.15.32:0/1668128043 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3715a620 0x562e3715ca10 secure :-1 s=READY pgs=583 cs=0 l=1 rev1=1 crypto rx=0x7f822c0cce20 tx=0x7f822c023000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=22abbf27e87af79 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:28.691 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.689+0000 7f8224ff9640 1 -- 172.21.15.32:0/1668128043 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f822c09b370 con 0x562e3715a620 2024-04-25T00:34:28.692 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.689+0000 7f8224ff9640 1 -- 172.21.15.32:0/1668128043 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f822c009de0 con 0x562e3715a620 2024-04-25T00:34:28.692 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.689+0000 7f8224ff9640 1 -- 172.21.15.32:0/1668128043 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f822c09ba10 con 0x562e3715a620 2024-04-25T00:34:28.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.690+0000 7f823f58c740 1 -- 172.21.15.32:0/1668128043 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3715a620 msgr2=0x562e3715ca10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:28.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.690+0000 7f8236d9d640 1 --2- 172.21.15.32:0/1823992528 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f82200a2e20 0x7f82200a52e0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f8230005c40 tx=0x7f8230004a40 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:28.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.690+0000 7f823f58c740 1 --2- 172.21.15.32:0/1668128043 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3715a620 0x562e3715ca10 secure :-1 s=READY pgs=583 cs=0 l=1 rev1=1 crypto rx=0x7f822c0cce20 tx=0x7f822c023000 comp rx=0 tx=0).stop 2024-04-25T00:34:28.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.690+0000 7f823f58c740 1 -- 172.21.15.32:0/1668128043 shutdown_connections 2024-04-25T00:34:28.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.690+0000 7f823f58c740 1 --2- 172.21.15.32:0/1668128043 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3715a620 0x562e3715ca10 unknown :-1 s=CLOSED pgs=583 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.690+0000 7f823f58c740 1 --2- 172.21.15.32:0/1668128043 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562e3714ff50 0x562e3715a0e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.690+0000 7f823f58c740 1 --2- 172.21.15.32:0/1668128043 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x562e3714f630 0x562e3714fa10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.690+0000 7f823f58c740 1 -- 172.21.15.32:0/1668128043 >> 172.21.15.32:0/1668128043 conn(0x562e3714b950 msgr2=0x562e3714bd30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:28.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.690+0000 7f823f58c740 1 -- 172.21.15.32:0/1668128043 shutdown_connections 2024-04-25T00:34:28.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.690+0000 7f823f58c740 1 -- 172.21.15.32:0/1668128043 wait complete. 2024-04-25T00:34:28.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.690+0000 7f823f58c740 1 -- start start 2024-04-25T00:34:28.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.690+0000 7f823f58c740 1 Processor -- start 2024-04-25T00:34:28.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.691+0000 7f823f58c740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x562e3714f630 0x562e37157580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.691+0000 7f823f58c740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562e3714ff50 0x562e37157ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.691+0000 7f823f58c740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3715a620 0x562e37158000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.691+0000 7f823f58c740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x562e3715d350 con 0x562e3715a620 2024-04-25T00:34:28.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.691+0000 7f823f58c740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x562e3706ff50 con 0x562e3714f630 2024-04-25T00:34:28.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.691+0000 7f823f58c740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x562e3715d4d0 con 0x562e3714ff50 2024-04-25T00:34:28.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.691+0000 7f8236d9d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3715a620 0x562e37158000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.691+0000 7f8236d9d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3715a620 0x562e37158000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40018/0 (socket says 172.21.15.32:40018) 2024-04-25T00:34:28.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.691+0000 7f8236d9d640 1 -- 172.21.15.32:0/1846735677 learned_addr learned my addr 172.21.15.32:0/1846735677 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:28.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.691+0000 7f823f58c740 10 client.0 _ll_register_callbacks cb 0x562e370ab000 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:34:28.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.691+0000 7f823659c640 1 --2- 172.21.15.32:0/1846735677 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x562e3714f630 0x562e37157580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.691+0000 7f8235d9b640 1 --2- 172.21.15.32:0/1846735677 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562e3714ff50 0x562e37157ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.691+0000 7f8236d9d640 1 -- 172.21.15.32:0/1846735677 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562e3714ff50 msgr2=0x562e37157ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:28.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.691+0000 7f8236d9d640 1 --2- 172.21.15.32:0/1846735677 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562e3714ff50 0x562e37157ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.691+0000 7f8236d9d640 1 -- 172.21.15.32:0/1846735677 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x562e3714f630 msgr2=0x562e37157580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:28.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.691+0000 7f8236d9d640 1 --2- 172.21.15.32:0/1846735677 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x562e3714f630 0x562e37157580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.691+0000 7f8236d9d640 1 -- 172.21.15.32:0/1846735677 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562e36dc83f0 con 0x562e3715a620 2024-04-25T00:34:28.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.691+0000 7f8235d9b640 1 --2- 172.21.15.32:0/1846735677 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562e3714ff50 0x562e37157ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:34:28.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.691+0000 7f823659c640 1 --2- 172.21.15.32:0/1846735677 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x562e3714f630 0x562e37157580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:34:28.698 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.692+0000 7f8236d9d640 1 --2- 172.21.15.32:0/1846735677 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3715a620 0x562e37158000 secure :-1 s=READY pgs=584 cs=0 l=1 rev1=1 crypto rx=0x7f823001d7d0 tx=0x7f8230005bd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=93f5682c594145ad server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:28.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.692+0000 7f821dffb640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:34:28.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.692+0000 7f821dffb640 1 -- 172.21.15.32:0/1846735677 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8230013d90 con 0x562e3715a620 2024-04-25T00:34:28.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.692+0000 7f821dffb640 1 -- 172.21.15.32:0/1846735677 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f823001a410 con 0x562e3715a620 2024-04-25T00:34:28.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.692+0000 7f823f58c740 1 -- 172.21.15.32:0/1846735677 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f822c009de0 con 0x562e3715a620 2024-04-25T00:34:28.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.692+0000 7f821dffb640 1 -- 172.21.15.32:0/1846735677 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f823001a610 con 0x562e3715a620 2024-04-25T00:34:28.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.692+0000 7f823f58c740 10 client.15963 fetch_fsmap learned FSMap version 25 2024-04-25T00:34:28.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.692+0000 7f823f58c740 1 -- 172.21.15.32:0/1846735677 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x562e36d880f0 con 0x562e3715a620 2024-04-25T00:34:28.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.692+0000 7f821dffb640 1 -- 172.21.15.32:0/1846735677 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f822c009de0 con 0x562e3715a620 2024-04-25T00:34:28.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.692+0000 7f821dffb640 1 -- 172.21.15.32:0/1846735677 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7f8230024070 con 0x562e3715a620 2024-04-25T00:34:28.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.692+0000 7f823f58c740 10 client.15963 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:34:28.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.692+0000 7f823f58c740 10 client.15963 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:34:28.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.692+0000 7f823f58c740 20 client.15963 populate_metadata read hostname 'smithi032' 2024-04-25T00:34:28.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.692+0000 7f823f58c740 1 --2- 172.21.15.32:0/1846735677 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x562e3716a8f0 0x562e3716cd20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.693+0000 7f823f58c740 4 client.15963 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:34:28.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.693+0000 7f823f58c740 1 -- 172.21.15.32:0/1846735677 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x562e370c1bf0 con 0x562e3716a8f0 2024-04-25T00:34:28.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.693+0000 7f823659c640 1 --2- 172.21.15.32:0/1846735677 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x562e3716a8f0 0x562e3716cd20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.693+0000 7f823659c640 1 --2- 172.21.15.32:0/1846735677 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x562e3716a8f0 0x562e3716cd20 crc :-1 s=READY pgs=59 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=816168ccbecb40de server_cookie=c04804146eeabf80 in_seq=0 out_seq=0 2024-04-25T00:34:28.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.693+0000 7f821dffb640 10 client.15963 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:34:28.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.694+0000 7f821dffb640 1 -- 172.21.15.32:0/1846735677 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x562e370c1bf0 con 0x562e3716a8f0 2024-04-25T00:34:28.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.694+0000 7f821dffb640 10 client.15963 handle_command_reply: tid=0 2024-04-25T00:34:28.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.694+0000 7f823f58c740 1 client.15963 shutdown 2024-04-25T00:34:28.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.694+0000 7f823f58c740 1 -- 172.21.15.32:0/1846735677 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3715a620 msgr2=0x562e37158000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:28.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.694+0000 7f823f58c740 1 --2- 172.21.15.32:0/1846735677 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3715a620 0x562e37158000 secure :-1 s=READY pgs=584 cs=0 l=1 rev1=1 crypto rx=0x7f823001d7d0 tx=0x7f8230005bd0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.694+0000 7f823f58c740 1 -- 172.21.15.32:0/1846735677 shutdown_connections 2024-04-25T00:34:28.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.694+0000 7f823f58c740 1 --2- 172.21.15.32:0/1846735677 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x562e3716a8f0 0x562e3716cd20 crc :-1 s=READY pgs=59 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.694+0000 7f823f58c740 1 --2- 172.21.15.32:0/1846735677 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3715a620 0x562e37158000 unknown :-1 s=CLOSED pgs=584 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.694+0000 7f823f58c740 1 --2- 172.21.15.32:0/1846735677 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562e3714ff50 0x562e37157ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.694+0000 7f821dffb640 0 client.15963 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:34:28.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.694+0000 7f823f58c740 1 --2- 172.21.15.32:0/1846735677 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x562e3714f630 0x562e37157580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.694+0000 7f823f58c740 1 -- 172.21.15.32:0/1846735677 >> 172.21.15.32:0/1846735677 conn(0x562e3714b950 msgr2=0x562e37151750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:28.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.694+0000 7f823f58c740 1 -- 172.21.15.32:0/1846735677 shutdown_connections 2024-04-25T00:34:28.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.694+0000 7f823f58c740 1 -- 172.21.15.32:0/1846735677 wait complete. 2024-04-25T00:34:28.701 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.694+0000 7f823f58c740 20 client.15963 trim_cache size 0 max 16384 2024-04-25T00:34:28.714 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.712+0000 7f823f58c740 1 Processor -- start 2024-04-25T00:34:28.714 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.712+0000 7f823f58c740 1 -- start start 2024-04-25T00:34:28.714 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.712+0000 7f823f58c740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x562e3714b950 0x562e37166240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.714 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.712+0000 7f823f58c740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3714f630 0x562e37151dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.714 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.712+0000 7f823f58c740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562e3714ff50 0x562e371725f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.714 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.712+0000 7f823f58c740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f82381b38e0 con 0x562e3714f630 2024-04-25T00:34:28.714 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.713+0000 7f823f58c740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f82381b11b0 con 0x562e3714b950 2024-04-25T00:34:28.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.713+0000 7f823f58c740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f82381af000 con 0x562e3714ff50 2024-04-25T00:34:28.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.713+0000 7f8235d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3714f630 0x562e37151dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.713+0000 7f8235d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3714f630 0x562e37151dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:40032/0 (socket says 172.21.15.32:40032) 2024-04-25T00:34:28.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.713+0000 7f8235d9b640 1 -- 172.21.15.32:0/3303065394 learned_addr learned my addr 172.21.15.32:0/3303065394 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:28.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.713+0000 7f823659c640 1 --2- 172.21.15.32:0/3303065394 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x562e3714b950 0x562e37166240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.713+0000 7f8236d9d640 1 --2- 172.21.15.32:0/3303065394 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562e3714ff50 0x562e371725f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.713+0000 7f8235d9b640 1 -- 172.21.15.32:0/3303065394 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562e3714ff50 msgr2=0x562e371725f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:28.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.713+0000 7f8235d9b640 1 --2- 172.21.15.32:0/3303065394 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562e3714ff50 0x562e371725f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.713+0000 7f8235d9b640 1 -- 172.21.15.32:0/3303065394 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x562e3714b950 msgr2=0x562e37166240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:28.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.713+0000 7f8235d9b640 1 --2- 172.21.15.32:0/3303065394 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x562e3714b950 0x562e37166240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.713+0000 7f8235d9b640 1 -- 172.21.15.32:0/3303065394 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562e3713d090 con 0x562e3714f630 2024-04-25T00:34:28.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.713+0000 7f8235d9b640 1 --2- 172.21.15.32:0/3303065394 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3714f630 0x562e37151dd0 secure :-1 s=READY pgs=585 cs=0 l=1 rev1=1 crypto rx=0x7f822c000c00 tx=0x7f822c010d70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d19b3aa385e40541 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:28.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.713+0000 7f821ffff640 1 -- 172.21.15.32:0/3303065394 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f822c009de0 con 0x562e3714f630 2024-04-25T00:34:28.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.713+0000 7f821ffff640 1 -- 172.21.15.32:0/3303065394 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f822c0ccc50 con 0x562e3714f630 2024-04-25T00:34:28.716 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.713+0000 7f821ffff640 1 -- 172.21.15.32:0/3303065394 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f822c099d10 con 0x562e3714f630 2024-04-25T00:34:28.721 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.714+0000 7f823f58c740 1 -- 172.21.15.32:0/3303065394 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3714f630 msgr2=0x562e37151dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:28.721 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.714+0000 7f823f58c740 1 --2- 172.21.15.32:0/3303065394 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3714f630 0x562e37151dd0 secure :-1 s=READY pgs=585 cs=0 l=1 rev1=1 crypto rx=0x7f822c000c00 tx=0x7f822c010d70 comp rx=0 tx=0).stop 2024-04-25T00:34:28.721 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.714+0000 7f823f58c740 1 -- 172.21.15.32:0/3303065394 shutdown_connections 2024-04-25T00:34:28.721 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.714+0000 7f823f58c740 1 --2- 172.21.15.32:0/3303065394 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562e3714ff50 0x562e371725f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.721 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.714+0000 7f823f58c740 1 --2- 172.21.15.32:0/3303065394 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3714f630 0x562e37151dd0 unknown :-1 s=CLOSED pgs=585 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.721 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.714+0000 7f823f58c740 1 --2- 172.21.15.32:0/3303065394 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x562e3714b950 0x562e37166240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.721 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.714+0000 7f823f58c740 1 -- 172.21.15.32:0/3303065394 >> 172.21.15.32:0/3303065394 conn(0x562e371561c0 msgr2=0x562e3714aaf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:28.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.714+0000 7f823f58c740 1 -- 172.21.15.32:0/3303065394 shutdown_connections 2024-04-25T00:34:28.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.714+0000 7f823f58c740 1 -- 172.21.15.32:0/3303065394 wait complete. 2024-04-25T00:34:28.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.715+0000 7f823f58c740 1 -- start start 2024-04-25T00:34:28.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.715+0000 7f823f58c740 1 Processor -- start 2024-04-25T00:34:28.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.715+0000 7f823f58c740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x562e3714b950 0x562e3715e490 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.715+0000 7f823f58c740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3714ff50 0x562e3715e9d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.715+0000 7f823f58c740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562e3715a620 0x562e371cfe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.715+0000 7f823f58c740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f82381af670 con 0x562e3714ff50 2024-04-25T00:34:28.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.715+0000 7f823f58c740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f82381b0ae0 con 0x562e3714b950 2024-04-25T00:34:28.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.715+0000 7f823f58c740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f82381afd40 con 0x562e3715a620 2024-04-25T00:34:28.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.715+0000 7f8235d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3714ff50 0x562e3715e9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.715+0000 7f823659c640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x562e3714b950 0x562e3715e490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.715+0000 7f823659c640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x562e3714b950 0x562e3715e490 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:48226/0 (socket says 172.21.15.32:48226) 2024-04-25T00:34:28.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.715+0000 7f823659c640 1 -- 172.21.15.32:0/1521666446 learned_addr learned my addr 172.21.15.32:0/1521666446 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:28.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.715+0000 7f823f58c740 10 client.0 _ll_register_callbacks cb 0x7f823001d450 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:34:28.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.715+0000 7f8236d9d640 1 --2- 172.21.15.32:0/1521666446 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562e3715a620 0x562e371cfe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.716+0000 7f8235d9b640 1 -- 172.21.15.32:0/1521666446 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562e3715a620 msgr2=0x562e371cfe60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:28.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.716+0000 7f8235d9b640 1 --2- 172.21.15.32:0/1521666446 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562e3715a620 0x562e371cfe60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.716+0000 7f8235d9b640 1 -- 172.21.15.32:0/1521666446 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x562e3714b950 msgr2=0x562e3715e490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:28.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.716+0000 7f8235d9b640 1 --2- 172.21.15.32:0/1521666446 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x562e3714b950 0x562e3715e490 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.716+0000 7f8235d9b640 1 -- 172.21.15.32:0/1521666446 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562e371d03a0 con 0x562e3714ff50 2024-04-25T00:34:28.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.716+0000 7f823659c640 1 --2- 172.21.15.32:0/1521666446 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x562e3714b950 0x562e3715e490 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:34:28.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.716+0000 7f8235d9b640 1 --2- 172.21.15.32:0/1521666446 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3714ff50 0x562e3715e9d0 secure :-1 s=READY pgs=586 cs=0 l=1 rev1=1 crypto rx=0x562e371475f0 tx=0x7f822c005b20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6d117f007b0dafc4 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:28.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.716+0000 7f821dffb640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:34:28.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.716+0000 7f821dffb640 1 -- 172.21.15.32:0/1521666446 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f822c014020 con 0x562e3714ff50 2024-04-25T00:34:28.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.716+0000 7f821dffb640 1 -- 172.21.15.32:0/1521666446 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f822c0afb70 con 0x562e3714ff50 2024-04-25T00:34:28.723 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.716+0000 7f823f58c740 1 -- 172.21.15.32:0/1521666446 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f822c0ccc50 con 0x562e3714ff50 2024-04-25T00:34:28.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.716+0000 7f821dffb640 1 -- 172.21.15.32:0/1521666446 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f822c020070 con 0x562e3714ff50 2024-04-25T00:34:28.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.716+0000 7f823f58c740 10 client.15969 fetch_fsmap learned FSMap version 25 2024-04-25T00:34:28.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.716+0000 7f823f58c740 1 -- 172.21.15.32:0/1521666446 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x562e371d1530 con 0x562e3714ff50 2024-04-25T00:34:28.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.717+0000 7f821dffb640 1 -- 172.21.15.32:0/1521666446 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f822c0ccc50 con 0x562e3714ff50 2024-04-25T00:34:28.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.717+0000 7f821dffb640 1 -- 172.21.15.32:0/1521666446 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 25) ==== 8346+0+0 (secure 0 0 0) 0x7f822c098080 con 0x562e3714ff50 2024-04-25T00:34:28.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.717+0000 7f823f58c740 10 client.15969 fetch_fsmap finished waiting for FSMap version 25 2024-04-25T00:34:28.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.717+0000 7f823f58c740 10 client.15969 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:34:28.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.717+0000 7f823f58c740 20 client.15969 populate_metadata read hostname 'smithi032' 2024-04-25T00:34:28.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.717+0000 7f823f58c740 1 --2- 172.21.15.32:0/1521666446 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x562e3716a8f0 0x562e371d37c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:28.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.717+0000 7f823f58c740 4 client.15969 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-25T00:34:28.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.717+0000 7f823f58c740 1 -- 172.21.15.32:0/1521666446 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "scrub status"}) -- 0x562e370bbf40 con 0x562e3716a8f0 2024-04-25T00:34:28.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.717+0000 7f823659c640 1 --2- 172.21.15.32:0/1521666446 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x562e3716a8f0 0x562e371d37c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:28.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.718+0000 7f823659c640 1 --2- 172.21.15.32:0/1521666446 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x562e3716a8f0 0x562e371d37c0 crc :-1 s=READY pgs=60 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=340fdf8dabfcc01a server_cookie=fef532d9abea18ad in_seq=0 out_seq=0 2024-04-25T00:34:28.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.718+0000 7f821dffb640 10 client.15969 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:34:28.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.718+0000 7f821dffb640 1 -- 172.21.15.32:0/1521666446 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+259 (crc 0 0 0) 0x562e370bbf40 con 0x562e3716a8f0 2024-04-25T00:34:28.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.718+0000 7f821dffb640 10 client.15969 handle_command_reply: tid=0 2024-04-25T00:34:28.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.718+0000 7f823f58c740 1 client.15969 shutdown 2024-04-25T00:34:28.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.718+0000 7f823f58c740 1 -- 172.21.15.32:0/1521666446 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3714ff50 msgr2=0x562e3715e9d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:28.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.718+0000 7f823f58c740 1 --2- 172.21.15.32:0/1521666446 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3714ff50 0x562e3715e9d0 secure :-1 s=READY pgs=586 cs=0 l=1 rev1=1 crypto rx=0x562e371475f0 tx=0x7f822c005b20 comp rx=0 tx=0).stop 2024-04-25T00:34:28.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.718+0000 7f823f58c740 1 -- 172.21.15.32:0/1521666446 shutdown_connections 2024-04-25T00:34:28.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.718+0000 7f823f58c740 1 --2- 172.21.15.32:0/1521666446 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x562e3716a8f0 0x562e371d37c0 crc :-1 s=READY pgs=60 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.718+0000 7f823f58c740 1 --2- 172.21.15.32:0/1521666446 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562e3715a620 0x562e371cfe60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.718+0000 7f823f58c740 1 --2- 172.21.15.32:0/1521666446 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x562e3714ff50 0x562e3715e9d0 unknown :-1 s=CLOSED pgs=586 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.718+0000 7f823f58c740 1 --2- 172.21.15.32:0/1521666446 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x562e3714b950 0x562e3715e490 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.718+0000 7f823f58c740 1 -- 172.21.15.32:0/1521666446 >> 172.21.15.32:0/1521666446 conn(0x562e371561c0 msgr2=0x562e3714a260 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:28.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.718+0000 7f821dffb640 0 client.15969 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:34:28.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.718+0000 7f823f58c740 1 -- 172.21.15.32:0/1521666446 shutdown_connections 2024-04-25T00:34:28.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.718+0000 7f823f58c740 1 -- 172.21.15.32:0/1521666446 wait complete. 2024-04-25T00:34:28.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.718+0000 7f823f58c740 20 client.15969 trim_cache size 0 max 16384 2024-04-25T00:34:28.726 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:34:28.726 INFO:teuthology.orchestra.run.smithi032.stdout: "status": "IDLE (waiting for more scrubs)", 2024-04-25T00:34:28.726 INFO:teuthology.orchestra.run.smithi032.stdout: "scrubs": { 2024-04-25T00:34:28.726 INFO:teuthology.orchestra.run.smithi032.stdout: "8cf2bbe8-8d6d-43b8-9cc6-1e35e7ad526b": { 2024-04-25T00:34:28.726 INFO:teuthology.orchestra.run.smithi032.stdout: "path": "/", 2024-04-25T00:34:28.726 INFO:teuthology.orchestra.run.smithi032.stdout: "tag": "8cf2bbe8-8d6d-43b8-9cc6-1e35e7ad526b", 2024-04-25T00:34:28.726 INFO:teuthology.orchestra.run.smithi032.stdout: "options": "recursive,force" 2024-04-25T00:34:28.726 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:34:28.727 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:34:28.727 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:34:28.728 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.719+0000 7f823c87d640 1 -- 172.21.15.32:0/1823992528 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f82200a2e20 msgr2=0x7f82200a52e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:28.728 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.719+0000 7f823c87d640 1 --2- 172.21.15.32:0/1823992528 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f82200a2e20 0x7f82200a52e0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f8230005c40 tx=0x7f8230004a40 comp rx=0 tx=0).stop 2024-04-25T00:34:28.728 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.719+0000 7f823c87d640 1 -- 172.21.15.32:0/1823992528 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8238074410 msgr2=0x7f823819a770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:28.728 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.719+0000 7f823c87d640 1 --2- 172.21.15.32:0/1823992528 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8238074410 0x7f823819a770 secure :-1 s=READY pgs=582 cs=0 l=1 rev1=1 crypto rx=0x7f822c00d950 tx=0x7f822c00de10 comp rx=0 tx=0).stop 2024-04-25T00:34:28.728 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.719+0000 7f823c87d640 1 -- 172.21.15.32:0/1823992528 shutdown_connections 2024-04-25T00:34:28.728 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.719+0000 7f823c87d640 1 --2- 172.21.15.32:0/1823992528 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f82200a2e20 0x7f82200a52e0 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.728 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.719+0000 7f823c87d640 1 --2- 172.21.15.32:0/1823992528 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f823810dfd0 0x7f823819eaf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.728 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.719+0000 7f823c87d640 1 --2- 172.21.15.32:0/1823992528 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f8238074410 0x7f823819a770 unknown :-1 s=CLOSED pgs=582 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.728 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.719+0000 7f823c87d640 1 --2- 172.21.15.32:0/1823992528 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f8238073af0 0x7f823819a230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:28.729 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.719+0000 7f823c87d640 1 -- 172.21.15.32:0/1823992528 >> 172.21.15.32:0/1823992528 conn(0x7f82380fd180 msgr2=0x7f82380fde90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:28.729 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.719+0000 7f823c87d640 1 -- 172.21.15.32:0/1823992528 shutdown_connections 2024-04-25T00:34:28.729 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:28.719+0000 7f823c87d640 1 -- 172.21.15.32:0/1823992528 wait complete. 2024-04-25T00:34:28.730 INFO:tasks.cephfs.filesystem:scrub status for tag:8cf2bbe8-8d6d-43b8-9cc6-1e35e7ad526b - {'path': '/', 'tag': '8cf2bbe8-8d6d-43b8-9cc6-1e35e7ad526b', 'options': 'recursive,force'} 2024-04-25T00:34:29.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:29 smithi121 ceph-mon[32231]: pgmap v38: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:34:29.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:29 smithi130 ceph-mon[32525]: pgmap v38: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:34:29.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:29 smithi032 ceph-mon[29357]: pgmap v38: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:34:30.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:30 smithi121 ceph-mon[32231]: scrub queued for path: / 2024-04-25T00:34:30.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:30 smithi121 ceph-mon[32231]: scrub summary: idle+waiting paths [/] 2024-04-25T00:34:30.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:30 smithi121 ceph-mon[32231]: scrub summary: active paths [/] 2024-04-25T00:34:30.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:30 smithi121 ceph-mon[32231]: scrub summary: idle+waiting paths [/] 2024-04-25T00:34:30.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:30 smithi130 ceph-mon[32525]: scrub queued for path: / 2024-04-25T00:34:30.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:30 smithi130 ceph-mon[32525]: scrub summary: idle+waiting paths [/] 2024-04-25T00:34:30.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:30 smithi130 ceph-mon[32525]: scrub summary: active paths [/] 2024-04-25T00:34:30.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:30 smithi130 ceph-mon[32525]: scrub summary: idle+waiting paths [/] 2024-04-25T00:34:30.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:30 smithi032 ceph-mon[29357]: scrub queued for path: / 2024-04-25T00:34:30.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:30 smithi032 ceph-mon[29357]: scrub summary: idle+waiting paths [/] 2024-04-25T00:34:30.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:30 smithi032 ceph-mon[29357]: scrub summary: active paths [/] 2024-04-25T00:34:30.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:30 smithi032 ceph-mon[29357]: scrub summary: idle+waiting paths [/] 2024-04-25T00:34:31.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:31 smithi121 ceph-mon[32231]: pgmap v39: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.9 KiB/s rd, 7 op/s 2024-04-25T00:34:31.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:31 smithi130 ceph-mon[32525]: pgmap v39: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.9 KiB/s rd, 7 op/s 2024-04-25T00:34:31.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:31 smithi032 ceph-mon[29357]: pgmap v39: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.9 KiB/s rd, 7 op/s 2024-04-25T00:34:33.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:33 smithi121 ceph-mon[32231]: pgmap v40: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 5 op/s 2024-04-25T00:34:33.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:33 smithi121 ceph-mon[32231]: scrub summary: idle 2024-04-25T00:34:33.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:33 smithi130 ceph-mon[32525]: pgmap v40: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 5 op/s 2024-04-25T00:34:33.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:33 smithi130 ceph-mon[32525]: scrub summary: idle 2024-04-25T00:34:33.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:33 smithi032 ceph-mon[29357]: pgmap v40: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 5 op/s 2024-04-25T00:34:33.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:33 smithi032 ceph-mon[29357]: scrub summary: idle 2024-04-25T00:34:35.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:35 smithi121 ceph-mon[32231]: pgmap v41: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.9 KiB/s rd, 7 op/s 2024-04-25T00:34:35.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:35 smithi130 ceph-mon[32525]: pgmap v41: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.9 KiB/s rd, 7 op/s 2024-04-25T00:34:35.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:35 smithi032 ceph-mon[29357]: pgmap v41: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 3.9 KiB/s rd, 7 op/s 2024-04-25T00:34:35.537 INFO:tasks.workunit.client.0.smithi032.stderr:Updating files: 97% (12156/12492) Updating files: 98% (12243/12492) Updating files: 99% (12368/12492) Updating files: 100% (12492/12492) Updating files: 100% (12492/12492), done. 2024-04-25T00:34:36.162 INFO:tasks.workunit.client.0.smithi032.stderr:Note: switching to 'd2c45d977142e024a59b68a8f1dce8f0b9840551'. 2024-04-25T00:34:36.162 INFO:tasks.workunit.client.0.smithi032.stderr: 2024-04-25T00:34:36.162 INFO:tasks.workunit.client.0.smithi032.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2024-04-25T00:34:36.162 INFO:tasks.workunit.client.0.smithi032.stderr:changes and commit them, and you can discard any commits you make in this 2024-04-25T00:34:36.162 INFO:tasks.workunit.client.0.smithi032.stderr:state without impacting any branches by switching back to a branch. 2024-04-25T00:34:36.162 INFO:tasks.workunit.client.0.smithi032.stderr: 2024-04-25T00:34:36.162 INFO:tasks.workunit.client.0.smithi032.stderr:If you want to create a new branch to retain commits you create, you may 2024-04-25T00:34:36.162 INFO:tasks.workunit.client.0.smithi032.stderr:do so (now or later) by using -c with the switch command. Example: 2024-04-25T00:34:36.162 INFO:tasks.workunit.client.0.smithi032.stderr: 2024-04-25T00:34:36.162 INFO:tasks.workunit.client.0.smithi032.stderr: git switch -c 2024-04-25T00:34:36.162 INFO:tasks.workunit.client.0.smithi032.stderr: 2024-04-25T00:34:36.163 INFO:tasks.workunit.client.0.smithi032.stderr:Or undo this operation with: 2024-04-25T00:34:36.163 INFO:tasks.workunit.client.0.smithi032.stderr: 2024-04-25T00:34:36.163 INFO:tasks.workunit.client.0.smithi032.stderr: git switch - 2024-04-25T00:34:36.163 INFO:tasks.workunit.client.0.smithi032.stderr: 2024-04-25T00:34:36.163 INFO:tasks.workunit.client.0.smithi032.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2024-04-25T00:34:36.163 INFO:tasks.workunit.client.0.smithi032.stderr: 2024-04-25T00:34:36.163 INFO:tasks.workunit.client.0.smithi032.stderr:HEAD is now at d2c45d97714 Merge pull request #56528 from adk3798/squid-mypy-bump 2024-04-25T00:34:36.166 DEBUG:teuthology.orchestra.run.smithi032:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-04-25T00:34:36.196 INFO:tasks.workunit.client.0.smithi032.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-04-25T00:34:36.198 INFO:tasks.workunit.client.0.smithi032.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-04-25T00:34:36.198 INFO:tasks.workunit.client.0.smithi032.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-04-25T00:34:36.819 INFO:tasks.workunit.client.0.smithi032.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-04-25T00:34:36.876 INFO:tasks.workunit.client.0.smithi032.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-04-25T00:34:36.917 INFO:tasks.workunit.client.0.smithi032.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-04-25T00:34:36.918 INFO:tasks.workunit.client.0.smithi032.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-04-25T00:34:36.918 INFO:tasks.workunit.client.0.smithi032.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-04-25T00:34:36.959 INFO:tasks.workunit.client.0.smithi032.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-04-25T00:34:36.962 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-04-25T00:34:36.963 DEBUG:teuthology.orchestra.run.smithi032:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-04-25T00:34:37.019 INFO:tasks.workunit:Running workunits matching suites/iogen.sh on client.0... 2024-04-25T00:34:37.020 INFO:tasks.workunit:Running workunit suites/iogen.sh... 2024-04-25T00:34:37.020 DEBUG:teuthology.orchestra.run.smithi032:workunit test suites/iogen.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=d2c45d977142e024a59b68a8f1dce8f0b9840551 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/iogen.sh 2024-04-25T00:34:37.080 INFO:tasks.workunit.client.0.smithi032.stdout:getting iogen 2024-04-25T00:34:37.080 INFO:tasks.workunit.client.0.smithi032.stderr:+ echo 'getting iogen' 2024-04-25T00:34:37.080 INFO:tasks.workunit.client.0.smithi032.stderr:+ wget http://download.ceph.com/qa/iogen_3.1p0.tar 2024-04-25T00:34:37.112 INFO:tasks.workunit.client.0.smithi032.stderr:--2024-04-25 00:34:37-- http://download.ceph.com/qa/iogen_3.1p0.tar 2024-04-25T00:34:37.112 INFO:tasks.workunit.client.0.smithi032.stderr:Resolving download.ceph.com (download.ceph.com)... 158.69.68.124, 2607:5300:201:2000::3:58a1 2024-04-25T00:34:37.148 INFO:tasks.workunit.client.0.smithi032.stderr:Connecting to download.ceph.com (download.ceph.com)|158.69.68.124|:80... connected. 2024-04-25T00:34:37.184 INFO:tasks.workunit.client.0.smithi032.stderr:HTTP request sent, awaiting response... 200 OK 2024-04-25T00:34:37.184 INFO:tasks.workunit.client.0.smithi032.stderr:Length: 8071 (7.9K) [application/octet-stream] 2024-04-25T00:34:37.186 INFO:tasks.workunit.client.0.smithi032.stderr:Saving to: ‘iogen_3.1p0.tar’ 2024-04-25T00:34:37.189 INFO:tasks.workunit.client.0.smithi032.stderr: 2024-04-25T00:34:37.189 INFO:tasks.workunit.client.0.smithi032.stderr: 0K ....... 100% 1.31G=0s 2024-04-25T00:34:37.189 INFO:tasks.workunit.client.0.smithi032.stderr: 2024-04-25T00:34:37.189 INFO:tasks.workunit.client.0.smithi032.stderr:2024-04-25 00:34:37 (1.31 GB/s) - ‘iogen_3.1p0.tar’ saved [8071/8071] 2024-04-25T00:34:37.189 INFO:tasks.workunit.client.0.smithi032.stderr: 2024-04-25T00:34:37.189 INFO:tasks.workunit.client.0.smithi032.stderr:+ tar -xvzf iogen_3.1p0.tar 2024-04-25T00:34:37.199 INFO:tasks.workunit.client.0.smithi032.stdout:iogen_3.1p0 2024-04-25T00:34:37.200 INFO:tasks.workunit.client.0.smithi032.stdout:iogen_3.1p0/src 2024-04-25T00:34:37.200 INFO:tasks.workunit.client.0.smithi032.stdout:iogen_3.1p0/src/iogen.8 2024-04-25T00:34:37.201 INFO:tasks.workunit.client.0.smithi032.stdout:iogen_3.1p0/src/iogen.c 2024-04-25T00:34:37.203 INFO:tasks.workunit.client.0.smithi032.stdout:iogen_3.1p0/README 2024-04-25T00:34:37.204 INFO:tasks.workunit.client.0.smithi032.stdout:iogen_3.1p0/Makefile.bsd 2024-04-25T00:34:37.205 INFO:tasks.workunit.client.0.smithi032.stdout:iogen_3.1p0/Makefile 2024-04-25T00:34:37.206 INFO:tasks.workunit.client.0.smithi032.stderr:+ cd iogen_3.1p0 2024-04-25T00:34:37.206 INFO:tasks.workunit.client.0.smithi032.stderr:+ echo 'making iogen' 2024-04-25T00:34:37.206 INFO:tasks.workunit.client.0.smithi032.stdout:making iogen 2024-04-25T00:34:37.206 INFO:tasks.workunit.client.0.smithi032.stderr:+ make 2024-04-25T00:34:37.217 INFO:tasks.workunit.client.0.smithi032.stdout:gcc -D_FILE_OFFSET_BITS=64 -DVERSION=\"3.1p0\" -Dstrlcpy=strncpy -Darc4random=rand -O2 src/iogen.c -o iogen 2024-04-25T00:34:37.300 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:37 smithi032 ceph-mon[29357]: pgmap v42: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 5 op/s 2024-04-25T00:34:37.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:37 smithi121 ceph-mon[32231]: pgmap v42: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 5 op/s 2024-04-25T00:34:37.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:37 smithi130 ceph-mon[32525]: pgmap v42: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 5 op/s 2024-04-25T00:34:37.450 INFO:tasks.workunit.client.0.smithi032.stdout:nroff -mandoc -Tascii src/iogen.8 > iogen.cat8 2024-04-25T00:34:37.534 INFO:tasks.workunit.client.0.smithi032.stdout:running iogen 2024-04-25T00:34:37.535 INFO:tasks.workunit.client.0.smithi032.stderr:+ echo 'running iogen' 2024-04-25T00:34:37.535 INFO:tasks.workunit.client.0.smithi032.stderr:+ ./iogen -n 5 -s 2g 2024-04-25T00:34:37.536 INFO:tasks.workunit.client.0.smithi032.stderr:child spawned 87676 2024-04-25T00:34:37.536 INFO:tasks.workunit.client.0.smithi032.stderr:child spawned 87677 2024-04-25T00:34:37.536 INFO:tasks.workunit.client.0.smithi032.stderr:child spawned 87678 2024-04-25T00:34:37.536 INFO:tasks.workunit.client.0.smithi032.stderr:child spawned 87679 2024-04-25T00:34:37.536 INFO:tasks.workunit.client.0.smithi032.stderr:child spawned 87680 2024-04-25T00:34:37.536 INFO:tasks.workunit.client.0.smithi032.stderr:+ echo 'sleep for 10 min' 2024-04-25T00:34:37.537 INFO:tasks.workunit.client.0.smithi032.stdout:sleep for 10 min 2024-04-25T00:34:37.537 INFO:tasks.workunit.client.0.smithi032.stderr:+ sleep 600 2024-04-25T00:34:39.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:39 smithi121 ceph-mon[32231]: pgmap v43: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 5 op/s 2024-04-25T00:34:39.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:39 smithi130 ceph-mon[32525]: pgmap v43: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 5 op/s 2024-04-25T00:34:39.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:39 smithi032 ceph-mon[29357]: pgmap v43: 129 pgs: 129 active+clean; 1.3 MiB data, 378 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 5 op/s 2024-04-25T00:34:41.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:41 smithi121 ceph-mon[32231]: pgmap v44: 129 pgs: 129 active+clean; 867 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 72 MiB/s wr, 61 op/s 2024-04-25T00:34:41.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:41 smithi130 ceph-mon[32525]: pgmap v44: 129 pgs: 129 active+clean; 867 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 72 MiB/s wr, 61 op/s 2024-04-25T00:34:41.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:41 smithi032 ceph-mon[29357]: pgmap v44: 129 pgs: 129 active+clean; 867 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 72 MiB/s wr, 61 op/s 2024-04-25T00:34:43.376 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:43 smithi032 ceph-mon[29357]: pgmap v45: 129 pgs: 129 active+clean; 867 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 72 MiB/s wr, 58 op/s 2024-04-25T00:34:43.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:43 smithi130 ceph-mon[32525]: pgmap v45: 129 pgs: 129 active+clean; 867 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 72 MiB/s wr, 58 op/s 2024-04-25T00:34:43.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:43 smithi121 ceph-mon[32231]: pgmap v45: 129 pgs: 129 active+clean; 867 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 72 MiB/s wr, 58 op/s 2024-04-25T00:34:45.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:45 smithi130 ceph-mon[32525]: pgmap v46: 129 pgs: 129 active+clean; 2.1 GiB data, 4.7 GiB used, 1.0 TiB / 1.0 TiB avail; 338 KiB/s rd, 181 MiB/s wr, 127 op/s 2024-04-25T00:34:45.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:45 smithi032 ceph-mon[29357]: pgmap v46: 129 pgs: 129 active+clean; 2.1 GiB data, 4.7 GiB used, 1.0 TiB / 1.0 TiB avail; 338 KiB/s rd, 181 MiB/s wr, 127 op/s 2024-04-25T00:34:45.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:45 smithi121 ceph-mon[32231]: pgmap v46: 129 pgs: 129 active+clean; 2.1 GiB data, 4.7 GiB used, 1.0 TiB / 1.0 TiB avail; 338 KiB/s rd, 181 MiB/s wr, 127 op/s 2024-04-25T00:34:47.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:47 smithi032 ceph-mon[29357]: pgmap v47: 129 pgs: 129 active+clean; 5.7 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 893 KiB/s rd, 490 MiB/s wr, 359 op/s 2024-04-25T00:34:47.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:47 smithi121 ceph-mon[32231]: pgmap v47: 129 pgs: 129 active+clean; 5.7 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 893 KiB/s rd, 490 MiB/s wr, 359 op/s 2024-04-25T00:34:47.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:47 smithi130 ceph-mon[32525]: pgmap v47: 129 pgs: 129 active+clean; 5.7 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 893 KiB/s rd, 490 MiB/s wr, 359 op/s 2024-04-25T00:34:49.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:49 smithi032 ceph-mon[29357]: pgmap v48: 129 pgs: 129 active+clean; 5.7 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 893 KiB/s rd, 490 MiB/s wr, 359 op/s 2024-04-25T00:34:49.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:49 smithi121 ceph-mon[32231]: pgmap v48: 129 pgs: 129 active+clean; 5.7 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 893 KiB/s rd, 490 MiB/s wr, 359 op/s 2024-04-25T00:34:49.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:49 smithi130 ceph-mon[32525]: pgmap v48: 129 pgs: 129 active+clean; 5.7 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 893 KiB/s rd, 490 MiB/s wr, 359 op/s 2024-04-25T00:34:51.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:51 smithi032 ceph-mon[29357]: pgmap v49: 129 pgs: 129 active+clean; 10 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 894 KiB/s rd, 906 MiB/s wr, 936 op/s 2024-04-25T00:34:51.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:51 smithi121 ceph-mon[32231]: pgmap v49: 129 pgs: 129 active+clean; 10 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 894 KiB/s rd, 906 MiB/s wr, 936 op/s 2024-04-25T00:34:51.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:51 smithi130 ceph-mon[32525]: pgmap v49: 129 pgs: 129 active+clean; 10 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 894 KiB/s rd, 906 MiB/s wr, 936 op/s 2024-04-25T00:34:53.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:53 smithi032 ceph-mon[29357]: pgmap v50: 129 pgs: 129 active+clean; 10 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 838 KiB/s rd, 834 MiB/s wr, 880 op/s 2024-04-25T00:34:53.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:53 smithi121 ceph-mon[32231]: pgmap v50: 129 pgs: 129 active+clean; 10 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 838 KiB/s rd, 834 MiB/s wr, 880 op/s 2024-04-25T00:34:53.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:53 smithi130 ceph-mon[32525]: pgmap v50: 129 pgs: 129 active+clean; 10 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 838 KiB/s rd, 834 MiB/s wr, 880 op/s 2024-04-25T00:34:55.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:55 smithi032 ceph-mon[29357]: pgmap v51: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 839 KiB/s rd, 999 MiB/s wr, 1.29k op/s 2024-04-25T00:34:55.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:55 smithi121 ceph-mon[32231]: pgmap v51: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 839 KiB/s rd, 999 MiB/s wr, 1.29k op/s 2024-04-25T00:34:55.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:55 smithi130 ceph-mon[32525]: pgmap v51: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 839 KiB/s rd, 999 MiB/s wr, 1.29k op/s 2024-04-25T00:34:57.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:57 smithi032 ceph-mon[29357]: pgmap v52: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 558 KiB/s rd, 1.1 GiB/s wr, 1.69k op/s 2024-04-25T00:34:57.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:57 smithi121 ceph-mon[32231]: pgmap v52: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 558 KiB/s rd, 1.1 GiB/s wr, 1.69k op/s 2024-04-25T00:34:57.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:57 smithi130 ceph-mon[32525]: pgmap v52: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 558 KiB/s rd, 1.1 GiB/s wr, 1.69k op/s 2024-04-25T00:34:58.731 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-25T00:34:58.732 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-25T00:34:59.081 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.080+0000 7f49d06bd640 1 -- 172.21.15.32:0/3368186461 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f49c8111440 msgr2=0x7f49c8119010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:59.081 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.080+0000 7f49d06bd640 1 --2- 172.21.15.32:0/3368186461 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f49c8111440 0x7f49c8119010 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f49c4009900 tx=0x7f49c402a440 comp rx=0 tx=0).stop 2024-04-25T00:34:59.086 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.086+0000 7f49d06bd640 1 -- 172.21.15.32:0/3368186461 shutdown_connections 2024-04-25T00:34:59.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.086+0000 7f49d06bd640 1 --2- 172.21.15.32:0/3368186461 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f49c8111440 0x7f49c8119010 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.086+0000 7f49d06bd640 1 --2- 172.21.15.32:0/3368186461 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f49c810cd20 0x7f49c8110d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.086+0000 7f49d06bd640 1 --2- 172.21.15.32:0/3368186461 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f49c810c370 0x7f49c810c750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.086+0000 7f49d06bd640 1 -- 172.21.15.32:0/3368186461 >> 172.21.15.32:0/3368186461 conn(0x7f49c806c760 msgr2=0x7f49c810a750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:59.090 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.089+0000 7f49d06bd640 1 -- 172.21.15.32:0/3368186461 shutdown_connections 2024-04-25T00:34:59.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.090+0000 7f49d06bd640 1 -- 172.21.15.32:0/3368186461 wait complete. 2024-04-25T00:34:59.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.090+0000 7f49d06bd640 1 Processor -- start 2024-04-25T00:34:59.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.091+0000 7f49d06bd640 1 -- start start 2024-04-25T00:34:59.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.091+0000 7f49d06bd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f49c810c370 0x7f49c811e6d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:59.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.091+0000 7f49d06bd640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f49c8111440 0x7f49c811ec10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:59.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.091+0000 7f49d06bd640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f49c8117b20 0x7f49c8117f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:59.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.091+0000 7f49d06bd640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f49c806e360 con 0x7f49c810c370 2024-04-25T00:34:59.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.091+0000 7f49d06bd640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f49c806e1e0 con 0x7f49c8111440 2024-04-25T00:34:59.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.091+0000 7f49d06bd640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f49c806e4e0 con 0x7f49c8117b20 2024-04-25T00:34:59.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.091+0000 7f49cec5a640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f49c8117b20 0x7f49c8117f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:59.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.091+0000 7f49cec5a640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f49c8117b20 0x7f49c8117f90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:36622/0 (socket says 172.21.15.32:36622) 2024-04-25T00:34:59.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.091+0000 7f49cec5a640 1 -- 172.21.15.32:0/4267412002 learned_addr learned my addr 172.21.15.32:0/4267412002 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:59.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.092+0000 7f49cec5a640 1 -- 172.21.15.32:0/4267412002 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f49c8111440 msgr2=0x7f49c811ec10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:59.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.092+0000 7f49cec5a640 1 --2- 172.21.15.32:0/4267412002 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f49c8111440 0x7f49c811ec10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.092+0000 7f49cec5a640 1 -- 172.21.15.32:0/4267412002 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f49c810c370 msgr2=0x7f49c811e6d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:59.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.092+0000 7f49cec5a640 1 --2- 172.21.15.32:0/4267412002 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f49c810c370 0x7f49c811e6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.092+0000 7f49cec5a640 1 -- 172.21.15.32:0/4267412002 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f49c8118850 con 0x7f49c8117b20 2024-04-25T00:34:59.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.095+0000 7f49cec5a640 1 --2- 172.21.15.32:0/4267412002 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f49c8117b20 0x7f49c8117f90 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f49c40058c0 tx=0x7f49c4005000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:59.102 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.097+0000 7f49bf7fe640 1 -- 172.21.15.32:0/4267412002 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f49c4040030 con 0x7f49c8117b20 2024-04-25T00:34:59.102 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.097+0000 7f49d06bd640 1 -- 172.21.15.32:0/4267412002 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f49c8122330 con 0x7f49c8117b20 2024-04-25T00:34:59.102 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.097+0000 7f49d06bd640 1 -- 172.21.15.32:0/4267412002 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f49c81227d0 con 0x7f49c8117b20 2024-04-25T00:34:59.102 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.097+0000 7f49bf7fe640 1 -- 172.21.15.32:0/4267412002 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f49c4005380 con 0x7f49c8117b20 2024-04-25T00:34:59.102 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.097+0000 7f49bf7fe640 1 -- 172.21.15.32:0/4267412002 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f49c403c6e0 con 0x7f49c8117b20 2024-04-25T00:34:59.105 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.103+0000 7f49d33d1740 1 Processor -- start 2024-04-25T00:34:59.112 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.110+0000 7f49d33d1740 1 -- start start 2024-04-25T00:34:59.113 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.112+0000 7f49bf7fe640 1 -- 172.21.15.32:0/4267412002 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 25) ==== 138066+0+0 (secure 0 0 0) 0x7f49c4043030 con 0x7f49c8117b20 2024-04-25T00:34:59.117 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.113+0000 7f49bf7fe640 1 --2- 172.21.15.32:0/4267412002 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f49b40a3030 0x7f49b40a54f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:59.117 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.114+0000 7f49bf7fe640 1 -- 172.21.15.32:0/4267412002 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f49c40f03a0 con 0x7f49c8117b20 2024-04-25T00:34:59.117 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.115+0000 7f49d33d1740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55edc0a91630 0x55edc0a91a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:59.117 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.115+0000 7f49d33d1740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55edc0a91f50 0x55edc0a9c0e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:59.117 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.115+0000 7f49d33d1740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55edc0a9c620 0x55edc0a9ea10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:59.117 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.115+0000 7f49d33d1740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55edc09f66b0 con 0x55edc0a91f50 2024-04-25T00:34:59.117 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.115+0000 7f49d33d1740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55edc099e880 con 0x55edc0a91630 2024-04-25T00:34:59.117 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.115+0000 7f49d33d1740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55edc09f2670 con 0x55edc0a9c620 2024-04-25T00:34:59.117 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.116+0000 7f49ce459640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55edc0a91630 0x55edc0a91a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:59.117 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.116+0000 7f49ce459640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55edc0a91630 0x55edc0a91a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:42590/0 (socket says 172.21.15.32:42590) 2024-04-25T00:34:59.118 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.116+0000 7f49ce459640 1 -- 172.21.15.32:0/4152960964 learned_addr learned my addr 172.21.15.32:0/4152960964 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:59.118 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.116+0000 7f49cec5a640 1 --2- 172.21.15.32:0/4152960964 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55edc0a9c620 0x55edc0a9ea10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:59.118 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.116+0000 7f49cdc58640 1 --2- 172.21.15.32:0/4152960964 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55edc0a91f50 0x55edc0a9c0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:59.118 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.116+0000 7f49cdc58640 1 --2- 172.21.15.32:0/4267412002 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f49b40a3030 0x7f49b40a54f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:59.119 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.117+0000 7f49cec5a640 1 -- 172.21.15.32:0/4152960964 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55edc0a91630 msgr2=0x55edc0a91a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:59.119 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.117+0000 7f49cec5a640 1 --2- 172.21.15.32:0/4152960964 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55edc0a91630 0x55edc0a91a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.119 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.117+0000 7f49cec5a640 1 -- 172.21.15.32:0/4152960964 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55edc0a91f50 msgr2=0x55edc0a9c0e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:59.119 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.117+0000 7f49cec5a640 1 --2- 172.21.15.32:0/4152960964 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55edc0a91f50 0x55edc0a9c0e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.119 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.117+0000 7f49cec5a640 1 -- 172.21.15.32:0/4152960964 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55edc07d47d0 con 0x55edc0a9c620 2024-04-25T00:34:59.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.120+0000 7f49cec5a640 1 --2- 172.21.15.32:0/4152960964 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55edc0a9c620 0x55edc0a9ea10 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f49c40ed560 tx=0x7f49c402a440 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9987b227d1666097 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:59.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.122+0000 7f49bcff9640 1 -- 172.21.15.32:0/4152960964 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f49c40ba630 con 0x55edc0a9c620 2024-04-25T00:34:59.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.122+0000 7f49bcff9640 1 -- 172.21.15.32:0/4152960964 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f49c4005530 con 0x55edc0a9c620 2024-04-25T00:34:59.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.122+0000 7f49bcff9640 1 -- 172.21.15.32:0/4152960964 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f49c40c4b10 con 0x55edc0a9c620 2024-04-25T00:34:59.129 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.127+0000 7f49d33d1740 1 -- 172.21.15.32:0/4152960964 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55edc0a9c620 msgr2=0x55edc0a9ea10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:59.129 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.127+0000 7f49d33d1740 1 --2- 172.21.15.32:0/4152960964 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55edc0a9c620 0x55edc0a9ea10 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f49c40ed560 tx=0x7f49c402a440 comp rx=0 tx=0).stop 2024-04-25T00:34:59.139 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.136+0000 7f49cdc58640 1 --2- 172.21.15.32:0/4267412002 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f49b40a3030 0x7f49b40a54f0 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x55edc0a92bf0 tx=0x7f49b8005850 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:59.143 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.142+0000 7f49d33d1740 1 -- 172.21.15.32:0/4152960964 shutdown_connections 2024-04-25T00:34:59.144 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.142+0000 7f49d33d1740 1 --2- 172.21.15.32:0/4152960964 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55edc0a9c620 0x55edc0a9ea10 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.144 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.142+0000 7f49d33d1740 1 --2- 172.21.15.32:0/4152960964 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55edc0a91f50 0x55edc0a9c0e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.144 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.142+0000 7f49d33d1740 1 --2- 172.21.15.32:0/4152960964 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55edc0a91630 0x55edc0a91a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.144 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.142+0000 7f49d33d1740 1 -- 172.21.15.32:0/4152960964 >> 172.21.15.32:0/4152960964 conn(0x55edc0a8d950 msgr2=0x55edc0a8dd30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:59.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.152+0000 7f49d33d1740 1 -- 172.21.15.32:0/4152960964 shutdown_connections 2024-04-25T00:34:59.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.161+0000 7f49d33d1740 1 -- 172.21.15.32:0/4152960964 wait complete. 2024-04-25T00:34:59.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.161+0000 7f49d33d1740 1 -- start start 2024-04-25T00:34:59.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.161+0000 7f49d33d1740 1 Processor -- start 2024-04-25T00:34:59.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.164+0000 7f49d33d1740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55edc0a91630 0x55edc0a99580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:59.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.165+0000 7f49d33d1740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55edc0a91f50 0x55edc0a99ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:59.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.165+0000 7f49d33d1740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55edc0a9c620 0x55edc0a9a000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:59.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.165+0000 7f49d33d1740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55edc09f2670 con 0x55edc0a9c620 2024-04-25T00:34:59.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.165+0000 7f49d33d1740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55edc09b1f50 con 0x55edc0a91630 2024-04-25T00:34:59.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.165+0000 7f49d33d1740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55edc0a9f1d0 con 0x55edc0a91f50 2024-04-25T00:34:59.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.165+0000 7f49d33d1740 10 client.0 _ll_register_callbacks cb 0x55edc09ed000 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:34:59.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.165+0000 7f49cdc58640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55edc0a91f50 0x55edc0a99ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:59.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.165+0000 7f49cdc58640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55edc0a91f50 0x55edc0a99ac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:36638/0 (socket says 172.21.15.32:36638) 2024-04-25T00:34:59.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.165+0000 7f49cdc58640 1 -- 172.21.15.32:0/428105003 learned_addr learned my addr 172.21.15.32:0/428105003 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:59.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.167+0000 7f49cdc58640 1 -- 172.21.15.32:0/428105003 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55edc0a91630 msgr2=0x55edc0a99580 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-25T00:34:59.171 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.167+0000 7f49cdc58640 1 --2- 172.21.15.32:0/428105003 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55edc0a91630 0x55edc0a99580 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.171 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.167+0000 7f49cdc58640 1 -- 172.21.15.32:0/428105003 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55edc0a9c620 msgr2=0x55edc0a9a000 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-25T00:34:59.171 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.167+0000 7f49cdc58640 1 --2- 172.21.15.32:0/428105003 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55edc0a9c620 0x55edc0a9a000 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.171 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.167+0000 7f49cdc58640 1 -- 172.21.15.32:0/428105003 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55edc070a3f0 con 0x55edc0a91f50 2024-04-25T00:34:59.171 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.168+0000 7f49cdc58640 1 --2- 172.21.15.32:0/428105003 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55edc0a91f50 0x55edc0a99ac0 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f49b8010c20 tx=0x7f49b8005f90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1071a294670b4ef2 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:59.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.172+0000 7f49b1ffb640 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:34:59.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.172+0000 7f49b1ffb640 1 -- 172.21.15.32:0/428105003 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f49b8024020 con 0x55edc0a91f50 2024-04-25T00:34:59.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.172+0000 7f49b1ffb640 1 -- 172.21.15.32:0/428105003 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f49b800fc00 con 0x55edc0a91f50 2024-04-25T00:34:59.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.175+0000 7f49d33d1740 1 -- 172.21.15.32:0/428105003 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f49c4005530 con 0x55edc0a91f50 2024-04-25T00:34:59.181 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.181+0000 7f49b1ffb640 1 -- 172.21.15.32:0/428105003 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7f49b8013e60 con 0x55edc0a91f50 2024-04-25T00:34:59.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.181+0000 7f49d33d1740 10 client.25495 fetch_fsmap learned FSMap version 26 2024-04-25T00:34:59.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.181+0000 7f49d33d1740 1 -- 172.21.15.32:0/428105003 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x55edc06ca0f0 con 0x55edc0a91f50 2024-04-25T00:34:59.188 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.188+0000 7f49b1ffb640 1 -- 172.21.15.32:0/428105003 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f49c4005530 con 0x55edc0a91f50 2024-04-25T00:34:59.188 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.188+0000 7f49b1ffb640 1 -- 172.21.15.32:0/428105003 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7f49b8031020 con 0x55edc0a91f50 2024-04-25T00:34:59.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.188+0000 7f49d33d1740 10 client.25495 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:34:59.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.188+0000 7f49d33d1740 10 client.25495 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:34:59.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.188+0000 7f49d33d1740 20 client.25495 populate_metadata read hostname 'smithi032' 2024-04-25T00:34:59.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.188+0000 7f49d33d1740 1 --2- 172.21.15.32:0/428105003 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55edc0aacad0 0x55edc0aaef50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:59.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.188+0000 7f49d33d1740 4 client.25495 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:34:59.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.188+0000 7f49d33d1740 1 -- 172.21.15.32:0/428105003 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55edc0a03bf0 con 0x55edc0aacad0 2024-04-25T00:34:59.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.192+0000 7f49ce459640 1 --2- 172.21.15.32:0/428105003 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55edc0aacad0 0x55edc0aaef50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:59.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.206+0000 7f49ce459640 1 --2- 172.21.15.32:0/428105003 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55edc0aacad0 0x55edc0aaef50 crc :-1 s=READY pgs=61 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=10f416a6ce265451 server_cookie=ca708a6e9b61ef95 in_seq=0 out_seq=0 2024-04-25T00:34:59.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.207+0000 7f49b1ffb640 10 client.25495 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:34:59.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.212+0000 7f49b1ffb640 1 -- 172.21.15.32:0/428105003 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x55edc0a03bf0 con 0x55edc0aacad0 2024-04-25T00:34:59.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.212+0000 7f49b1ffb640 10 client.25495 handle_command_reply: tid=0 2024-04-25T00:34:59.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.212+0000 7f49d33d1740 1 client.25495 shutdown 2024-04-25T00:34:59.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.217+0000 7f49d33d1740 1 -- 172.21.15.32:0/428105003 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55edc0a91f50 msgr2=0x55edc0a99ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:59.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.217+0000 7f49d33d1740 1 --2- 172.21.15.32:0/428105003 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55edc0a91f50 0x55edc0a99ac0 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f49b8010c20 tx=0x7f49b8005f90 comp rx=0 tx=0).stop 2024-04-25T00:34:59.224 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.224+0000 7f49d33d1740 1 -- 172.21.15.32:0/428105003 shutdown_connections 2024-04-25T00:34:59.224 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.224+0000 7f49d33d1740 1 --2- 172.21.15.32:0/428105003 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55edc0aacad0 0x55edc0aaef50 crc :-1 s=READY pgs=61 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.224 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.224+0000 7f49d33d1740 1 --2- 172.21.15.32:0/428105003 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55edc0a9c620 0x55edc0a9a000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.224 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.224+0000 7f49d33d1740 1 --2- 172.21.15.32:0/428105003 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55edc0a91f50 0x55edc0a99ac0 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.224 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.224+0000 7f49d33d1740 1 --2- 172.21.15.32:0/428105003 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55edc0a91630 0x55edc0a99580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.224 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.224+0000 7f49d33d1740 1 -- 172.21.15.32:0/428105003 >> 172.21.15.32:0/428105003 conn(0x55edc0a8d950 msgr2=0x55edc0a8caf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:59.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.226+0000 7f49b1ffb640 0 client.25495 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:34:59.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.226+0000 7f49d33d1740 1 -- 172.21.15.32:0/428105003 shutdown_connections 2024-04-25T00:34:59.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.228+0000 7f49d33d1740 1 -- 172.21.15.32:0/428105003 wait complete. 2024-04-25T00:34:59.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.237+0000 7f49d33d1740 20 client.25495 trim_cache size 0 max 16384 2024-04-25T00:34:59.277 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.277+0000 7f49d33d1740 1 Processor -- start 2024-04-25T00:34:59.278 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.278+0000 7f49d33d1740 1 -- start start 2024-04-25T00:34:59.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.278+0000 7f49d33d1740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55edc0a91630 0x55edc0aa7f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:59.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.279+0000 7f49d33d1740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55edc0a91f50 0x55edc0a99e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:59.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.279+0000 7f49d33d1740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55edc0a9c620 0x55edc0a8c970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:59.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.279+0000 7f49ce459640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55edc0a91630 0x55edc0aa7f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:59.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.279+0000 7f49ce459640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55edc0a91630 0x55edc0aa7f70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:42602/0 (socket says 172.21.15.32:42602) 2024-04-25T00:34:59.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.279+0000 7f49ce459640 1 -- 172.21.15.32:0/1937570300 learned_addr learned my addr 172.21.15.32:0/1937570300 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:59.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.279+0000 7f49cec5a640 1 --2- 172.21.15.32:0/1937570300 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55edc0a9c620 0x55edc0a8c970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:59.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.279+0000 7f49d33d1740 1 -- 172.21.15.32:0/1937570300 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f49ac0128f0 con 0x55edc0a91f50 2024-04-25T00:34:59.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.279+0000 7f49d33d1740 1 -- 172.21.15.32:0/1937570300 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f49ac0101c0 con 0x55edc0a91630 2024-04-25T00:34:59.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.279+0000 7f49d33d1740 1 -- 172.21.15.32:0/1937570300 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f49ac00e010 con 0x55edc0a9c620 2024-04-25T00:34:59.281 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.280+0000 7f49cdc58640 1 --2- 172.21.15.32:0/1937570300 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55edc0a91f50 0x55edc0a99e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:59.282 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.281+0000 7f49ce459640 1 -- 172.21.15.32:0/1937570300 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55edc0a9c620 msgr2=0x55edc0a8c970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:59.282 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.281+0000 7f49ce459640 1 --2- 172.21.15.32:0/1937570300 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55edc0a9c620 0x55edc0a8c970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.282 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.281+0000 7f49ce459640 1 -- 172.21.15.32:0/1937570300 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55edc0a91f50 msgr2=0x55edc0a99e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:59.282 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.281+0000 7f49ce459640 1 --2- 172.21.15.32:0/1937570300 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55edc0a91f50 0x55edc0a99e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.282 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.281+0000 7f49ce459640 1 -- 172.21.15.32:0/1937570300 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55edc0a7f090 con 0x55edc0a91630 2024-04-25T00:34:59.283 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.282+0000 7f49ce459640 1 --2- 172.21.15.32:0/1937570300 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55edc0a91630 0x55edc0aa7f70 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f49c0008400 tx=0x7f49c000b040 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7b5232e935d1fabd server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:59.283 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.282+0000 7f49b3fff640 1 -- 172.21.15.32:0/1937570300 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f49c0019330 con 0x55edc0a91630 2024-04-25T00:34:59.283 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.282+0000 7f49b3fff640 1 -- 172.21.15.32:0/1937570300 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f49c0018410 con 0x55edc0a91630 2024-04-25T00:34:59.283 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.282+0000 7f49b3fff640 1 -- 172.21.15.32:0/1937570300 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f49c0017610 con 0x55edc0a91630 2024-04-25T00:34:59.286 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.285+0000 7f49d33d1740 1 -- 172.21.15.32:0/1937570300 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55edc0a91630 msgr2=0x55edc0aa7f70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:59.286 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.285+0000 7f49d33d1740 1 --2- 172.21.15.32:0/1937570300 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55edc0a91630 0x55edc0aa7f70 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f49c0008400 tx=0x7f49c000b040 comp rx=0 tx=0).stop 2024-04-25T00:34:59.286 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.285+0000 7f49d33d1740 1 -- 172.21.15.32:0/1937570300 shutdown_connections 2024-04-25T00:34:59.286 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.285+0000 7f49d33d1740 1 --2- 172.21.15.32:0/1937570300 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55edc0a9c620 0x55edc0a8c970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.286 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.286+0000 7f49d33d1740 1 --2- 172.21.15.32:0/1937570300 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55edc0a91f50 0x55edc0a99e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.286 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.286+0000 7f49d33d1740 1 --2- 172.21.15.32:0/1937570300 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55edc0a91630 0x55edc0aa7f70 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.287 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.286+0000 7f49d33d1740 1 -- 172.21.15.32:0/1937570300 >> 172.21.15.32:0/1937570300 conn(0x55edc0a8d950 msgr2=0x55edc0a8c260 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:59.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.287+0000 7f49d33d1740 1 -- 172.21.15.32:0/1937570300 shutdown_connections 2024-04-25T00:34:59.288 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.287+0000 7f49d33d1740 1 -- 172.21.15.32:0/1937570300 wait complete. 2024-04-25T00:34:59.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.288+0000 7f49d33d1740 1 -- start start 2024-04-25T00:34:59.289 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.289+0000 7f49d33d1740 1 Processor -- start 2024-04-25T00:34:59.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.290+0000 7f49d33d1740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55edc0a91630 0x55edc0aa4710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:59.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.290+0000 7f49d33d1740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55edc0a91f50 0x55edc0a9fa80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:59.291 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.291+0000 7f49d33d1740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55edc0a9c620 0x55edc0a9ffc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:59.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.291+0000 7f49d33d1740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f49ac00e680 con 0x55edc0a9c620 2024-04-25T00:34:59.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.291+0000 7f49d33d1740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f49ac00faf0 con 0x55edc0a91f50 2024-04-25T00:34:59.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.291+0000 7f49ce459640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55edc0a91630 0x55edc0aa4710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:59.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.291+0000 7f49d33d1740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f49ac00ed50 con 0x55edc0a91630 2024-04-25T00:34:59.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.291+0000 7f49ce459640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55edc0a91630 0x55edc0aa4710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:36660/0 (socket says 172.21.15.32:36660) 2024-04-25T00:34:59.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.291+0000 7f49ce459640 1 -- 172.21.15.32:0/2734595494 learned_addr learned my addr 172.21.15.32:0/2734595494 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:59.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.291+0000 7f49d33d1740 10 client.0 _ll_register_callbacks cb 0x7f49b80097d0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:34:59.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.291+0000 7f49cdc58640 1 --2- 172.21.15.32:0/2734595494 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55edc0a91f50 0x55edc0a9fa80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:59.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.293+0000 7f49cec5a640 1 --2- 172.21.15.32:0/2734595494 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55edc0a9c620 0x55edc0a9ffc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:59.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.294+0000 7f49cec5a640 1 -- 172.21.15.32:0/2734595494 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55edc0a91630 msgr2=0x55edc0aa4710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:59.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.294+0000 7f49cec5a640 1 --2- 172.21.15.32:0/2734595494 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55edc0a91630 0x55edc0aa4710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.294+0000 7f49cec5a640 1 -- 172.21.15.32:0/2734595494 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55edc0a91f50 msgr2=0x55edc0a9fa80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:34:59.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.294+0000 7f49cec5a640 1 --2- 172.21.15.32:0/2734595494 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55edc0a91f50 0x55edc0a9fa80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.294+0000 7f49cec5a640 1 -- 172.21.15.32:0/2734595494 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55edc0aa06b0 con 0x55edc0a9c620 2024-04-25T00:34:59.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.294+0000 7f49cec5a640 1 --2- 172.21.15.32:0/2734595494 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55edc0a9c620 0x55edc0a9ffc0 secure :-1 s=READY pgs=587 cs=0 l=1 rev1=1 crypto rx=0x55edc0a8eae0 tx=0x7f49c40bbfa0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3ccef9e37db467ef server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:59.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.294+0000 7f49b1ffb640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:34:59.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.295+0000 7f49b1ffb640 1 -- 172.21.15.32:0/2734595494 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f49c40bca30 con 0x55edc0a9c620 2024-04-25T00:34:59.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.295+0000 7f49b1ffb640 1 -- 172.21.15.32:0/2734595494 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f49c40d50d0 con 0x55edc0a9c620 2024-04-25T00:34:59.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.295+0000 7f49d33d1740 1 -- 172.21.15.32:0/2734595494 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f49c0018410 con 0x55edc0a9c620 2024-04-25T00:34:59.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.296+0000 7f49b1ffb640 1 -- 172.21.15.32:0/2734595494 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7f49c40d52d0 con 0x55edc0a9c620 2024-04-25T00:34:59.301 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.301+0000 7f49d33d1740 10 client.15978 fetch_fsmap learned FSMap version 26 2024-04-25T00:34:59.301 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.301+0000 7f49d33d1740 1 -- 172.21.15.32:0/2734595494 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x55edc0b12fa0 con 0x55edc0a9c620 2024-04-25T00:34:59.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.302+0000 7f49b1ffb640 1 -- 172.21.15.32:0/2734595494 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f49c0018410 con 0x55edc0a9c620 2024-04-25T00:34:59.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.302+0000 7f49b1ffb640 1 -- 172.21.15.32:0/2734595494 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7f49c40c8080 con 0x55edc0a9c620 2024-04-25T00:34:59.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.302+0000 7f49d33d1740 10 client.15978 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:34:59.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.302+0000 7f49d33d1740 10 client.15978 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:34:59.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.302+0000 7f49d33d1740 20 client.15978 populate_metadata read hostname 'smithi032' 2024-04-25T00:34:59.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.302+0000 7f49d33d1740 1 --2- 172.21.15.32:0/2734595494 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55edc0b13470 0x55edc0b15890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:59.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.302+0000 7f49d33d1740 4 client.15978 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-25T00:34:59.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.302+0000 7f49d33d1740 1 -- 172.21.15.32:0/2734595494 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "scrub status"}) -- 0x55edc09fdf40 con 0x55edc0b13470 2024-04-25T00:34:59.304 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.303+0000 7f49ce459640 1 --2- 172.21.15.32:0/2734595494 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55edc0b13470 0x55edc0b15890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:59.307 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.304+0000 7f49ce459640 1 --2- 172.21.15.32:0/2734595494 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55edc0b13470 0x55edc0b15890 crc :-1 s=READY pgs=62 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=741d56239390d01c server_cookie=97f25bfef1247c1d in_seq=0 out_seq=0 2024-04-25T00:34:59.308 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.305+0000 7f49b1ffb640 10 client.15978 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:34:59.309 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.308+0000 7f49b1ffb640 1 -- 172.21.15.32:0/2734595494 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+63 (crc 0 0 0) 0x55edc09fdf40 con 0x55edc0b13470 2024-04-25T00:34:59.309 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.308+0000 7f49b1ffb640 10 client.15978 handle_command_reply: tid=0 2024-04-25T00:34:59.309 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.308+0000 7f49d33d1740 1 client.15978 shutdown 2024-04-25T00:34:59.309 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.308+0000 7f49d33d1740 1 -- 172.21.15.32:0/2734595494 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55edc0a9c620 msgr2=0x55edc0a9ffc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:59.309 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.308+0000 7f49d33d1740 1 --2- 172.21.15.32:0/2734595494 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55edc0a9c620 0x55edc0a9ffc0 secure :-1 s=READY pgs=587 cs=0 l=1 rev1=1 crypto rx=0x55edc0a8eae0 tx=0x7f49c40bbfa0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.310 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.308+0000 7f49d33d1740 1 -- 172.21.15.32:0/2734595494 shutdown_connections 2024-04-25T00:34:59.310 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.308+0000 7f49d33d1740 1 --2- 172.21.15.32:0/2734595494 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55edc0b13470 0x55edc0b15890 crc :-1 s=READY pgs=62 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.310 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.308+0000 7f49d33d1740 1 --2- 172.21.15.32:0/2734595494 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55edc0a9c620 0x55edc0a9ffc0 unknown :-1 s=CLOSED pgs=587 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.310 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.308+0000 7f49d33d1740 1 --2- 172.21.15.32:0/2734595494 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55edc0a91f50 0x55edc0a9fa80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.310 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.308+0000 7f49d33d1740 1 --2- 172.21.15.32:0/2734595494 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55edc0a91630 0x55edc0aa4710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.310 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.308+0000 7f49d33d1740 1 -- 172.21.15.32:0/2734595494 >> 172.21.15.32:0/2734595494 conn(0x55edc0a8d950 msgr2=0x55edc0a8caf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:59.310 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.309+0000 7f49b1ffb640 0 client.15978 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:34:59.311 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.309+0000 7f49d33d1740 1 -- 172.21.15.32:0/2734595494 shutdown_connections 2024-04-25T00:34:59.311 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.309+0000 7f49d33d1740 1 -- 172.21.15.32:0/2734595494 wait complete. 2024-04-25T00:34:59.312 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.309+0000 7f49d33d1740 20 client.15978 trim_cache size 0 max 16384 2024-04-25T00:34:59.312 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:34:59.312 INFO:teuthology.orchestra.run.smithi032.stdout: "status": "no active scrubs running", 2024-04-25T00:34:59.312 INFO:teuthology.orchestra.run.smithi032.stdout: "scrubs": {} 2024-04-25T00:34:59.312 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:34:59.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.310+0000 7f49d06bd640 1 -- 172.21.15.32:0/4267412002 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f49b40a3030 msgr2=0x7f49b40a54f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:59.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.310+0000 7f49d06bd640 1 --2- 172.21.15.32:0/4267412002 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f49b40a3030 0x7f49b40a54f0 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x55edc0a92bf0 tx=0x7f49b8005850 comp rx=0 tx=0).stop 2024-04-25T00:34:59.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.310+0000 7f49d06bd640 1 -- 172.21.15.32:0/4267412002 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f49c8117b20 msgr2=0x7f49c8117f90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:59.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.311+0000 7f49d06bd640 1 --2- 172.21.15.32:0/4267412002 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f49c8117b20 0x7f49c8117f90 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f49c40058c0 tx=0x7f49c4005000 comp rx=0 tx=0).stop 2024-04-25T00:34:59.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.311+0000 7f49d06bd640 1 -- 172.21.15.32:0/4267412002 shutdown_connections 2024-04-25T00:34:59.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.311+0000 7f49d06bd640 1 --2- 172.21.15.32:0/4267412002 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f49b40a3030 0x7f49b40a54f0 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.311+0000 7f49d06bd640 1 --2- 172.21.15.32:0/4267412002 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f49c8117b20 0x7f49c8117f90 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.311+0000 7f49d06bd640 1 --2- 172.21.15.32:0/4267412002 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f49c8111440 0x7f49c811ec10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.316 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.311+0000 7f49d06bd640 1 --2- 172.21.15.32:0/4267412002 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f49c810c370 0x7f49c811e6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.316 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.311+0000 7f49d06bd640 1 -- 172.21.15.32:0/4267412002 >> 172.21.15.32:0/4267412002 conn(0x7f49c806c760 msgr2=0x7f49c8070420 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:59.316 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.311+0000 7f49d06bd640 1 -- 172.21.15.32:0/4267412002 shutdown_connections 2024-04-25T00:34:59.316 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.311+0000 7f49d06bd640 1 -- 172.21.15.32:0/4267412002 wait complete. 2024-04-25T00:34:59.345 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-25T00:34:59.345 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:34:59.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:59 smithi121 ceph-mon[32231]: pgmap v53: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 2.0 KiB/s rd, 767 MiB/s wr, 1.46k op/s 2024-04-25T00:34:59.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:34:59 smithi121 ceph-mon[32231]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:34:59.644 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:59 smithi130 ceph-mon[32525]: pgmap v53: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 2.0 KiB/s rd, 767 MiB/s wr, 1.46k op/s 2024-04-25T00:34:59.645 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:34:59 smithi130 ceph-mon[32525]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:34:59.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.683+0000 7fa0e66ae640 1 -- 172.21.15.32:0/2209273373 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa0e01089b0 msgr2=0x7fa0e0108d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:59.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.683+0000 7fa0e66ae640 1 --2- 172.21.15.32:0/2209273373 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa0e01089b0 0x7fa0e0108d90 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7fa0dc008570 tx=0x7fa0dc00ac00 comp rx=0 tx=0).stop 2024-04-25T00:34:59.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.684+0000 7fa0e66ae640 1 -- 172.21.15.32:0/2209273373 shutdown_connections 2024-04-25T00:34:59.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.684+0000 7fa0e66ae640 1 --2- 172.21.15.32:0/2209273373 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0e010d2c0 0x7fa0e0114e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.684+0000 7fa0e66ae640 1 --2- 172.21.15.32:0/2209273373 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa0e0109360 0x7fa0e010cbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.684+0000 7fa0e66ae640 1 --2- 172.21.15.32:0/2209273373 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa0e01089b0 0x7fa0e0108d90 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.684+0000 7fa0e66ae640 1 -- 172.21.15.32:0/2209273373 >> 172.21.15.32:0/2209273373 conn(0x7fa0e006cab0 msgr2=0x7fa0e006cec0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:34:59.692 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.685+0000 7fa0e66ae640 1 -- 172.21.15.32:0/2209273373 shutdown_connections 2024-04-25T00:34:59.692 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.685+0000 7fa0e66ae640 1 -- 172.21.15.32:0/2209273373 wait complete. 2024-04-25T00:34:59.692 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.686+0000 7fa0e66ae640 1 Processor -- start 2024-04-25T00:34:59.693 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.686+0000 7fa0e66ae640 1 -- start start 2024-04-25T00:34:59.693 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.686+0000 7fa0e66ae640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa0e01089b0 0x7fa0e019e450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:59.693 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.686+0000 7fa0e66ae640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa0e010d2c0 0x7fa0e019e990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:59.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.686+0000 7fa0e66ae640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0e01a2d10 0x7fa0e01a31c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:59.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.686+0000 7fa0e66ae640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fa0e0117f70 con 0x7fa0e010d2c0 2024-04-25T00:34:59.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.686+0000 7fa0e66ae640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fa0e0117df0 con 0x7fa0e01089b0 2024-04-25T00:34:59.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.686+0000 7fa0e66ae640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fa0e01180f0 con 0x7fa0e01a2d10 2024-04-25T00:34:59.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.687+0000 7fa0e5ead640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0e01a2d10 0x7fa0e01a31c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:59.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.687+0000 7fa0e5ead640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0e01a2d10 0x7fa0e01a31c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:36682/0 (socket says 172.21.15.32:36682) 2024-04-25T00:34:59.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.687+0000 7fa0e5ead640 1 -- 172.21.15.32:0/2767979487 learned_addr learned my addr 172.21.15.32:0/2767979487 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:34:59.695 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.688+0000 7fa0e4eab640 1 --2- 172.21.15.32:0/2767979487 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa0e010d2c0 0x7fa0e019e990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:59.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.688+0000 7fa0e5ead640 1 -- 172.21.15.32:0/2767979487 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa0e01089b0 msgr2=0x7fa0e019e450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:59.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.688+0000 7fa0e5ead640 1 --2- 172.21.15.32:0/2767979487 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa0e01089b0 0x7fa0e019e450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.688+0000 7fa0e5ead640 1 -- 172.21.15.32:0/2767979487 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa0e010d2c0 msgr2=0x7fa0e019e990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:34:59.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.688+0000 7fa0e5ead640 1 --2- 172.21.15.32:0/2767979487 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa0e010d2c0 0x7fa0e019e990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:34:59.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.688+0000 7fa0e5ead640 1 -- 172.21.15.32:0/2767979487 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa0e01a3800 con 0x7fa0e01a2d10 2024-04-25T00:34:59.700 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.689+0000 7fa0e5ead640 1 --2- 172.21.15.32:0/2767979487 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0e01a2d10 0x7fa0e01a31c0 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7fa0d0009f40 tx=0x7fa0d000ee00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:59.702 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.690+0000 7fa0d67fc640 1 -- 172.21.15.32:0/2767979487 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa0d000ccc0 con 0x7fa0e01a2d10 2024-04-25T00:34:59.703 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.690+0000 7fa0d67fc640 1 -- 172.21.15.32:0/2767979487 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa0d000ce60 con 0x7fa0e01a2d10 2024-04-25T00:34:59.703 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.691+0000 7fa0d67fc640 1 -- 172.21.15.32:0/2767979487 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa0d0004a00 con 0x7fa0e01a2d10 2024-04-25T00:34:59.703 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.691+0000 7fa0e66ae640 1 -- 172.21.15.32:0/2767979487 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa0e01a3af0 con 0x7fa0e01a2d10 2024-04-25T00:34:59.703 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.691+0000 7fa0e66ae640 1 -- 172.21.15.32:0/2767979487 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa0e01ab260 con 0x7fa0e01a2d10 2024-04-25T00:34:59.705 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.695+0000 7fa0d67fc640 1 -- 172.21.15.32:0/2767979487 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 25) ==== 138066+0+0 (secure 0 0 0) 0x7fa0d0022020 con 0x7fa0e01a2d10 2024-04-25T00:34:59.705 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.696+0000 7fa0d67fc640 1 --2- 172.21.15.32:0/2767979487 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa0cc0a3030 0x7fa0cc0a54f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:34:59.706 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.696+0000 7fa0e66ae640 1 -- 172.21.15.32:0/2767979487 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa0e0068480 con 0x7fa0e01a2d10 2024-04-25T00:34:59.706 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.697+0000 7fa0d67fc640 1 -- 172.21.15.32:0/2767979487 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fa0d00d3b30 con 0x7fa0e01a2d10 2024-04-25T00:34:59.706 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.697+0000 7fa0e56ac640 1 --2- 172.21.15.32:0/2767979487 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa0cc0a3030 0x7fa0cc0a54f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:34:59.710 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.697+0000 7fa0e56ac640 1 --2- 172.21.15.32:0/2767979487 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa0cc0a3030 0x7fa0cc0a54f0 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7fa0dc008330 tx=0x7fa0dc000bc0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:34:59.710 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:34:59.703+0000 7fa0d67fc640 1 -- 172.21.15.32:0/2767979487 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fa0d00a02d0 con 0x7fa0e01a2d10 2024-04-25T00:34:59.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:59 smithi032 ceph-mon[29357]: pgmap v53: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 2.0 KiB/s rd, 767 MiB/s wr, 1.46k op/s 2024-04-25T00:34:59.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:34:59 smithi032 ceph-mon[29357]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:35:00.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.289+0000 7fa0e66ae640 1 -- 172.21.15.32:0/2767979487 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fa0e002d170 con 0x7fa0e01a2d10 2024-04-25T00:35:00.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.290+0000 7fa0d67fc640 1 -- 172.21.15.32:0/2767979487 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 26 v26) ==== 94+0+11877 (secure 0 0 0) 0x7fa0d009fbd0 con 0x7fa0e01a2d10 2024-04-25T00:35:00.293 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:35:00.293 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":26,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14643,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6839/187492674","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6838","nonce":187492674},{"type":"v1","addr":"172.21.15.32:6839","nonce":187492674}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14655,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6841/1948785545","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6840","nonce":1948785545},{"type":"v1","addr":"172.21.15.32:6841","nonce":1948785545}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":26,"flags":306,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":true},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-25T00:30:47.866875+0000","modified":"2024-04-25T00:34:57.347792+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24506,"mds_1":24481,"mds_2":24476},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24443":{"gid":24443,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24445":{"gid":24445,"name":"f","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24457":{"gid":24457,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24476":{"gid":24476,"name":"h","rank":2,"incarnation":22,"state":"up:active","state_seq":12,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24481":{"gid":24481,"name":"i","rank":1,"incarnation":20,"state":"up:active","state_seq":11,"addr":"172.21.15.130:6839/4267125809","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":4267125809},{"type":"v1","addr":"172.21.15.130:6839","nonce":4267125809}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24506":{"gid":24506,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.121:6839/258444853","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6838","nonce":258444853},{"type":"v1","addr":"172.21.15.121:6839","nonce":258444853}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24506,"qdb_cluster":[24481,24506,24476]},"id":1}]} 2024-04-25T00:35:00.294 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 26 2024-04-25T00:35:00.298 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.297+0000 7fa0cbfff640 1 -- 172.21.15.32:0/2767979487 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa0cc0a3030 msgr2=0x7fa0cc0a54f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:00.298 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.297+0000 7fa0cbfff640 1 --2- 172.21.15.32:0/2767979487 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa0cc0a3030 0x7fa0cc0a54f0 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7fa0dc008330 tx=0x7fa0dc000bc0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.298 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.297+0000 7fa0cbfff640 1 -- 172.21.15.32:0/2767979487 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0e01a2d10 msgr2=0x7fa0e01a31c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:00.298 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.297+0000 7fa0cbfff640 1 --2- 172.21.15.32:0/2767979487 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0e01a2d10 0x7fa0e01a31c0 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7fa0d0009f40 tx=0x7fa0d000ee00 comp rx=0 tx=0).stop 2024-04-25T00:35:00.299 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.298+0000 7fa0cbfff640 1 -- 172.21.15.32:0/2767979487 shutdown_connections 2024-04-25T00:35:00.299 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.298+0000 7fa0cbfff640 1 --2- 172.21.15.32:0/2767979487 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa0cc0a3030 0x7fa0cc0a54f0 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.299 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.298+0000 7fa0cbfff640 1 --2- 172.21.15.32:0/2767979487 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0e01a2d10 0x7fa0e01a31c0 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.299 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.298+0000 7fa0cbfff640 1 --2- 172.21.15.32:0/2767979487 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa0e010d2c0 0x7fa0e019e990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.299 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.298+0000 7fa0cbfff640 1 --2- 172.21.15.32:0/2767979487 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa0e01089b0 0x7fa0e019e450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.300 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.298+0000 7fa0cbfff640 1 -- 172.21.15.32:0/2767979487 >> 172.21.15.32:0/2767979487 conn(0x7fa0e006cab0 msgr2=0x7fa0e0071330 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:00.301 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.300+0000 7fa0cbfff640 1 -- 172.21.15.32:0/2767979487 shutdown_connections 2024-04-25T00:35:00.306 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.302+0000 7fa0cbfff640 1 -- 172.21.15.32:0/2767979487 wait complete. 2024-04-25T00:35:00.334 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-25T00:35:00.750 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.746+0000 7fee24d80640 1 -- 172.21.15.32:0/354155465 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fee2010fde0 msgr2=0x7fee201101c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:00.750 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.746+0000 7fee24d80640 1 --2- 172.21.15.32:0/354155465 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fee2010fde0 0x7fee201101c0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fee10009900 tx=0x7fee1002a4d0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.750 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.747+0000 7fee24d80640 1 -- 172.21.15.32:0/354155465 shutdown_connections 2024-04-25T00:35:00.750 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.747+0000 7fee24d80640 1 --2- 172.21.15.32:0/354155465 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fee201091c0 0x7fee2011cf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.750 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.747+0000 7fee24d80640 1 --2- 172.21.15.32:0/354155465 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fee20108740 0x7fee20108ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.751 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.747+0000 7fee24d80640 1 --2- 172.21.15.32:0/354155465 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fee2010fde0 0x7fee201101c0 secure :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fee10009900 tx=0x7fee1002a4d0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.751 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.747+0000 7fee24d80640 1 -- 172.21.15.32:0/354155465 >> 172.21.15.32:0/354155465 conn(0x7fee2006c6c0 msgr2=0x7fee2006cad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:00.758 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.757+0000 7fee24d80640 1 -- 172.21.15.32:0/354155465 shutdown_connections 2024-04-25T00:35:00.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.758+0000 7fee24d80640 1 -- 172.21.15.32:0/354155465 wait complete. 2024-04-25T00:35:00.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.759+0000 7fee24d80640 1 Processor -- start 2024-04-25T00:35:00.763 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.760+0000 7fee24d80640 1 -- start start 2024-04-25T00:35:00.763 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.760+0000 7fee24d80640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fee20108740 0x7fee2011cd20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:00.763 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.760+0000 7fee24d80640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fee201091c0 0x7fee20115da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:00.763 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.760+0000 7fee24d80640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fee201162e0 0x7fee20116790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:00.763 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.760+0000 7fee24d80640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fee20120890 con 0x7fee201162e0 2024-04-25T00:35:00.763 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.760+0000 7fee24d80640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fee20120710 con 0x7fee20108740 2024-04-25T00:35:00.763 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.760+0000 7fee24d80640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fee20120a10 con 0x7fee201091c0 2024-04-25T00:35:00.763 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.761+0000 7fee1effd640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fee201091c0 0x7fee20115da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:00.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.761+0000 7fee1effd640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fee201091c0 0x7fee20115da0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:36696/0 (socket says 172.21.15.32:36696) 2024-04-25T00:35:00.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.761+0000 7fee1effd640 1 -- 172.21.15.32:0/3184399336 learned_addr learned my addr 172.21.15.32:0/3184399336 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:00.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.761+0000 7fee1f7fe640 1 --2- 172.21.15.32:0/3184399336 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fee20108740 0x7fee2011cd20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:00.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.762+0000 7fee1effd640 1 -- 172.21.15.32:0/3184399336 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fee20108740 msgr2=0x7fee2011cd20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:00.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.762+0000 7fee1effd640 1 --2- 172.21.15.32:0/3184399336 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fee20108740 0x7fee2011cd20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.762+0000 7fee1effd640 1 -- 172.21.15.32:0/3184399336 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fee201162e0 msgr2=0x7fee20116790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:00.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.762+0000 7fee1effd640 1 --2- 172.21.15.32:0/3184399336 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fee201162e0 0x7fee20116790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.762+0000 7fee1effd640 1 -- 172.21.15.32:0/3184399336 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fee2011a940 con 0x7fee201091c0 2024-04-25T00:35:00.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.763+0000 7fee1effd640 1 --2- 172.21.15.32:0/3184399336 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fee201091c0 0x7fee20115da0 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7fee1400b4d0 tx=0x7fee1400b990 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:00.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.766+0000 7fee1cff9640 1 -- 172.21.15.32:0/3184399336 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fee14014070 con 0x7fee201091c0 2024-04-25T00:35:00.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.766+0000 7fee24d80640 1 -- 172.21.15.32:0/3184399336 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fee2011ac30 con 0x7fee201091c0 2024-04-25T00:35:00.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.766+0000 7fee24d80640 1 -- 172.21.15.32:0/3184399336 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fee2011b180 con 0x7fee201091c0 2024-04-25T00:35:00.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.766+0000 7fee1cff9640 1 -- 172.21.15.32:0/3184399336 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fee14010d50 con 0x7fee201091c0 2024-04-25T00:35:00.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.768+0000 7fee1cff9640 1 -- 172.21.15.32:0/3184399336 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fee14017360 con 0x7fee201091c0 2024-04-25T00:35:00.772 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.770+0000 7fee1cff9640 1 -- 172.21.15.32:0/3184399336 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 25) ==== 138066+0+0 (secure 0 0 0) 0x7fee14017070 con 0x7fee201091c0 2024-04-25T00:35:00.777 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.775+0000 7fee1cff9640 1 --2- 172.21.15.32:0/3184399336 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fee080a2ed0 0x7fee080a5390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:00.777 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.775+0000 7fee28dc5740 1 Processor -- start 2024-04-25T00:35:00.778 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.776+0000 7fee28dc5740 1 -- start start 2024-04-25T00:35:00.778 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.776+0000 7fee28dc5740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557fec750720 0x557fec750b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:00.778 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.776+0000 7fee28dc5740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557fec751040 0x557fec75b1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:00.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.776+0000 7fee28dc5740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557fec75b710 0x557fec75db00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:00.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.776+0000 7fee28dc5740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x557fec6b5670 con 0x557fec751040 2024-04-25T00:35:00.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.776+0000 7fee28dc5740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x557fec65d970 con 0x557fec75b710 2024-04-25T00:35:00.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.776+0000 7fee28dc5740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x557fec6b1630 con 0x557fec750720 2024-04-25T00:35:00.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.777+0000 7fee1cff9640 1 -- 172.21.15.32:0/3184399336 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fee140cefe0 con 0x7fee201091c0 2024-04-25T00:35:00.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.777+0000 7fee1f7fe640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557fec750720 0x557fec750b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:00.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.777+0000 7fee1f7fe640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557fec750720 0x557fec750b00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:36708/0 (socket says 172.21.15.32:36708) 2024-04-25T00:35:00.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.777+0000 7fee1f7fe640 1 -- 172.21.15.32:0/2471025094 learned_addr learned my addr 172.21.15.32:0/2471025094 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:00.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.777+0000 7fee1f7fe640 1 -- 172.21.15.32:0/2471025094 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557fec75b710 msgr2=0x557fec75db00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:00.780 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.777+0000 7fee1f7fe640 1 --2- 172.21.15.32:0/2471025094 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557fec75b710 0x557fec75db00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.780 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.777+0000 7fee1f7fe640 1 -- 172.21.15.32:0/2471025094 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557fec751040 msgr2=0x557fec75b1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:00.780 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.777+0000 7fee1f7fe640 1 --2- 172.21.15.32:0/2471025094 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557fec751040 0x557fec75b1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.780 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.777+0000 7fee1f7fe640 1 -- 172.21.15.32:0/2471025094 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557fec681400 con 0x557fec750720 2024-04-25T00:35:00.780 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.777+0000 7fee1f7fe640 1 --2- 172.21.15.32:0/3184399336 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fee080a2ed0 0x7fee080a5390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:00.781 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.778+0000 7fee1f7fe640 1 --2- 172.21.15.32:0/2471025094 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557fec750720 0x557fec750b00 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7fee2006d2a0 tx=0x7fee10005c00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e81bc23fcc706513 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:00.781 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.780+0000 7fee05ffb640 1 -- 172.21.15.32:0/2471025094 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fee1004d070 con 0x557fec750720 2024-04-25T00:35:00.782 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.780+0000 7fee05ffb640 1 -- 172.21.15.32:0/2471025094 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fee10004040 con 0x557fec750720 2024-04-25T00:35:00.782 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.780+0000 7fee05ffb640 1 -- 172.21.15.32:0/2471025094 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fee10035040 con 0x557fec750720 2024-04-25T00:35:00.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.781+0000 7fee28dc5740 1 -- 172.21.15.32:0/2471025094 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557fec750720 msgr2=0x557fec750b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:00.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.781+0000 7fee28dc5740 1 --2- 172.21.15.32:0/2471025094 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557fec750720 0x557fec750b00 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7fee2006d2a0 tx=0x7fee10005c00 comp rx=0 tx=0).stop 2024-04-25T00:35:00.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.781+0000 7fee1f7fe640 1 --2- 172.21.15.32:0/3184399336 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fee080a2ed0 0x7fee080a5390 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7fee1004a000 tx=0x7fee10003e70 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:00.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.782+0000 7fee28dc5740 1 -- 172.21.15.32:0/2471025094 shutdown_connections 2024-04-25T00:35:00.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.782+0000 7fee28dc5740 1 --2- 172.21.15.32:0/2471025094 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557fec75b710 0x557fec75db00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.782+0000 7fee28dc5740 1 --2- 172.21.15.32:0/2471025094 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557fec751040 0x557fec75b1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.782+0000 7fee28dc5740 1 --2- 172.21.15.32:0/2471025094 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557fec750720 0x557fec750b00 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.782+0000 7fee28dc5740 1 -- 172.21.15.32:0/2471025094 >> 172.21.15.32:0/2471025094 conn(0x557fec74ca40 msgr2=0x557fec74ce20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:00.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.788+0000 7fee28dc5740 1 -- 172.21.15.32:0/2471025094 shutdown_connections 2024-04-25T00:35:00.792 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.789+0000 7fee28dc5740 1 -- 172.21.15.32:0/2471025094 wait complete. 2024-04-25T00:35:00.792 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.790+0000 7fee28dc5740 1 -- start start 2024-04-25T00:35:00.792 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.790+0000 7fee28dc5740 1 Processor -- start 2024-04-25T00:35:00.800 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.798+0000 7fee28dc5740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557fec750720 0x557fec75b020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:00.800 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.798+0000 7fee28dc5740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557fec751040 0x557fec758310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:00.800 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.798+0000 7fee28dc5740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557fec75b710 0x557fec758850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:00.800 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.798+0000 7fee28dc5740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x557fec75e500 con 0x557fec750720 2024-04-25T00:35:00.800 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.798+0000 7fee28dc5740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x557fec671050 con 0x557fec75b710 2024-04-25T00:35:00.800 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.798+0000 7fee28dc5740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x557fec75e680 con 0x557fec751040 2024-04-25T00:35:00.800 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.798+0000 7fee28dc5740 10 client.0 _ll_register_callbacks cb 0x557fec3c8be0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:35:00.800 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.798+0000 7fee1effd640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557fec751040 0x557fec758310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:00.800 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.798+0000 7fee1ffff640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557fec75b710 0x557fec758850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:00.800 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.798+0000 7fee1effd640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557fec751040 0x557fec758310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:36712/0 (socket says 172.21.15.32:36712) 2024-04-25T00:35:00.800 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.798+0000 7fee1effd640 1 -- 172.21.15.32:0/3705284956 learned_addr learned my addr 172.21.15.32:0/3705284956 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:00.801 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.799+0000 7fee1effd640 1 -- 172.21.15.32:0/3705284956 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557fec75b710 msgr2=0x557fec758850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:00.801 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.799+0000 7fee1effd640 1 --2- 172.21.15.32:0/3705284956 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557fec75b710 0x557fec758850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.801 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.799+0000 7fee1effd640 1 -- 172.21.15.32:0/3705284956 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557fec750720 msgr2=0x557fec75b020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:00.801 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.799+0000 7fee1effd640 1 --2- 172.21.15.32:0/3705284956 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557fec750720 0x557fec75b020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.801 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.799+0000 7fee1effd640 1 -- 172.21.15.32:0/3705284956 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557fec642dc0 con 0x557fec751040 2024-04-25T00:35:00.801 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.799+0000 7fee1effd640 1 --2- 172.21.15.32:0/3705284956 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557fec751040 0x557fec758310 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7fee1409af40 tx=0x7fee1400fe90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8ce511dff25f26d1 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:00.805 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.801+0000 7fedfeffd640 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:35:00.805 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.801+0000 7fedfeffd640 1 -- 172.21.15.32:0/3705284956 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fee1401e070 con 0x557fec751040 2024-04-25T00:35:00.805 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.801+0000 7fedfeffd640 1 -- 172.21.15.32:0/3705284956 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fee1409a400 con 0x557fec751040 2024-04-25T00:35:00.805 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.803+0000 7fee28dc5740 1 -- 172.21.15.32:0/3705284956 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fee10004040 con 0x557fec751040 2024-04-25T00:35:00.807 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.805+0000 7fedfeffd640 1 -- 172.21.15.32:0/3705284956 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7fee140bb0b0 con 0x557fec751040 2024-04-25T00:35:00.808 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.806+0000 7fee28dc5740 10 client.25519 fetch_fsmap learned FSMap version 26 2024-04-25T00:35:00.808 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.806+0000 7fee28dc5740 1 -- 172.21.15.32:0/3705284956 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x557fec3c93f0 con 0x557fec751040 2024-04-25T00:35:00.810 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.806+0000 7fedfeffd640 1 -- 172.21.15.32:0/3705284956 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fee10004040 con 0x557fec751040 2024-04-25T00:35:00.810 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.808+0000 7fedfeffd640 1 -- 172.21.15.32:0/3705284956 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7fee140b8070 con 0x557fec751040 2024-04-25T00:35:00.810 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.808+0000 7fee28dc5740 10 client.25519 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:35:00.810 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.808+0000 7fee28dc5740 10 client.25519 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-25T00:35:00.810 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.808+0000 7fee28dc5740 20 client.25519 populate_metadata read hostname 'smithi032' 2024-04-25T00:35:00.811 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.808+0000 7fee28dc5740 1 --2- 172.21.15.32:0/3705284956 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x557fec76bc30 0x557fec76e070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:00.811 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.808+0000 7fee28dc5740 4 client.25519 mds_command: new command op to 24476 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:35:00.811 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.808+0000 7fee28dc5740 1 -- 172.21.15.32:0/3705284956 --> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x557fec6c2bb0 con 0x557fec76bc30 2024-04-25T00:35:00.819 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.815+0000 7fee1f7fe640 1 --2- 172.21.15.32:0/3705284956 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x557fec76bc30 0x557fec76e070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:00.821 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.818+0000 7fee1f7fe640 1 --2- 172.21.15.32:0/3705284956 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x557fec76bc30 0x557fec76e070 crc :-1 s=READY pgs=38 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=94c2eb4bbec24c1b server_cookie=b69caa1f745a141c in_seq=0 out_seq=0 2024-04-25T00:35:00.821 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.818+0000 7fedfeffd640 10 client.25519 ms_handle_connect on v2:172.21.15.121:6836/3456022860 2024-04-25T00:35:00.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.819+0000 7fedfeffd640 1 -- 172.21.15.32:0/3705284956 <== mds.2 v2:172.21.15.121:6836/3456022860 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x557fec6c2bb0 con 0x557fec76bc30 2024-04-25T00:35:00.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.819+0000 7fedfeffd640 10 client.25519 handle_command_reply: tid=0 2024-04-25T00:35:00.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.819+0000 7fee28dc5740 1 client.25519 shutdown 2024-04-25T00:35:00.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.820+0000 7fee28dc5740 1 -- 172.21.15.32:0/3705284956 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557fec751040 msgr2=0x557fec758310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:00.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.820+0000 7fee28dc5740 1 --2- 172.21.15.32:0/3705284956 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557fec751040 0x557fec758310 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7fee1409af40 tx=0x7fee1400fe90 comp rx=0 tx=0).stop 2024-04-25T00:35:00.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.820+0000 7fee28dc5740 1 -- 172.21.15.32:0/3705284956 shutdown_connections 2024-04-25T00:35:00.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.820+0000 7fee28dc5740 1 --2- 172.21.15.32:0/3705284956 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x557fec76bc30 0x557fec76e070 crc :-1 s=READY pgs=38 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.820+0000 7fee28dc5740 1 --2- 172.21.15.32:0/3705284956 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557fec75b710 0x557fec758850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.820+0000 7fee28dc5740 1 --2- 172.21.15.32:0/3705284956 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557fec751040 0x557fec758310 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.820+0000 7fee28dc5740 1 --2- 172.21.15.32:0/3705284956 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557fec750720 0x557fec75b020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.820+0000 7fee28dc5740 1 -- 172.21.15.32:0/3705284956 >> 172.21.15.32:0/3705284956 conn(0x557fec74ca40 msgr2=0x557fec752740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:00.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.820+0000 7fedfeffd640 0 client.25519 ms_handle_reset on v2:172.21.15.121:6836/3456022860 2024-04-25T00:35:00.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.820+0000 7fee28dc5740 1 -- 172.21.15.32:0/3705284956 shutdown_connections 2024-04-25T00:35:00.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.820+0000 7fee28dc5740 1 -- 172.21.15.32:0/3705284956 wait complete. 2024-04-25T00:35:00.823 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.820+0000 7fee28dc5740 20 client.25519 trim_cache size 0 max 16384 2024-04-25T00:35:00.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.852+0000 7fee28dc5740 1 Processor -- start 2024-04-25T00:35:00.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.852+0000 7fee28dc5740 1 -- start start 2024-04-25T00:35:00.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.852+0000 7fee28dc5740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557fec750720 0x557fec767420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:00.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.852+0000 7fee28dc5740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557fec751040 0x557fec753880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:00.856 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.852+0000 7fee28dc5740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557fec75b710 0x557fec75ab50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:00.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.852+0000 7fee28dc5740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fee0c012720 con 0x557fec750720 2024-04-25T00:35:00.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.852+0000 7fee28dc5740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fee0c00fff0 con 0x557fec751040 2024-04-25T00:35:00.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.852+0000 7fee28dc5740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fee0c00def0 con 0x557fec75b710 2024-04-25T00:35:00.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.853+0000 7fee1effd640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557fec751040 0x557fec753880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:00.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.853+0000 7fee1effd640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557fec751040 0x557fec753880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:42684/0 (socket says 172.21.15.32:42684) 2024-04-25T00:35:00.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.853+0000 7fee1effd640 1 -- 172.21.15.32:0/1960876683 learned_addr learned my addr 172.21.15.32:0/1960876683 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:00.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.854+0000 7fee1effd640 1 -- 172.21.15.32:0/1960876683 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557fec75b710 msgr2=0x557fec75ab50 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-25T00:35:00.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.854+0000 7fee1effd640 1 --2- 172.21.15.32:0/1960876683 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557fec75b710 0x557fec75ab50 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.854+0000 7fee1effd640 1 -- 172.21.15.32:0/1960876683 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557fec750720 msgr2=0x557fec767420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:00.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.854+0000 7fee1effd640 1 --2- 172.21.15.32:0/1960876683 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557fec750720 0x557fec767420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.854+0000 7fee1effd640 1 -- 172.21.15.32:0/1960876683 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557fec4937d0 con 0x557fec751040 2024-04-25T00:35:00.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.854+0000 7fee1effd640 1 --2- 172.21.15.32:0/1960876683 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557fec751040 0x557fec753880 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7fee1409b170 tx=0x7fee14006da0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d0605c2e8b09d72c server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:00.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.854+0000 7fee057fa640 1 -- 172.21.15.32:0/1960876683 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fee1401e070 con 0x557fec751040 2024-04-25T00:35:00.857 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.854+0000 7fee057fa640 1 -- 172.21.15.32:0/1960876683 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fee140a3c20 con 0x557fec751040 2024-04-25T00:35:00.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.855+0000 7fee28dc5740 1 -- 172.21.15.32:0/1960876683 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557fec751040 msgr2=0x557fec753880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:00.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.855+0000 7fee28dc5740 1 --2- 172.21.15.32:0/1960876683 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557fec751040 0x557fec753880 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7fee1409b170 tx=0x7fee14006da0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.857+0000 7fee28dc5740 1 -- 172.21.15.32:0/1960876683 shutdown_connections 2024-04-25T00:35:00.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.857+0000 7fee28dc5740 1 --2- 172.21.15.32:0/1960876683 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557fec75b710 0x557fec75ab50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.857+0000 7fee28dc5740 1 --2- 172.21.15.32:0/1960876683 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557fec751040 0x557fec753880 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.857+0000 7fee28dc5740 1 --2- 172.21.15.32:0/1960876683 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557fec750720 0x557fec767420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.857+0000 7fee28dc5740 1 -- 172.21.15.32:0/1960876683 >> 172.21.15.32:0/1960876683 conn(0x557fec74ca40 msgr2=0x557fec753170 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:00.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.858+0000 7fee28dc5740 1 -- 172.21.15.32:0/1960876683 shutdown_connections 2024-04-25T00:35:00.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.859+0000 7fee28dc5740 1 -- 172.21.15.32:0/1960876683 wait complete. 2024-04-25T00:35:00.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.859+0000 7fee28dc5740 1 -- start start 2024-04-25T00:35:00.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.860+0000 7fee28dc5740 1 Processor -- start 2024-04-25T00:35:00.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.860+0000 7fee28dc5740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557fec750720 0x557fec7639b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:00.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.860+0000 7fee28dc5740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557fec751040 0x557fec75ee90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:00.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.860+0000 7fee28dc5740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557fec75b710 0x557fec75f3d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:00.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.860+0000 7fee28dc5740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fee0c00e4b0 con 0x557fec750720 2024-04-25T00:35:00.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.860+0000 7fee28dc5740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fee0c00f920 con 0x557fec751040 2024-04-25T00:35:00.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.860+0000 7fee28dc5740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fee0c00eb80 con 0x557fec75b710 2024-04-25T00:35:00.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.860+0000 7fee28dc5740 10 client.0 _ll_register_callbacks cb 0x7fee14097250 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:35:00.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.861+0000 7fee1effd640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557fec751040 0x557fec75ee90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:00.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.861+0000 7fee1effd640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557fec751040 0x557fec75ee90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:42690/0 (socket says 172.21.15.32:42690) 2024-04-25T00:35:00.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.861+0000 7fee1effd640 1 -- 172.21.15.32:0/3384707118 learned_addr learned my addr 172.21.15.32:0/3384707118 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:00.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.861+0000 7fee1effd640 1 -- 172.21.15.32:0/3384707118 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557fec75b710 msgr2=0x557fec75f3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:00.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.861+0000 7fee1effd640 1 --2- 172.21.15.32:0/3384707118 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557fec75b710 0x557fec75f3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.861+0000 7fee1effd640 1 -- 172.21.15.32:0/3384707118 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557fec750720 msgr2=0x557fec7639b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:00.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.861+0000 7fee1effd640 1 --2- 172.21.15.32:0/3384707118 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557fec750720 0x557fec7639b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.861+0000 7fee1effd640 1 -- 172.21.15.32:0/3384707118 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557fec75f910 con 0x557fec751040 2024-04-25T00:35:00.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.862+0000 7fee1effd640 1 --2- 172.21.15.32:0/3384707118 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557fec751040 0x557fec75ee90 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7fee14097980 tx=0x7fee14096c20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7760653255a7dacb server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:00.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.864+0000 7fedfeffd640 10 client.0 ms_handle_connect on v2:172.21.15.121:3300/0 2024-04-25T00:35:00.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.864+0000 7fedfeffd640 1 -- 172.21.15.32:0/3384707118 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fee1401e040 con 0x557fec751040 2024-04-25T00:35:00.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.864+0000 7fedfeffd640 1 -- 172.21.15.32:0/3384707118 <== mon.2 v2:172.21.15.121:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fee1400f3c0 con 0x557fec751040 2024-04-25T00:35:00.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.864+0000 7fee28dc5740 1 -- 172.21.15.32:0/3384707118 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x557fec758be0 con 0x557fec751040 2024-04-25T00:35:00.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.866+0000 7fedfeffd640 1 -- 172.21.15.32:0/3384707118 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7fee140a3c20 con 0x557fec751040 2024-04-25T00:35:00.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.866+0000 7fee28dc5740 10 client.25541 fetch_fsmap learned FSMap version 26 2024-04-25T00:35:00.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.866+0000 7fee28dc5740 1 -- 172.21.15.32:0/3384707118 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x557fec75f910 con 0x557fec751040 2024-04-25T00:35:00.873 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.870+0000 7fedfeffd640 1 -- 172.21.15.32:0/3384707118 <== mon.2 v2:172.21.15.121:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x557fec758be0 con 0x557fec751040 2024-04-25T00:35:00.873 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.871+0000 7fedfeffd640 1 -- 172.21.15.32:0/3384707118 <== mon.2 v2:172.21.15.121:3300/0 5 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7fee140b8040 con 0x557fec751040 2024-04-25T00:35:00.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.871+0000 7fee28dc5740 10 client.25541 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:35:00.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.871+0000 7fee28dc5740 10 client.25541 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-25T00:35:00.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.871+0000 7fee28dc5740 20 client.25541 populate_metadata read hostname 'smithi032' 2024-04-25T00:35:00.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.871+0000 7fee28dc5740 1 --2- 172.21.15.32:0/3384707118 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x557fec7d21a0 0x557fec7d4590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:00.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.871+0000 7fee28dc5740 4 client.25541 mds_command: new command op to 24476 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-25T00:35:00.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.871+0000 7fee28dc5740 1 -- 172.21.15.32:0/3384707118 --> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] -- command(tid 0: {"prefix": "damage ls"}) -- 0x557fec6bcf00 con 0x557fec7d21a0 2024-04-25T00:35:00.878 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.871+0000 7fee1f7fe640 1 --2- 172.21.15.32:0/3384707118 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x557fec7d21a0 0x557fec7d4590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:00.878 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.872+0000 7fee1f7fe640 1 --2- 172.21.15.32:0/3384707118 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x557fec7d21a0 0x557fec7d4590 crc :-1 s=READY pgs=39 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=d8aa27beb389a8f2 server_cookie=c6781c4055304691 in_seq=0 out_seq=0 2024-04-25T00:35:00.878 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.875+0000 7fedfeffd640 10 client.25541 ms_handle_connect on v2:172.21.15.121:6836/3456022860 2024-04-25T00:35:00.878 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.875+0000 7fedfeffd640 1 -- 172.21.15.32:0/3384707118 <== mds.2 v2:172.21.15.121:6836/3456022860 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x557fec6bcf00 con 0x557fec7d21a0 2024-04-25T00:35:00.878 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.875+0000 7fedfeffd640 10 client.25541 handle_command_reply: tid=0 2024-04-25T00:35:00.879 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.875+0000 7fee28dc5740 1 client.25541 shutdown 2024-04-25T00:35:00.884 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.877+0000 7fee28dc5740 1 -- 172.21.15.32:0/3384707118 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557fec751040 msgr2=0x557fec75ee90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:00.884 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.877+0000 7fee28dc5740 1 --2- 172.21.15.32:0/3384707118 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557fec751040 0x557fec75ee90 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7fee14097980 tx=0x7fee14096c20 comp rx=0 tx=0).stop 2024-04-25T00:35:00.884 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.878+0000 7fee28dc5740 1 -- 172.21.15.32:0/3384707118 shutdown_connections 2024-04-25T00:35:00.884 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.878+0000 7fee28dc5740 1 --2- 172.21.15.32:0/3384707118 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x557fec7d21a0 0x557fec7d4590 crc :-1 s=READY pgs=39 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.884 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.878+0000 7fee28dc5740 1 --2- 172.21.15.32:0/3384707118 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557fec75b710 0x557fec75f3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.884 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.878+0000 7fee28dc5740 1 --2- 172.21.15.32:0/3384707118 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557fec751040 0x557fec75ee90 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.884 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.878+0000 7fee28dc5740 1 --2- 172.21.15.32:0/3384707118 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557fec750720 0x557fec7639b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.884 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.878+0000 7fee28dc5740 1 -- 172.21.15.32:0/3384707118 >> 172.21.15.32:0/3384707118 conn(0x557fec74ca40 msgr2=0x557fec74b330 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:00.884 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.880+0000 7fedfeffd640 0 client.25541 ms_handle_reset on v2:172.21.15.121:6836/3456022860 2024-04-25T00:35:00.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.884+0000 7fee28dc5740 1 -- 172.21.15.32:0/3384707118 shutdown_connections 2024-04-25T00:35:00.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.887+0000 7fee28dc5740 1 -- 172.21.15.32:0/3384707118 wait complete. 2024-04-25T00:35:00.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.889+0000 7fee28dc5740 20 client.25541 trim_cache size 0 max 16384 2024-04-25T00:35:00.912 INFO:teuthology.orchestra.run.smithi032.stdout:[] 2024-04-25T00:35:00.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.924+0000 7fee24d80640 1 -- 172.21.15.32:0/3184399336 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fee080a2ed0 msgr2=0x7fee080a5390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:00.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.924+0000 7fee24d80640 1 --2- 172.21.15.32:0/3184399336 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fee080a2ed0 0x7fee080a5390 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7fee1004a000 tx=0x7fee10003e70 comp rx=0 tx=0).stop 2024-04-25T00:35:00.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.924+0000 7fee24d80640 1 -- 172.21.15.32:0/3184399336 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fee201091c0 msgr2=0x7fee20115da0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:00.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.924+0000 7fee24d80640 1 --2- 172.21.15.32:0/3184399336 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fee201091c0 0x7fee20115da0 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7fee1400b4d0 tx=0x7fee1400b990 comp rx=0 tx=0).stop 2024-04-25T00:35:00.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.925+0000 7fee24d80640 1 -- 172.21.15.32:0/3184399336 shutdown_connections 2024-04-25T00:35:00.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.925+0000 7fee24d80640 1 --2- 172.21.15.32:0/3184399336 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fee080a2ed0 0x7fee080a5390 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.925+0000 7fee24d80640 1 --2- 172.21.15.32:0/3184399336 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fee201162e0 0x7fee20116790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.925+0000 7fee24d80640 1 --2- 172.21.15.32:0/3184399336 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fee201091c0 0x7fee20115da0 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.925+0000 7fee24d80640 1 --2- 172.21.15.32:0/3184399336 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fee20108740 0x7fee2011cd20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:00.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.925+0000 7fee24d80640 1 -- 172.21.15.32:0/3184399336 >> 172.21.15.32:0/3184399336 conn(0x7fee2006c6c0 msgr2=0x7fee200718e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:00.931 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.929+0000 7fee24d80640 1 -- 172.21.15.32:0/3184399336 shutdown_connections 2024-04-25T00:35:00.931 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:00.930+0000 7fee24d80640 1 -- 172.21.15.32:0/3184399336 wait complete. 2024-04-25T00:35:00.990 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-25T00:35:01.352 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.350+0000 7f821bfff640 1 -- 172.21.15.32:0/2248027981 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f821c1089b0 msgr2=0x7f821c108d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:01.352 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.350+0000 7f821bfff640 1 --2- 172.21.15.32:0/2248027981 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f821c1089b0 0x7f821c108d90 secure :-1 s=READY pgs=588 cs=0 l=1 rev1=1 crypto rx=0x7f820c005ee0 tx=0x7f820c02a6e0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.352 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.350+0000 7f821b7fe640 1 --2- 172.21.15.32:0/2248027981 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f821c10cf00 0x7f821c0799c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:35:01.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.352+0000 7f821bfff640 1 -- 172.21.15.32:0/2248027981 shutdown_connections 2024-04-25T00:35:01.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.352+0000 7f821bfff640 1 --2- 172.21.15.32:0/2248027981 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f821c10cf00 0x7f821c0799c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.352+0000 7f821bfff640 1 --2- 172.21.15.32:0/2248027981 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f821c109360 0x7f821c10c820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.352+0000 7f821bfff640 1 --2- 172.21.15.32:0/2248027981 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f821c1089b0 0x7f821c108d90 unknown :-1 s=CLOSED pgs=588 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.352+0000 7f821bfff640 1 -- 172.21.15.32:0/2248027981 >> 172.21.15.32:0/2248027981 conn(0x7f821c06c6c0 msgr2=0x7f821c06cad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:01.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.359+0000 7f821bfff640 1 -- 172.21.15.32:0/2248027981 shutdown_connections 2024-04-25T00:35:01.367 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.365+0000 7f821bfff640 1 -- 172.21.15.32:0/2248027981 wait complete. 2024-04-25T00:35:01.369 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.366+0000 7f821bfff640 1 Processor -- start 2024-04-25T00:35:01.371 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.368+0000 7f821bfff640 1 -- start start 2024-04-25T00:35:01.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.369+0000 7f821bfff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f821c1089b0 0x7f821c077e80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:01.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.369+0000 7f821bfff640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f821c10cf00 0x7f821c0783c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:01.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.369+0000 7f821bfff640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f821c079b20 0x7f821c1c3fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:01.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.369+0000 7f821bfff640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f821c081380 con 0x7f821c1089b0 2024-04-25T00:35:01.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.369+0000 7f821bfff640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f821c081200 con 0x7f821c079b20 2024-04-25T00:35:01.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.369+0000 7f821bfff640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f821c081500 con 0x7f821c10cf00 2024-04-25T00:35:01.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.369+0000 7f821affd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f821c1089b0 0x7f821c077e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:01.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.369+0000 7f821affd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f821c1089b0 0x7f821c077e80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:59010/0 (socket says 172.21.15.32:59010) 2024-04-25T00:35:01.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.369+0000 7f821affd640 1 -- 172.21.15.32:0/997129757 learned_addr learned my addr 172.21.15.32:0/997129757 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:01.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.370+0000 7f821affd640 1 -- 172.21.15.32:0/997129757 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f821c10cf00 msgr2=0x7f821c0783c0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:35:01.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.370+0000 7f821affd640 1 --2- 172.21.15.32:0/997129757 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f821c10cf00 0x7f821c0783c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.370+0000 7f821affd640 1 -- 172.21.15.32:0/997129757 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f821c079b20 msgr2=0x7f821c1c3fe0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:35:01.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.370+0000 7f821affd640 1 --2- 172.21.15.32:0/997129757 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f821c079b20 0x7f821c1c3fe0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.370+0000 7f821affd640 1 -- 172.21.15.32:0/997129757 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f821c1b57a0 con 0x7f821c1089b0 2024-04-25T00:35:01.374 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.370+0000 7f821affd640 1 --2- 172.21.15.32:0/997129757 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f821c1089b0 0x7f821c077e80 secure :-1 s=READY pgs=589 cs=0 l=1 rev1=1 crypto rx=0x7f820c02ac50 tx=0x7f820c031610 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:01.374 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.372+0000 7f8203fff640 1 -- 172.21.15.32:0/997129757 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f820c004030 con 0x7f821c1089b0 2024-04-25T00:35:01.374 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.372+0000 7f821bfff640 1 -- 172.21.15.32:0/997129757 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f821c1b5a30 con 0x7f821c1089b0 2024-04-25T00:35:01.374 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.372+0000 7f821bfff640 1 -- 172.21.15.32:0/997129757 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f821c1b5f20 con 0x7f821c1089b0 2024-04-25T00:35:01.375 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.372+0000 7f8203fff640 1 -- 172.21.15.32:0/997129757 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f820c02f9f0 con 0x7f821c1089b0 2024-04-25T00:35:01.375 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.373+0000 7f8203fff640 1 -- 172.21.15.32:0/997129757 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f820c004990 con 0x7f821c1089b0 2024-04-25T00:35:01.379 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.377+0000 7f82245c9740 1 Processor -- start 2024-04-25T00:35:01.382 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.381+0000 7f8203fff640 1 -- 172.21.15.32:0/997129757 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 25) ==== 138066+0+0 (secure 0 0 0) 0x7f820c039040 con 0x7f821c1089b0 2024-04-25T00:35:01.387 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.385+0000 7f82245c9740 1 -- start start 2024-04-25T00:35:01.387 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.385+0000 7f82245c9740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55cd10aee720 0x55cd10aeeb00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:01.387 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.385+0000 7f82245c9740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55cd10aef040 0x55cd10af91d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:01.387 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.385+0000 7f82245c9740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cd10af9710 0x55cd10afbb00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:01.387 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.386+0000 7f82245c9740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55cd10a53670 con 0x55cd10aef040 2024-04-25T00:35:01.387 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.386+0000 7f82245c9740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55cd109fb970 con 0x55cd10aee720 2024-04-25T00:35:01.387 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.386+0000 7f82245c9740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55cd10a4f630 con 0x55cd10af9710 2024-04-25T00:35:01.388 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.386+0000 7f821affd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55cd10aef040 0x55cd10af91d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:01.388 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.386+0000 7f821affd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55cd10aef040 0x55cd10af91d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:59014/0 (socket says 172.21.15.32:59014) 2024-04-25T00:35:01.388 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.386+0000 7f821affd640 1 -- 172.21.15.32:0/2882061952 learned_addr learned my addr 172.21.15.32:0/2882061952 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:01.388 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.386+0000 7f821affd640 1 -- 172.21.15.32:0/2882061952 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cd10af9710 msgr2=0x55cd10afbb00 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-25T00:35:01.388 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.387+0000 7f821affd640 1 --2- 172.21.15.32:0/2882061952 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cd10af9710 0x55cd10afbb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.388 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.387+0000 7f821affd640 1 -- 172.21.15.32:0/2882061952 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55cd10aee720 msgr2=0x55cd10aeeb00 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-25T00:35:01.388 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.387+0000 7f821affd640 1 --2- 172.21.15.32:0/2882061952 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55cd10aee720 0x55cd10aeeb00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.389 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.387+0000 7f821affd640 1 -- 172.21.15.32:0/2882061952 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55cd10a1f400 con 0x55cd10aef040 2024-04-25T00:35:01.389 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.387+0000 7f8203fff640 1 --2- 172.21.15.32:0/997129757 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f82040a2e00 0x7f82040a52c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:01.389 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.387+0000 7f821affd640 1 --2- 172.21.15.32:0/2882061952 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55cd10aef040 0x55cd10af91d0 secure :-1 s=READY pgs=590 cs=0 l=1 rev1=1 crypto rx=0x7f820c0edc20 tx=0x7f820c0f6cd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5e8341807c79ea6 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:01.389 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.387+0000 7f8203fff640 1 -- 172.21.15.32:0/997129757 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f820c0f0970 con 0x7f821c1089b0 2024-04-25T00:35:01.389 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.387+0000 7f821b7fe640 1 --2- 172.21.15.32:0/997129757 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f82040a2e00 0x7f82040a52c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:01.395 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.394+0000 7f821b7fe640 1 --2- 172.21.15.32:0/997129757 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f82040a2e00 0x7f82040a52c0 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f8214009370 tx=0x7f8214009240 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:01.399 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.398+0000 7f82017fa640 1 -- 172.21.15.32:0/2882061952 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f820c0bc730 con 0x55cd10aef040 2024-04-25T00:35:01.400 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.398+0000 7f82017fa640 1 -- 172.21.15.32:0/2882061952 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f820c0041d0 con 0x55cd10aef040 2024-04-25T00:35:01.400 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.398+0000 7f82017fa640 1 -- 172.21.15.32:0/2882061952 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f820c0bc2c0 con 0x55cd10aef040 2024-04-25T00:35:01.401 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.399+0000 7f82245c9740 1 -- 172.21.15.32:0/2882061952 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55cd10aef040 msgr2=0x55cd10af91d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:01.401 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.399+0000 7f82245c9740 1 --2- 172.21.15.32:0/2882061952 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55cd10aef040 0x55cd10af91d0 secure :-1 s=READY pgs=590 cs=0 l=1 rev1=1 crypto rx=0x7f820c0edc20 tx=0x7f820c0f6cd0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.401 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.400+0000 7f82245c9740 1 -- 172.21.15.32:0/2882061952 shutdown_connections 2024-04-25T00:35:01.402 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.400+0000 7f82245c9740 1 --2- 172.21.15.32:0/2882061952 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cd10af9710 0x55cd10afbb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.402 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.400+0000 7f82245c9740 1 --2- 172.21.15.32:0/2882061952 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55cd10aef040 0x55cd10af91d0 unknown :-1 s=CLOSED pgs=590 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.402 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.400+0000 7f82245c9740 1 --2- 172.21.15.32:0/2882061952 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55cd10aee720 0x55cd10aeeb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.402 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.400+0000 7f82245c9740 1 -- 172.21.15.32:0/2882061952 >> 172.21.15.32:0/2882061952 conn(0x55cd10aeaa40 msgr2=0x55cd10aeae20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:01.402 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.401+0000 7f82245c9740 1 -- 172.21.15.32:0/2882061952 shutdown_connections 2024-04-25T00:35:01.403 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.401+0000 7f82245c9740 1 -- 172.21.15.32:0/2882061952 wait complete. 2024-04-25T00:35:01.403 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.402+0000 7f82245c9740 1 -- start start 2024-04-25T00:35:01.403 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.402+0000 7f82245c9740 1 Processor -- start 2024-04-25T00:35:01.404 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.402+0000 7f82245c9740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cd10aee720 0x55cd10af9050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:01.404 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.402+0000 7f82245c9740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55cd10aef040 0x55cd10af6340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:01.404 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.402+0000 7f82245c9740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55cd10af9710 0x55cd10af6880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:01.404 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.403+0000 7f82245c9740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55cd10afc2c0 con 0x55cd10af9710 2024-04-25T00:35:01.404 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.403+0000 7f82245c9740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55cd10a0f050 con 0x55cd10aef040 2024-04-25T00:35:01.405 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.403+0000 7f82245c9740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55cd10afc440 con 0x55cd10aee720 2024-04-25T00:35:01.405 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.403+0000 7f821affd640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cd10aee720 0x55cd10af9050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:01.405 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.403+0000 7f82245c9740 10 client.0 _ll_register_callbacks cb 0x55cd10766be0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:35:01.405 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.403+0000 7f821affd640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cd10aee720 0x55cd10af9050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:36782/0 (socket says 172.21.15.32:36782) 2024-04-25T00:35:01.405 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.403+0000 7f821affd640 1 -- 172.21.15.32:0/45441683 learned_addr learned my addr 172.21.15.32:0/45441683 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:01.406 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.403+0000 7f821affd640 1 -- 172.21.15.32:0/45441683 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55cd10aef040 msgr2=0x55cd10af6340 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-25T00:35:01.407 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.403+0000 7f821affd640 1 --2- 172.21.15.32:0/45441683 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55cd10aef040 0x55cd10af6340 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.407 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.403+0000 7f821affd640 1 -- 172.21.15.32:0/45441683 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55cd10af9710 msgr2=0x55cd10af6880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:01.407 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.403+0000 7f821affd640 1 --2- 172.21.15.32:0/45441683 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55cd10af9710 0x55cd10af6880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.407 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.403+0000 7f821affd640 1 -- 172.21.15.32:0/45441683 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55cd109e0dc0 con 0x55cd10aee720 2024-04-25T00:35:01.407 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.404+0000 7f821affd640 1 --2- 172.21.15.32:0/45441683 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cd10aee720 0x55cd10af9050 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f820c009a90 tx=0x7f820c0f6340 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9f25a1adc29f3516 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:01.407 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.405+0000 7f81feff5640 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:35:01.407 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.405+0000 7f81feff5640 1 -- 172.21.15.32:0/45441683 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f820c0bc4a0 con 0x55cd10aee720 2024-04-25T00:35:01.407 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.405+0000 7f81feff5640 1 -- 172.21.15.32:0/45441683 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f820c0bcbc0 con 0x55cd10aee720 2024-04-25T00:35:01.408 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.405+0000 7f82245c9740 1 -- 172.21.15.32:0/45441683 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x55cd10a60bb0 con 0x55cd10aee720 2024-04-25T00:35:01.409 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.406+0000 7f81feff5640 1 -- 172.21.15.32:0/45441683 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7f820c0bc2c0 con 0x55cd10aee720 2024-04-25T00:35:01.409 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.407+0000 7f82245c9740 10 client.25525 fetch_fsmap learned FSMap version 26 2024-04-25T00:35:01.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.407+0000 7f82245c9740 1 -- 172.21.15.32:0/45441683 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x55cd107673f0 con 0x55cd10aee720 2024-04-25T00:35:01.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.407+0000 7f81feff5640 1 -- 172.21.15.32:0/45441683 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x55cd10a60bb0 con 0x55cd10aee720 2024-04-25T00:35:01.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.407+0000 7f81feff5640 1 -- 172.21.15.32:0/45441683 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7f820c0c5140 con 0x55cd10aee720 2024-04-25T00:35:01.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.407+0000 7f82245c9740 10 client.25525 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:35:01.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.407+0000 7f82245c9740 10 client.25525 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-25T00:35:01.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.407+0000 7f82245c9740 20 client.25525 populate_metadata read hostname 'smithi032' 2024-04-25T00:35:01.411 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.408+0000 7f82245c9740 1 --2- 172.21.15.32:0/45441683 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55cd10b09430 0x55cd10b0b880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:01.411 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.408+0000 7f82245c9740 4 client.25525 mds_command: new command op to 24481 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:35:01.411 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.408+0000 7f82245c9740 1 -- 172.21.15.32:0/45441683 --> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55cd10a5af00 con 0x55cd10b09430 2024-04-25T00:35:01.411 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.408+0000 7f821a7fc640 1 --2- 172.21.15.32:0/45441683 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55cd10b09430 0x55cd10b0b880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:01.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.410+0000 7f81feff5640 10 client.25525 ms_handle_connect on v2:172.21.15.130:6838/4267125809 2024-04-25T00:35:01.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.410+0000 7f821a7fc640 1 --2- 172.21.15.32:0/45441683 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55cd10b09430 0x55cd10b0b880 crc :-1 s=READY pgs=34 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=f5512096ed247e6b server_cookie=56e3ee36c0ca60a0 in_seq=0 out_seq=0 2024-04-25T00:35:01.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.412+0000 7f81feff5640 1 -- 172.21.15.32:0/45441683 <== mds.1 v2:172.21.15.130:6838/4267125809 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x55cd10a5af00 con 0x55cd10b09430 2024-04-25T00:35:01.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.412+0000 7f81feff5640 10 client.25525 handle_command_reply: tid=0 2024-04-25T00:35:01.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.412+0000 7f82245c9740 1 client.25525 shutdown 2024-04-25T00:35:01.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.412+0000 7f82245c9740 1 -- 172.21.15.32:0/45441683 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cd10aee720 msgr2=0x55cd10af9050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:01.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.412+0000 7f82245c9740 1 --2- 172.21.15.32:0/45441683 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cd10aee720 0x55cd10af9050 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f820c009a90 tx=0x7f820c0f6340 comp rx=0 tx=0).stop 2024-04-25T00:35:01.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.413+0000 7f82245c9740 1 -- 172.21.15.32:0/45441683 shutdown_connections 2024-04-25T00:35:01.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.413+0000 7f82245c9740 1 --2- 172.21.15.32:0/45441683 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55cd10b09430 0x55cd10b0b880 crc :-1 s=READY pgs=34 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.413+0000 7f82245c9740 1 --2- 172.21.15.32:0/45441683 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55cd10af9710 0x55cd10af6880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.413+0000 7f82245c9740 1 --2- 172.21.15.32:0/45441683 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55cd10aef040 0x55cd10af6340 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.413+0000 7f82245c9740 1 --2- 172.21.15.32:0/45441683 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cd10aee720 0x55cd10af9050 secure :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f820c009a90 tx=0x7f820c0f6340 comp rx=0 tx=0).stop 2024-04-25T00:35:01.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.413+0000 7f82245c9740 1 -- 172.21.15.32:0/45441683 >> 172.21.15.32:0/45441683 conn(0x55cd10aeaa40 msgr2=0x55cd10ae9bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:01.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.416+0000 7f81feff5640 0 client.25525 ms_handle_reset on v2:172.21.15.130:6838/4267125809 2024-04-25T00:35:01.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.420+0000 7f82245c9740 1 -- 172.21.15.32:0/45441683 shutdown_connections 2024-04-25T00:35:01.426 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.424+0000 7f82245c9740 1 -- 172.21.15.32:0/45441683 wait complete. 2024-04-25T00:35:01.427 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.425+0000 7f82245c9740 20 client.25525 trim_cache size 0 max 16384 2024-04-25T00:35:01.457 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.455+0000 7f82245c9740 1 Processor -- start 2024-04-25T00:35:01.457 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.455+0000 7f82245c9740 1 -- start start 2024-04-25T00:35:01.457 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.456+0000 7f82245c9740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cd10aeaa40 0x55cd10b05180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:01.457 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.456+0000 7f82245c9740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55cd10aef040 0x55cd10af8b80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:01.458 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.456+0000 7f82245c9740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55cd10af9710 0x55cd10b11190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:01.458 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.456+0000 7f82245c9740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f8208012940 con 0x55cd10aef040 2024-04-25T00:35:01.459 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.457+0000 7f82245c9740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f8208010210 con 0x55cd10af9710 2024-04-25T00:35:01.459 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.457+0000 7f82245c9740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f820800e060 con 0x55cd10aeaa40 2024-04-25T00:35:01.459 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.457+0000 7f821affd640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cd10aeaa40 0x55cd10b05180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:01.459 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.457+0000 7f821affd640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cd10aeaa40 0x55cd10b05180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:36784/0 (socket says 172.21.15.32:36784) 2024-04-25T00:35:01.459 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.457+0000 7f821affd640 1 -- 172.21.15.32:0/2079293188 learned_addr learned my addr 172.21.15.32:0/2079293188 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:01.459 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.457+0000 7f821b7fe640 1 --2- 172.21.15.32:0/2079293188 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55cd10af9710 0x55cd10b11190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:01.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.461+0000 7f821a7fc640 1 --2- 172.21.15.32:0/2079293188 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55cd10aef040 0x55cd10af8b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:01.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.461+0000 7f821b7fe640 1 -- 172.21.15.32:0/2079293188 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cd10aeaa40 msgr2=0x55cd10b05180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:01.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.461+0000 7f821b7fe640 1 --2- 172.21.15.32:0/2079293188 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cd10aeaa40 0x55cd10b05180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.461+0000 7f821b7fe640 1 -- 172.21.15.32:0/2079293188 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55cd10aef040 msgr2=0x55cd10af8b80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:01.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.461+0000 7f821b7fe640 1 --2- 172.21.15.32:0/2079293188 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55cd10aef040 0x55cd10af8b80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.461+0000 7f821b7fe640 1 -- 172.21.15.32:0/2079293188 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55cd108317d0 con 0x55cd10af9710 2024-04-25T00:35:01.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.461+0000 7f821a7fc640 1 --2- 172.21.15.32:0/2079293188 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55cd10aef040 0x55cd10af8b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:35:01.466 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.465+0000 7f821b7fe640 1 --2- 172.21.15.32:0/2079293188 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55cd10af9710 0x55cd10b11190 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x55cd10af0fe0 tx=0x7f8214004a40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5149271aee2978c8 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:01.467 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.465+0000 7f8200ff9640 1 -- 172.21.15.32:0/2079293188 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8214021020 con 0x55cd10af9710 2024-04-25T00:35:01.467 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.465+0000 7f8200ff9640 1 -- 172.21.15.32:0/2079293188 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8214016030 con 0x55cd10af9710 2024-04-25T00:35:01.468 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.466+0000 7f8200ff9640 1 -- 172.21.15.32:0/2079293188 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f821400f720 con 0x55cd10af9710 2024-04-25T00:35:01.469 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.468+0000 7f82245c9740 1 -- 172.21.15.32:0/2079293188 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55cd10af9710 msgr2=0x55cd10b11190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:01.470 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.468+0000 7f82245c9740 1 --2- 172.21.15.32:0/2079293188 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55cd10af9710 0x55cd10b11190 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x55cd10af0fe0 tx=0x7f8214004a40 comp rx=0 tx=0).stop 2024-04-25T00:35:01.473 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.471+0000 7f82245c9740 1 -- 172.21.15.32:0/2079293188 shutdown_connections 2024-04-25T00:35:01.473 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.471+0000 7f82245c9740 1 --2- 172.21.15.32:0/2079293188 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55cd10af9710 0x55cd10b11190 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.473 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.471+0000 7f82245c9740 1 --2- 172.21.15.32:0/2079293188 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55cd10aef040 0x55cd10af8b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.473 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.471+0000 7f82245c9740 1 --2- 172.21.15.32:0/2079293188 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cd10aeaa40 0x55cd10b05180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.473 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.471+0000 7f82245c9740 1 -- 172.21.15.32:0/2079293188 >> 172.21.15.32:0/2079293188 conn(0x55cd10af69e0 msgr2=0x55cd10ae9bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:01.478 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.475+0000 7f82245c9740 1 -- 172.21.15.32:0/2079293188 shutdown_connections 2024-04-25T00:35:01.480 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.478+0000 7f82245c9740 1 -- 172.21.15.32:0/2079293188 wait complete. 2024-04-25T00:35:01.480 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.479+0000 7f82245c9740 1 -- start start 2024-04-25T00:35:01.480 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.479+0000 7f82245c9740 1 Processor -- start 2024-04-25T00:35:01.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.479+0000 7f82245c9740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55cd10aeaa40 0x55cd10b01980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:01.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.479+0000 7f82245c9740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55cd10aef040 0x55cd10af2cf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:01.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.480+0000 7f82245c9740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cd10af9710 0x55cd10af3230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:01.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.480+0000 7f82245c9740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f820800e6d0 con 0x55cd10aef040 2024-04-25T00:35:01.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.480+0000 7f82245c9740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f820800fb40 con 0x55cd10aeaa40 2024-04-25T00:35:01.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.480+0000 7f82245c9740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f820800eda0 con 0x55cd10af9710 2024-04-25T00:35:01.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.480+0000 7f82245c9740 10 client.0 _ll_register_callbacks cb 0x7f820c0f6a90 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:35:01.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.480+0000 7f821affd640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55cd10aeaa40 0x55cd10b01980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:01.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.480+0000 7f821b7fe640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cd10af9710 0x55cd10af3230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:01.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.480+0000 7f821b7fe640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cd10af9710 0x55cd10af3230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:36796/0 (socket says 172.21.15.32:36796) 2024-04-25T00:35:01.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.480+0000 7f821b7fe640 1 -- 172.21.15.32:0/3748362177 learned_addr learned my addr 172.21.15.32:0/3748362177 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:01.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.480+0000 7f821affd640 1 -- 172.21.15.32:0/3748362177 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cd10af9710 msgr2=0x55cd10af3230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:01.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.480+0000 7f821affd640 1 --2- 172.21.15.32:0/3748362177 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cd10af9710 0x55cd10af3230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.480+0000 7f821affd640 1 -- 172.21.15.32:0/3748362177 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55cd10aef040 msgr2=0x55cd10af2cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:01.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.480+0000 7f821affd640 1 --2- 172.21.15.32:0/3748362177 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55cd10aef040 0x55cd10af2cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.480+0000 7f821affd640 1 -- 172.21.15.32:0/3748362177 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55cd10af3970 con 0x55cd10aeaa40 2024-04-25T00:35:01.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.484+0000 7f821affd640 1 --2- 172.21.15.32:0/3748362177 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55cd10aeaa40 0x55cd10b01980 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f820c0edca0 tx=0x7f820c0db4b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d8210cc5e31a716b server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:01.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.487+0000 7f81feff5640 10 client.0 ms_handle_connect on v2:172.21.15.121:3300/0 2024-04-25T00:35:01.489 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.487+0000 7f81feff5640 1 -- 172.21.15.32:0/3748362177 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f820c0ca060 con 0x55cd10aeaa40 2024-04-25T00:35:01.489 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.487+0000 7f81feff5640 1 -- 172.21.15.32:0/3748362177 <== mon.2 v2:172.21.15.121:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f820c0bb820 con 0x55cd10aeaa40 2024-04-25T00:35:01.489 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.487+0000 7f82245c9740 1 -- 172.21.15.32:0/3748362177 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8214016030 con 0x55cd10aeaa40 2024-04-25T00:35:01.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.491+0000 7f81feff5640 1 -- 172.21.15.32:0/3748362177 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7f820c0bc730 con 0x55cd10aeaa40 2024-04-25T00:35:01.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.491+0000 7f82245c9740 10 client.25550 fetch_fsmap learned FSMap version 26 2024-04-25T00:35:01.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.491+0000 7f82245c9740 1 -- 172.21.15.32:0/3748362177 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x55cd10af3970 con 0x55cd10aeaa40 2024-04-25T00:35:01.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.491+0000 7f81feff5640 1 -- 172.21.15.32:0/3748362177 <== mon.2 v2:172.21.15.121:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8214016030 con 0x55cd10aeaa40 2024-04-25T00:35:01.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.491+0000 7f81feff5640 1 -- 172.21.15.32:0/3748362177 <== mon.2 v2:172.21.15.121:3300/0 5 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7f820c0ede70 con 0x55cd10aeaa40 2024-04-25T00:35:01.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.491+0000 7f82245c9740 10 client.25550 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:35:01.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.491+0000 7f82245c9740 10 client.25550 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-25T00:35:01.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.491+0000 7f82245c9740 20 client.25550 populate_metadata read hostname 'smithi032' 2024-04-25T00:35:01.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.492+0000 7f82245c9740 1 --2- 172.21.15.32:0/3748362177 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55cd10b6fbe0 0x55cd10b71fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:01.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.492+0000 7f82245c9740 4 client.25550 mds_command: new command op to 24481 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-25T00:35:01.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.492+0000 7f82245c9740 1 -- 172.21.15.32:0/3748362177 --> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55cd109d8820 con 0x55cd10b6fbe0 2024-04-25T00:35:01.497 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.495+0000 7f821a7fc640 1 --2- 172.21.15.32:0/3748362177 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55cd10b6fbe0 0x55cd10b71fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:01.504 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.502+0000 7f821a7fc640 1 --2- 172.21.15.32:0/3748362177 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55cd10b6fbe0 0x55cd10b71fd0 crc :-1 s=READY pgs=35 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=2429510d986bfecb server_cookie=f7a9dc2902c66f07 in_seq=0 out_seq=0 2024-04-25T00:35:01.506 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.503+0000 7f81feff5640 10 client.25550 ms_handle_connect on v2:172.21.15.130:6838/4267125809 2024-04-25T00:35:01.506 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.503+0000 7f81feff5640 1 -- 172.21.15.32:0/3748362177 <== mds.1 v2:172.21.15.130:6838/4267125809 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55cd109d8820 con 0x55cd10b6fbe0 2024-04-25T00:35:01.506 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.503+0000 7f81feff5640 10 client.25550 handle_command_reply: tid=0 2024-04-25T00:35:01.506 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.503+0000 7f82245c9740 1 client.25550 shutdown 2024-04-25T00:35:01.507 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.506+0000 7f82245c9740 1 -- 172.21.15.32:0/3748362177 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55cd10aeaa40 msgr2=0x55cd10b01980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:01.507 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.506+0000 7f82245c9740 1 --2- 172.21.15.32:0/3748362177 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55cd10aeaa40 0x55cd10b01980 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f820c0edca0 tx=0x7f820c0db4b0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.516 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.514+0000 7f82245c9740 1 -- 172.21.15.32:0/3748362177 shutdown_connections 2024-04-25T00:35:01.516 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.514+0000 7f82245c9740 1 --2- 172.21.15.32:0/3748362177 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cd10af9710 0x55cd10af3230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.516 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.514+0000 7f82245c9740 1 --2- 172.21.15.32:0/3748362177 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55cd10b6fbe0 0x55cd10b71fd0 crc :-1 s=READY pgs=35 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.516 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.514+0000 7f82245c9740 1 --2- 172.21.15.32:0/3748362177 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55cd10aef040 0x55cd10af2cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.516 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.514+0000 7f82245c9740 1 --2- 172.21.15.32:0/3748362177 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55cd10aeaa40 0x55cd10b01980 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.516 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.514+0000 7f81feff5640 0 client.25550 ms_handle_reset on v2:172.21.15.130:6838/4267125809 2024-04-25T00:35:01.516 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.514+0000 7f82245c9740 1 -- 172.21.15.32:0/3748362177 >> 172.21.15.32:0/3748362177 conn(0x55cd10af69e0 msgr2=0x55cd10af1010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:01.519 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.517+0000 7f82245c9740 1 -- 172.21.15.32:0/3748362177 shutdown_connections 2024-04-25T00:35:01.519 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.517+0000 7f82245c9740 1 -- 172.21.15.32:0/3748362177 wait complete. 2024-04-25T00:35:01.520 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.519+0000 7f82245c9740 20 client.25550 trim_cache size 0 max 16384 2024-04-25T00:35:01.527 INFO:teuthology.orchestra.run.smithi032.stdout:[] 2024-04-25T00:35:01.534 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.533+0000 7f821bfff640 1 -- 172.21.15.32:0/997129757 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f82040a2e00 msgr2=0x7f82040a52c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:01.534 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.533+0000 7f821bfff640 1 --2- 172.21.15.32:0/997129757 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f82040a2e00 0x7f82040a52c0 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f8214009370 tx=0x7f8214009240 comp rx=0 tx=0).stop 2024-04-25T00:35:01.534 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.533+0000 7f821bfff640 1 -- 172.21.15.32:0/997129757 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f821c1089b0 msgr2=0x7f821c077e80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:01.535 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.533+0000 7f821bfff640 1 --2- 172.21.15.32:0/997129757 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f821c1089b0 0x7f821c077e80 secure :-1 s=READY pgs=589 cs=0 l=1 rev1=1 crypto rx=0x7f820c02ac50 tx=0x7f820c031610 comp rx=0 tx=0).stop 2024-04-25T00:35:01.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.534+0000 7f821bfff640 1 -- 172.21.15.32:0/997129757 shutdown_connections 2024-04-25T00:35:01.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.534+0000 7f821bfff640 1 --2- 172.21.15.32:0/997129757 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f82040a2e00 0x7f82040a52c0 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.534+0000 7f821bfff640 1 --2- 172.21.15.32:0/997129757 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f821c079b20 0x7f821c1c3fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.534+0000 7f821bfff640 1 --2- 172.21.15.32:0/997129757 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f821c10cf00 0x7f821c0783c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.534+0000 7f821bfff640 1 --2- 172.21.15.32:0/997129757 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f821c1089b0 0x7f821c077e80 unknown :-1 s=CLOSED pgs=589 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.535+0000 7f821bfff640 1 -- 172.21.15.32:0/997129757 >> 172.21.15.32:0/997129757 conn(0x7f821c06c6c0 msgr2=0x7f821c072720 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:01.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.539+0000 7f821bfff640 1 -- 172.21.15.32:0/997129757 shutdown_connections 2024-04-25T00:35:01.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.539+0000 7f821bfff640 1 -- 172.21.15.32:0/997129757 wait complete. 2024-04-25T00:35:01.568 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:01 smithi032 ceph-mon[29357]: pgmap v54: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 2.4 KiB/s rd, 1.0 GiB/s wr, 2.00k op/s 2024-04-25T00:35:01.568 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:01 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2767979487' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:35:01.570 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-25T00:35:01.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:01 smithi121 ceph-mon[32231]: pgmap v54: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 2.4 KiB/s rd, 1.0 GiB/s wr, 2.00k op/s 2024-04-25T00:35:01.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:01 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2767979487' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:35:01.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:01 smithi130 ceph-mon[32525]: pgmap v54: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 2.4 KiB/s rd, 1.0 GiB/s wr, 2.00k op/s 2024-04-25T00:35:01.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:01 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2767979487' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:35:01.977 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.970+0000 7f1f94353640 1 -- 172.21.15.32:0/2948667397 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1f8c11af20 msgr2=0x7f1f8c11d310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:01.978 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.970+0000 7f1f94353640 1 --2- 172.21.15.32:0/2948667397 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1f8c11af20 0x7f1f8c11d310 secure :-1 s=READY pgs=591 cs=0 l=1 rev1=1 crypto rx=0x7f1f7c009930 tx=0x7f1f7c02a460 comp rx=0 tx=0).stop 2024-04-25T00:35:01.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.979+0000 7f1f94353640 1 -- 172.21.15.32:0/2948667397 shutdown_connections 2024-04-25T00:35:01.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.979+0000 7f1f94353640 1 --2- 172.21.15.32:0/2948667397 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1f8c11af20 0x7f1f8c11d310 unknown :-1 s=CLOSED pgs=591 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.979+0000 7f1f94353640 1 --2- 172.21.15.32:0/2948667397 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1f8c10dfc0 0x7f1f8c10e440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.979+0000 7f1f94353640 1 --2- 172.21.15.32:0/2948667397 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1f8c10ff90 0x7f1f8c10da80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:01.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.979+0000 7f1f94353640 1 -- 172.21.15.32:0/2948667397 >> 172.21.15.32:0/2948667397 conn(0x7f1f8c06c850 msgr2=0x7f1f8c06cc60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:01.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.983+0000 7f1f94353640 1 -- 172.21.15.32:0/2948667397 shutdown_connections 2024-04-25T00:35:01.986 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.985+0000 7f1f94353640 1 -- 172.21.15.32:0/2948667397 wait complete. 2024-04-25T00:35:01.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.986+0000 7f1f94353640 1 Processor -- start 2024-04-25T00:35:01.989 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.988+0000 7f1f94353640 1 -- start start 2024-04-25T00:35:01.990 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.988+0000 7f1f94353640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1f8c10dfc0 0x7f1f8c1a6ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:01.990 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.989+0000 7f1f94353640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1f8c10ff90 0x7f1f8c1a73e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:01.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.989+0000 7f1f94353640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1f8c11af20 0x7f1f8c1ab760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:01.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.989+0000 7f1f94353640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f1f8c120920 con 0x7f1f8c10ff90 2024-04-25T00:35:01.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.989+0000 7f1f94353640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f1f8c1207a0 con 0x7f1f8c11af20 2024-04-25T00:35:01.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.989+0000 7f1f94353640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f1f8c120aa0 con 0x7f1f8c10dfc0 2024-04-25T00:35:01.994 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.992+0000 7f1f920ef640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1f8c10dfc0 0x7f1f8c1a6ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:01.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.994+0000 7f1f920ef640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1f8c10dfc0 0x7f1f8c1a6ea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:54658/0 (socket says 172.21.15.32:54658) 2024-04-25T00:35:01.997 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.994+0000 7f1f920ef640 1 -- 172.21.15.32:0/2624429287 learned_addr learned my addr 172.21.15.32:0/2624429287 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:01.997 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.994+0000 7f1f928f0640 1 --2- 172.21.15.32:0/2624429287 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1f8c11af20 0x7f1f8c1ab760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:01.998 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.996+0000 7f1f918ee640 1 --2- 172.21.15.32:0/2624429287 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1f8c10ff90 0x7f1f8c1a73e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:02.002 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.999+0000 7f1f920ef640 1 -- 172.21.15.32:0/2624429287 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1f8c11af20 msgr2=0x7f1f8c1ab760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:02.002 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.999+0000 7f1f920ef640 1 --2- 172.21.15.32:0/2624429287 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1f8c11af20 0x7f1f8c1ab760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.002 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.999+0000 7f1f920ef640 1 -- 172.21.15.32:0/2624429287 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1f8c10ff90 msgr2=0x7f1f8c1a73e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:02.002 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.999+0000 7f1f920ef640 1 --2- 172.21.15.32:0/2624429287 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1f8c10ff90 0x7f1f8c1a73e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.002 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:01.999+0000 7f1f920ef640 1 -- 172.21.15.32:0/2624429287 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1f8c1abe90 con 0x7f1f8c10dfc0 2024-04-25T00:35:02.008 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.004+0000 7f1f920ef640 1 --2- 172.21.15.32:0/2624429287 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1f8c10dfc0 0x7f1f8c1a6ea0 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f1f8800b450 tx=0x7f1f8800b910 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:02.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.007+0000 7f1f837fe640 1 -- 172.21.15.32:0/2624429287 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1f88015020 con 0x7f1f8c10dfc0 2024-04-25T00:35:02.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.007+0000 7f1f837fe640 1 -- 172.21.15.32:0/2624429287 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1f88011d70 con 0x7f1f8c10dfc0 2024-04-25T00:35:02.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.007+0000 7f1f837fe640 1 -- 172.21.15.32:0/2624429287 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1f88015020 con 0x7f1f8c10dfc0 2024-04-25T00:35:02.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.010+0000 7f1f94353640 1 -- 172.21.15.32:0/2624429287 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1f8c1ac180 con 0x7f1f8c10dfc0 2024-04-25T00:35:02.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.011+0000 7f1f94353640 1 -- 172.21.15.32:0/2624429287 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1f8c1b3a10 con 0x7f1f8c10dfc0 2024-04-25T00:35:02.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.018+0000 7f1f837fe640 1 -- 172.21.15.32:0/2624429287 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 25) ==== 138066+0+0 (secure 0 0 0) 0x7f1f88018020 con 0x7f1f8c10dfc0 2024-04-25T00:35:02.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.018+0000 7f1f9705d740 1 Processor -- start 2024-04-25T00:35:02.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.019+0000 7f1f837fe640 1 --2- 172.21.15.32:0/2624429287 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f1f780a3270 0x7f1f780a5730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:02.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.020+0000 7f1f928f0640 1 --2- 172.21.15.32:0/2624429287 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f1f780a3270 0x7f1f780a5730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:02.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.020+0000 7f1f837fe640 1 -- 172.21.15.32:0/2624429287 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f1f88011690 con 0x7f1f8c10dfc0 2024-04-25T00:35:02.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.021+0000 7f1f9705d740 1 -- start start 2024-04-25T00:35:02.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.021+0000 7f1f9705d740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5605aa82d580 0x5605aa82d960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:02.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.021+0000 7f1f9705d740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82dea0 0x5605aa838030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:02.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.021+0000 7f1f9705d740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5605aa838570 0x5605aa83a960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:02.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.021+0000 7f1f9705d740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5605aa792670 con 0x5605aa82d580 2024-04-25T00:35:02.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.021+0000 7f1f9705d740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5605aa73a970 con 0x5605aa82dea0 2024-04-25T00:35:02.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.021+0000 7f1f9705d740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5605aa78e630 con 0x5605aa838570 2024-04-25T00:35:02.025 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.024+0000 7f1f928f0640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5605aa838570 0x5605aa83a960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:02.026 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.025+0000 7f1f918ee640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82dea0 0x5605aa838030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:02.027 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.026+0000 7f1f918ee640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82dea0 0x5605aa838030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:43816/0 (socket says 172.21.15.32:43816) 2024-04-25T00:35:02.027 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.026+0000 7f1f920ef640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5605aa82d580 0x5605aa82d960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:02.027 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.026+0000 7f1f918ee640 1 -- 172.21.15.32:0/3985404190 learned_addr learned my addr 172.21.15.32:0/3985404190 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:02.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.026+0000 7f1f920ef640 1 -- 172.21.15.32:0/3985404190 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5605aa838570 msgr2=0x5605aa83a960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:02.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.026+0000 7f1f920ef640 1 --2- 172.21.15.32:0/3985404190 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5605aa838570 0x5605aa83a960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.026+0000 7f1f920ef640 1 -- 172.21.15.32:0/3985404190 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82dea0 msgr2=0x5605aa838030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:02.031 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.026+0000 7f1f920ef640 1 --2- 172.21.15.32:0/3985404190 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82dea0 0x5605aa838030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.031 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.026+0000 7f1f920ef640 1 -- 172.21.15.32:0/3985404190 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5605aa75e400 con 0x5605aa82d580 2024-04-25T00:35:02.031 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.027+0000 7f1f920ef640 1 --2- 172.21.15.32:0/3985404190 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5605aa82d580 0x5605aa82d960 secure :-1 s=READY pgs=592 cs=0 l=1 rev1=1 crypto rx=0x7f1f880cbfc0 tx=0x7f1f880a0190 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1c88e5474ce63c17 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:02.039 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.034+0000 7f1f928f0640 1 --2- 172.21.15.32:0/2624429287 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f1f780a3270 0x7f1f780a5730 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x5605aa82aa30 tx=0x7f1f7c0095e0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:02.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.041+0000 7f1f80bfe640 1 -- 172.21.15.32:0/3985404190 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1f880976f0 con 0x5605aa82d580 2024-04-25T00:35:02.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.041+0000 7f1f80bfe640 1 -- 172.21.15.32:0/3985404190 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1f8801ee50 con 0x5605aa82d580 2024-04-25T00:35:02.042 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.041+0000 7f1f80bfe640 1 -- 172.21.15.32:0/3985404190 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1f88097890 con 0x5605aa82d580 2024-04-25T00:35:02.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.042+0000 7f1f9705d740 1 -- 172.21.15.32:0/3985404190 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5605aa82d580 msgr2=0x5605aa82d960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:02.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.042+0000 7f1f9705d740 1 --2- 172.21.15.32:0/3985404190 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5605aa82d580 0x5605aa82d960 secure :-1 s=READY pgs=592 cs=0 l=1 rev1=1 crypto rx=0x7f1f880cbfc0 tx=0x7f1f880a0190 comp rx=0 tx=0).stop 2024-04-25T00:35:02.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.047+0000 7f1f9705d740 1 -- 172.21.15.32:0/3985404190 shutdown_connections 2024-04-25T00:35:02.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.047+0000 7f1f9705d740 1 --2- 172.21.15.32:0/3985404190 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5605aa838570 0x5605aa83a960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.047+0000 7f1f9705d740 1 --2- 172.21.15.32:0/3985404190 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82dea0 0x5605aa838030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.047+0000 7f1f9705d740 1 --2- 172.21.15.32:0/3985404190 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5605aa82d580 0x5605aa82d960 unknown :-1 s=CLOSED pgs=592 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.047+0000 7f1f9705d740 1 -- 172.21.15.32:0/3985404190 >> 172.21.15.32:0/3985404190 conn(0x5605aa8298a0 msgr2=0x5605aa829c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:02.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.047+0000 7f1f9705d740 1 -- 172.21.15.32:0/3985404190 shutdown_connections 2024-04-25T00:35:02.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.047+0000 7f1f9705d740 1 -- 172.21.15.32:0/3985404190 wait complete. 2024-04-25T00:35:02.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.049+0000 7f1f9705d740 1 -- start start 2024-04-25T00:35:02.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.049+0000 7f1f9705d740 1 Processor -- start 2024-04-25T00:35:02.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.050+0000 7f1f9705d740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82d580 0x5605aa837fa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:02.052 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.050+0000 7f1f9705d740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5605aa82dea0 0x5605aa835290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:02.052 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.050+0000 7f1f9705d740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5605aa838570 0x5605aa8359d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:02.052 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.051+0000 7f1f9705d740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5605aa83b300 con 0x5605aa82dea0 2024-04-25T00:35:02.052 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.051+0000 7f1f9705d740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5605aa74e050 con 0x5605aa82d580 2024-04-25T00:35:02.052 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.051+0000 7f1f9705d740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5605aa83b480 con 0x5605aa838570 2024-04-25T00:35:02.053 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.051+0000 7f1f9705d740 10 client.0 _ll_register_callbacks cb 0x5605aa4a5be0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:35:02.053 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.051+0000 7f1f920ef640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82d580 0x5605aa837fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:02.053 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.051+0000 7f1f920ef640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82d580 0x5605aa837fa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:43826/0 (socket says 172.21.15.32:43826) 2024-04-25T00:35:02.053 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.051+0000 7f1f920ef640 1 -- 172.21.15.32:0/3807806701 learned_addr learned my addr 172.21.15.32:0/3807806701 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:02.053 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.052+0000 7f1f918ee640 1 --2- 172.21.15.32:0/3807806701 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5605aa82dea0 0x5605aa835290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:02.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.055+0000 7f1f928f0640 1 --2- 172.21.15.32:0/3807806701 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5605aa838570 0x5605aa8359d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:02.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.056+0000 7f1f928f0640 1 -- 172.21.15.32:0/3807806701 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82d580 msgr2=0x5605aa837fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:02.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.056+0000 7f1f928f0640 1 --2- 172.21.15.32:0/3807806701 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82d580 0x5605aa837fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.056+0000 7f1f928f0640 1 -- 172.21.15.32:0/3807806701 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5605aa82dea0 msgr2=0x5605aa835290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:02.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.056+0000 7f1f928f0640 1 --2- 172.21.15.32:0/3807806701 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5605aa82dea0 0x5605aa835290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.056+0000 7f1f928f0640 1 -- 172.21.15.32:0/3807806701 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5605aa71fdc0 con 0x5605aa838570 2024-04-25T00:35:02.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.056+0000 7f1f928f0640 1 --2- 172.21.15.32:0/3807806701 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5605aa838570 0x5605aa8359d0 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f1f7c009230 tx=0x7f1f7c009260 comp rx=0 tx=0).ready entity=mon.1 client_cookie=44294949d4711842 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:02.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.056+0000 7f1f918ee640 1 --2- 172.21.15.32:0/3807806701 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5605aa82dea0 0x5605aa835290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:35:02.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.057+0000 7f1f75ffb640 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:35:02.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.058+0000 7f1f75ffb640 1 -- 172.21.15.32:0/3807806701 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1f7c04c020 con 0x5605aa838570 2024-04-25T00:35:02.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.059+0000 7f1f9705d740 1 -- 172.21.15.32:0/3807806701 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1f8801ee50 con 0x5605aa838570 2024-04-25T00:35:02.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.059+0000 7f1f75ffb640 1 -- 172.21.15.32:0/3807806701 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f1f7c004980 con 0x5605aa838570 2024-04-25T00:35:02.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.060+0000 7f1f75ffb640 1 -- 172.21.15.32:0/3807806701 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7f1f7c057050 con 0x5605aa838570 2024-04-25T00:35:02.067 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.064+0000 7f1f9705d740 10 client.25537 fetch_fsmap learned FSMap version 26 2024-04-25T00:35:02.067 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.064+0000 7f1f9705d740 1 -- 172.21.15.32:0/3807806701 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x5605aa4a63f0 con 0x5605aa838570 2024-04-25T00:35:02.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.067+0000 7f1f75ffb640 1 -- 172.21.15.32:0/3807806701 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1f8801ee50 con 0x5605aa838570 2024-04-25T00:35:02.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.068+0000 7f1f75ffb640 1 -- 172.21.15.32:0/3807806701 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7f1f7c0035f0 con 0x5605aa838570 2024-04-25T00:35:02.076 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.075+0000 7f1f9705d740 10 client.25537 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:35:02.077 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.075+0000 7f1f9705d740 10 client.25537 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:35:02.077 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.075+0000 7f1f9705d740 20 client.25537 populate_metadata read hostname 'smithi032' 2024-04-25T00:35:02.077 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.075+0000 7f1f9705d740 1 --2- 172.21.15.32:0/3807806701 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5605aa848610 0x5605aa84aa30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:02.077 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.075+0000 7f1f9705d740 4 client.25537 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:35:02.077 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.075+0000 7f1f9705d740 1 -- 172.21.15.32:0/3807806701 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5605aa79fbb0 con 0x5605aa848610 2024-04-25T00:35:02.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.082+0000 7f1f920ef640 1 --2- 172.21.15.32:0/3807806701 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5605aa848610 0x5605aa84aa30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:02.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.083+0000 7f1f920ef640 1 --2- 172.21.15.32:0/3807806701 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5605aa848610 0x5605aa84aa30 crc :-1 s=READY pgs=63 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e421e9914c56730e server_cookie=8adafacbb1a4fb4c in_seq=0 out_seq=0 2024-04-25T00:35:02.085 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.084+0000 7f1f75ffb640 10 client.25537 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:35:02.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.085+0000 7f1f75ffb640 1 -- 172.21.15.32:0/3807806701 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x5605aa79fbb0 con 0x5605aa848610 2024-04-25T00:35:02.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.085+0000 7f1f75ffb640 10 client.25537 handle_command_reply: tid=0 2024-04-25T00:35:02.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.085+0000 7f1f9705d740 1 client.25537 shutdown 2024-04-25T00:35:02.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.086+0000 7f1f9705d740 1 -- 172.21.15.32:0/3807806701 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5605aa838570 msgr2=0x5605aa8359d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:02.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.086+0000 7f1f9705d740 1 --2- 172.21.15.32:0/3807806701 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5605aa838570 0x5605aa8359d0 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f1f7c009230 tx=0x7f1f7c009260 comp rx=0 tx=0).stop 2024-04-25T00:35:02.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.088+0000 7f1f9705d740 1 -- 172.21.15.32:0/3807806701 shutdown_connections 2024-04-25T00:35:02.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.088+0000 7f1f9705d740 1 --2- 172.21.15.32:0/3807806701 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5605aa848610 0x5605aa84aa30 crc :-1 s=READY pgs=63 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.090 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.088+0000 7f1f9705d740 1 --2- 172.21.15.32:0/3807806701 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5605aa838570 0x5605aa8359d0 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.090 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.088+0000 7f1f9705d740 1 --2- 172.21.15.32:0/3807806701 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5605aa82dea0 0x5605aa835290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.090 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.088+0000 7f1f9705d740 1 --2- 172.21.15.32:0/3807806701 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82d580 0x5605aa837fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.090 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.088+0000 7f1f9705d740 1 -- 172.21.15.32:0/3807806701 >> 172.21.15.32:0/3807806701 conn(0x5605aa8298a0 msgr2=0x5605aa828b40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:02.090 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.088+0000 7f1f75ffb640 0 client.25537 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:35:02.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.090+0000 7f1f9705d740 1 -- 172.21.15.32:0/3807806701 shutdown_connections 2024-04-25T00:35:02.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.090+0000 7f1f9705d740 1 -- 172.21.15.32:0/3807806701 wait complete. 2024-04-25T00:35:02.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.090+0000 7f1f9705d740 20 client.25537 trim_cache size 0 max 16384 2024-04-25T00:35:02.129 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.127+0000 7f1f9705d740 1 Processor -- start 2024-04-25T00:35:02.129 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.127+0000 7f1f9705d740 1 -- start start 2024-04-25T00:35:02.130 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.128+0000 7f1f9705d740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82d580 0x5605aa8441c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:02.130 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.128+0000 7f1f9705d740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5605aa82dea0 0x5605aa8306e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:02.131 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.129+0000 7f1f918ee640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5605aa82dea0 0x5605aa8306e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:02.131 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.129+0000 7f1f920ef640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82d580 0x5605aa8441c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:02.131 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.129+0000 7f1f920ef640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82d580 0x5605aa8441c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:43838/0 (socket says 172.21.15.32:43838) 2024-04-25T00:35:02.131 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.129+0000 7f1f920ef640 1 -- 172.21.15.32:0/2358468959 learned_addr learned my addr 172.21.15.32:0/2358468959 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:02.131 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.129+0000 7f1f9705d740 1 --2- 172.21.15.32:0/2358468959 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5605aa838570 0x5605aa837ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:02.131 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.129+0000 7f1f9705d740 1 -- 172.21.15.32:0/2358468959 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f1f8c1c0530 con 0x5605aa82dea0 2024-04-25T00:35:02.131 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.129+0000 7f1f9705d740 1 -- 172.21.15.32:0/2358468959 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f1f8c1bde00 con 0x5605aa82d580 2024-04-25T00:35:02.131 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.129+0000 7f1f9705d740 1 -- 172.21.15.32:0/2358468959 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f1f8c1bbc50 con 0x5605aa838570 2024-04-25T00:35:02.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.130+0000 7f1f928f0640 1 --2- 172.21.15.32:0/2358468959 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5605aa838570 0x5605aa837ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:02.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.130+0000 7f1f920ef640 1 -- 172.21.15.32:0/2358468959 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5605aa838570 msgr2=0x5605aa837ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:02.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.130+0000 7f1f920ef640 1 --2- 172.21.15.32:0/2358468959 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5605aa838570 0x5605aa837ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.130+0000 7f1f920ef640 1 -- 172.21.15.32:0/2358468959 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5605aa82dea0 msgr2=0x5605aa8306e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:02.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.130+0000 7f1f920ef640 1 --2- 172.21.15.32:0/2358468959 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5605aa82dea0 0x5605aa8306e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.130+0000 7f1f920ef640 1 -- 172.21.15.32:0/2358468959 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5605aa82dcd0 con 0x5605aa82d580 2024-04-25T00:35:02.135 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.134+0000 7f1f920ef640 1 --2- 172.21.15.32:0/2358468959 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82d580 0x5605aa8441c0 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f1f880109f0 tx=0x7f1f8809f260 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e85b8d3903bebdf4 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:02.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.139+0000 7f1f77fff640 1 -- 172.21.15.32:0/2358468959 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1f88015040 con 0x5605aa82d580 2024-04-25T00:35:02.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.139+0000 7f1f77fff640 1 -- 172.21.15.32:0/2358468959 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1f8809b2f0 con 0x5605aa82d580 2024-04-25T00:35:02.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.139+0000 7f1f77fff640 1 -- 172.21.15.32:0/2358468959 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1f88097a30 con 0x5605aa82d580 2024-04-25T00:35:02.150 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.148+0000 7f1f9705d740 1 -- 172.21.15.32:0/2358468959 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82d580 msgr2=0x5605aa8441c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:02.150 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.148+0000 7f1f9705d740 1 --2- 172.21.15.32:0/2358468959 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82d580 0x5605aa8441c0 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f1f880109f0 tx=0x7f1f8809f260 comp rx=0 tx=0).stop 2024-04-25T00:35:02.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.149+0000 7f1f9705d740 1 -- 172.21.15.32:0/2358468959 shutdown_connections 2024-04-25T00:35:02.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.149+0000 7f1f9705d740 1 --2- 172.21.15.32:0/2358468959 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5605aa838570 0x5605aa837ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.149+0000 7f1f9705d740 1 --2- 172.21.15.32:0/2358468959 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5605aa82dea0 0x5605aa8306e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.149+0000 7f1f9705d740 1 --2- 172.21.15.32:0/2358468959 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82d580 0x5605aa8441c0 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.149+0000 7f1f9705d740 1 -- 172.21.15.32:0/2358468959 >> 172.21.15.32:0/2358468959 conn(0x5605aa8298a0 msgr2=0x5605aa82ffd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:02.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.152+0000 7f1f9705d740 1 -- 172.21.15.32:0/2358468959 shutdown_connections 2024-04-25T00:35:02.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.153+0000 7f1f9705d740 1 -- 172.21.15.32:0/2358468959 wait complete. 2024-04-25T00:35:02.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.153+0000 7f1f9705d740 1 -- start start 2024-04-25T00:35:02.155 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.154+0000 7f1f9705d740 1 Processor -- start 2024-04-25T00:35:02.156 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.154+0000 7f1f9705d740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5605aa82d580 0x5605aa840c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:02.156 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.155+0000 7f1f9705d740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82dea0 0x5605aa83c020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:02.156 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.155+0000 7f1f9705d740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5605aa838570 0x5605aa83c760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:02.157 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.155+0000 7f1f920ef640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5605aa82d580 0x5605aa840c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:02.157 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.155+0000 7f1f920ef640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5605aa82d580 0x5605aa840c10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:54688/0 (socket says 172.21.15.32:54688) 2024-04-25T00:35:02.157 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.155+0000 7f1f918ee640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82dea0 0x5605aa83c020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:02.157 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.155+0000 7f1f918ee640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82dea0 0x5605aa83c020 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:43844/0 (socket says 172.21.15.32:43844) 2024-04-25T00:35:02.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.155+0000 7f1f920ef640 1 -- 172.21.15.32:0/683114072 learned_addr learned my addr 172.21.15.32:0/683114072 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:02.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.155+0000 7f1f9705d740 1 -- 172.21.15.32:0/683114072 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f1f8c1bc2c0 con 0x5605aa838570 2024-04-25T00:35:02.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.155+0000 7f1f9705d740 1 -- 172.21.15.32:0/683114072 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f1f8c1bd730 con 0x5605aa82dea0 2024-04-25T00:35:02.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.155+0000 7f1f9705d740 1 -- 172.21.15.32:0/683114072 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f1f8c1bc990 con 0x5605aa82d580 2024-04-25T00:35:02.158 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.156+0000 7f1f928f0640 1 --2- 172.21.15.32:0/683114072 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5605aa838570 0x5605aa83c760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:02.159 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.156+0000 7f1f920ef640 1 -- 172.21.15.32:0/683114072 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82dea0 msgr2=0x5605aa83c020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:02.159 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.156+0000 7f1f920ef640 1 --2- 172.21.15.32:0/683114072 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82dea0 0x5605aa83c020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.159 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.156+0000 7f1f920ef640 1 -- 172.21.15.32:0/683114072 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5605aa838570 msgr2=0x5605aa83c760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:02.159 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.156+0000 7f1f920ef640 1 --2- 172.21.15.32:0/683114072 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5605aa838570 0x5605aa83c760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.159 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.156+0000 7f1f920ef640 1 -- 172.21.15.32:0/683114072 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5605aa5707d0 con 0x5605aa82d580 2024-04-25T00:35:02.159 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.156+0000 7f1f9705d740 10 client.0 _ll_register_callbacks cb 0x7f1f7c009780 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:35:02.159 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.156+0000 7f1f920ef640 1 --2- 172.21.15.32:0/683114072 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5605aa82d580 0x5605aa840c10 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f1f880a38a0 tx=0x7f1f880cbf20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=56250a28dae949d0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:02.159 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.157+0000 7f1f75ffb640 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:35:02.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.157+0000 7f1f9705d740 1 -- 172.21.15.32:0/683114072 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1f8809b2f0 con 0x5605aa82d580 2024-04-25T00:35:02.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.157+0000 7f1f75ffb640 1 -- 172.21.15.32:0/683114072 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1f88015070 con 0x5605aa82d580 2024-04-25T00:35:02.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.157+0000 7f1f75ffb640 1 -- 172.21.15.32:0/683114072 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f1f8809a620 con 0x5605aa82d580 2024-04-25T00:35:02.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.158+0000 7f1f75ffb640 1 -- 172.21.15.32:0/683114072 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7f1f88097c00 con 0x5605aa82d580 2024-04-25T00:35:02.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.158+0000 7f1f9705d740 10 client.25543 fetch_fsmap learned FSMap version 26 2024-04-25T00:35:02.161 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.158+0000 7f1f9705d740 1 -- 172.21.15.32:0/683114072 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({fsmap=26}) -- 0x5605aa8ae870 con 0x5605aa82d580 2024-04-25T00:35:02.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.160+0000 7f1f75ffb640 1 -- 172.21.15.32:0/683114072 <== mon.1 v2:172.21.15.130:3300/0 4 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7f1f88022060 con 0x5605aa82d580 2024-04-25T00:35:02.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.161+0000 7f1f9705d740 10 client.25543 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:35:02.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.161+0000 7f1f9705d740 10 client.25543 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:35:02.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.161+0000 7f1f9705d740 20 client.25543 populate_metadata read hostname 'smithi032' 2024-04-25T00:35:02.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.161+0000 7f1f9705d740 1 --2- 172.21.15.32:0/683114072 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5605aa8ae840 0x5605aa8b0c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:02.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.161+0000 7f1f9705d740 4 client.25543 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-25T00:35:02.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.161+0000 7f1f9705d740 1 -- 172.21.15.32:0/683114072 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "damage ls"}) -- 0x5605aa799f00 con 0x5605aa8ae840 2024-04-25T00:35:02.169 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.167+0000 7f1f918ee640 1 --2- 172.21.15.32:0/683114072 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5605aa8ae840 0x5605aa8b0c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:02.171 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.169+0000 7f1f918ee640 1 --2- 172.21.15.32:0/683114072 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5605aa8ae840 0x5605aa8b0c90 crc :-1 s=READY pgs=64 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7261b3a10f2b0d1c server_cookie=27062a25ef5a4c50 in_seq=0 out_seq=0 2024-04-25T00:35:02.171 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.169+0000 7f1f75ffb640 10 client.25543 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:35:02.171 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.170+0000 7f1f75ffb640 1 -- 172.21.15.32:0/683114072 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x5605aa799f00 con 0x5605aa8ae840 2024-04-25T00:35:02.172 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.170+0000 7f1f75ffb640 10 client.25543 handle_command_reply: tid=0 2024-04-25T00:35:02.172 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.170+0000 7f1f9705d740 1 client.25543 shutdown 2024-04-25T00:35:02.178 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.174+0000 7f1f9705d740 1 -- 172.21.15.32:0/683114072 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5605aa82d580 msgr2=0x5605aa840c10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:02.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.174+0000 7f1f9705d740 1 --2- 172.21.15.32:0/683114072 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5605aa82d580 0x5605aa840c10 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f1f880a38a0 tx=0x7f1f880cbf20 comp rx=0 tx=0).stop 2024-04-25T00:35:02.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.175+0000 7f1f9705d740 1 -- 172.21.15.32:0/683114072 shutdown_connections 2024-04-25T00:35:02.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.175+0000 7f1f9705d740 1 --2- 172.21.15.32:0/683114072 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5605aa8ae840 0x5605aa8b0c90 crc :-1 s=READY pgs=64 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.175+0000 7f1f9705d740 1 --2- 172.21.15.32:0/683114072 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5605aa838570 0x5605aa83c760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.175+0000 7f1f9705d740 1 --2- 172.21.15.32:0/683114072 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5605aa82dea0 0x5605aa83c020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.175+0000 7f1f9705d740 1 --2- 172.21.15.32:0/683114072 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5605aa82d580 0x5605aa840c10 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.175+0000 7f1f9705d740 1 -- 172.21.15.32:0/683114072 >> 172.21.15.32:0/683114072 conn(0x5605aa8298a0 msgr2=0x5605aa828b40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:02.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.177+0000 7f1f75ffb640 0 client.25543 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:35:02.180 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.178+0000 7f1f9705d740 1 -- 172.21.15.32:0/683114072 shutdown_connections 2024-04-25T00:35:02.180 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.178+0000 7f1f9705d740 1 -- 172.21.15.32:0/683114072 wait complete. 2024-04-25T00:35:02.180 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.178+0000 7f1f9705d740 20 client.25543 trim_cache size 0 max 16384 2024-04-25T00:35:02.182 INFO:teuthology.orchestra.run.smithi032.stdout:[] 2024-04-25T00:35:02.186 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.183+0000 7f1f94353640 1 -- 172.21.15.32:0/2624429287 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f1f780a3270 msgr2=0x7f1f780a5730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:02.186 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.183+0000 7f1f94353640 1 --2- 172.21.15.32:0/2624429287 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f1f780a3270 0x7f1f780a5730 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x5605aa82aa30 tx=0x7f1f7c0095e0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.186 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.183+0000 7f1f94353640 1 -- 172.21.15.32:0/2624429287 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1f8c10dfc0 msgr2=0x7f1f8c1a6ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:02.186 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.183+0000 7f1f94353640 1 --2- 172.21.15.32:0/2624429287 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1f8c10dfc0 0x7f1f8c1a6ea0 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f1f8800b450 tx=0x7f1f8800b910 comp rx=0 tx=0).stop 2024-04-25T00:35:02.187 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.185+0000 7f1f94353640 1 -- 172.21.15.32:0/2624429287 shutdown_connections 2024-04-25T00:35:02.187 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.185+0000 7f1f94353640 1 --2- 172.21.15.32:0/2624429287 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f1f780a3270 0x7f1f780a5730 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.187 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.185+0000 7f1f94353640 1 --2- 172.21.15.32:0/2624429287 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1f8c11af20 0x7f1f8c1ab760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.187 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.185+0000 7f1f94353640 1 --2- 172.21.15.32:0/2624429287 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1f8c10ff90 0x7f1f8c1a73e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.187 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.185+0000 7f1f94353640 1 --2- 172.21.15.32:0/2624429287 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1f8c10dfc0 0x7f1f8c1a6ea0 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:02.187 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.185+0000 7f1f94353640 1 -- 172.21.15.32:0/2624429287 >> 172.21.15.32:0/2624429287 conn(0x7f1f8c06c850 msgr2=0x7f1f8c072240 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:02.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.186+0000 7f1f94353640 1 -- 172.21.15.32:0/2624429287 shutdown_connections 2024-04-25T00:35:02.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:02.186+0000 7f1f94353640 1 -- 172.21.15.32:0/2624429287 wait complete. 2024-04-25T00:35:03.225 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-25T00:35:03.226 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:35:03.226 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-25T00:35:03.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:03 smithi121 ceph-mon[32231]: pgmap v55: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 1.7 KiB/s rd, 683 MiB/s wr, 1.46k op/s 2024-04-25T00:35:03.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.599+0000 7fe7394d2640 1 -- 172.21.15.32:0/156079600 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe7341087d0 msgr2=0x7fe734108c50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:03.604 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.599+0000 7fe7394d2640 1 --2- 172.21.15.32:0/156079600 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe7341087d0 0x7fe734108c50 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7fe724009900 tx=0x7fe72402a440 comp rx=0 tx=0).stop 2024-04-25T00:35:03.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.599+0000 7fe7394d2640 1 -- 172.21.15.32:0/156079600 shutdown_connections 2024-04-25T00:35:03.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.599+0000 7fe7394d2640 1 --2- 172.21.15.32:0/156079600 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe73411af20 0x7fe73411d310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.599+0000 7fe7394d2640 1 --2- 172.21.15.32:0/156079600 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe7341087d0 0x7fe734108c50 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.599+0000 7fe7394d2640 1 --2- 172.21.15.32:0/156079600 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe73410fdb0 0x7fe734110190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.599+0000 7fe7394d2640 1 -- 172.21.15.32:0/156079600 >> 172.21.15.32:0/156079600 conn(0x7fe73406c720 msgr2=0x7fe73406cb30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:03.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.600+0000 7fe7394d2640 1 -- 172.21.15.32:0/156079600 shutdown_connections 2024-04-25T00:35:03.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.601+0000 7fe7394d2640 1 -- 172.21.15.32:0/156079600 wait complete. 2024-04-25T00:35:03.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.602+0000 7fe7394d2640 1 Processor -- start 2024-04-25T00:35:03.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.602+0000 7fe7394d2640 1 -- start start 2024-04-25T00:35:03.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.603+0000 7fe7394d2640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe7341087d0 0x7fe73411a800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:03.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.603+0000 7fe7394d2640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe73411af20 0x7fe7341148c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:03.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.603+0000 7fe7394d2640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe734114e00 0x7fe7341152b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:03.606 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.603+0000 7fe7394d2640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fe734111150 con 0x7fe734114e00 2024-04-25T00:35:03.606 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.603+0000 7fe7394d2640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fe734110fd0 con 0x7fe73411af20 2024-04-25T00:35:03.606 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.603+0000 7fe7394d2640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fe7341112d0 con 0x7fe7341087d0 2024-04-25T00:35:03.606 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.603+0000 7fe7337fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe734114e00 0x7fe7341152b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:03.606 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.603+0000 7fe7337fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe734114e00 0x7fe7341152b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53344/0 (socket says 172.21.15.32:53344) 2024-04-25T00:35:03.607 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.603+0000 7fe7337fe640 1 -- 172.21.15.32:0/1863151233 learned_addr learned my addr 172.21.15.32:0/1863151233 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:03.607 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.604+0000 7fe7337fe640 1 -- 172.21.15.32:0/1863151233 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe7341087d0 msgr2=0x7fe73411a800 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-04-25T00:35:03.607 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.604+0000 7fe7337fe640 1 --2- 172.21.15.32:0/1863151233 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe7341087d0 0x7fe73411a800 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.607 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.604+0000 7fe7337fe640 1 -- 172.21.15.32:0/1863151233 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe73411af20 msgr2=0x7fe7341148c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:03.607 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.604+0000 7fe7337fe640 1 --2- 172.21.15.32:0/1863151233 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe73411af20 0x7fe7341148c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.607 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.604+0000 7fe7337fe640 1 -- 172.21.15.32:0/1863151233 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe7341bbdd0 con 0x7fe734114e00 2024-04-25T00:35:03.607 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.605+0000 7fe7337fe640 1 --2- 172.21.15.32:0/1863151233 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe734114e00 0x7fe7341152b0 secure :-1 s=READY pgs=593 cs=0 l=1 rev1=1 crypto rx=0x7fe72c00b970 tx=0x7fe72c00be30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:03.607 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.606+0000 7fe71bfff640 1 -- 172.21.15.32:0/1863151233 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe72c019070 con 0x7fe734114e00 2024-04-25T00:35:03.612 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.606+0000 7fe7394d2640 1 -- 172.21.15.32:0/1863151233 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe7341bc0c0 con 0x7fe734114e00 2024-04-25T00:35:03.613 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.606+0000 7fe7394d2640 1 -- 172.21.15.32:0/1863151233 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe7341bc610 con 0x7fe734114e00 2024-04-25T00:35:03.617 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.616+0000 7fe73c1dd740 1 Processor -- start 2024-04-25T00:35:03.618 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.616+0000 7fe73c1dd740 1 -- start start 2024-04-25T00:35:03.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.617+0000 7fe73c1dd740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94ccb70 0x5595e94caeb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:03.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.617+0000 7fe73c1dd740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5595e94cb480 0x5595e94cb900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:03.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.617+0000 7fe73c1dd740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5595e94d6200 0x5595e94d85f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:03.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.618+0000 7fe7327fc640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94ccb70 0x5595e94caeb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:03.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.618+0000 7fe7327fc640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94ccb70 0x5595e94caeb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:43870/0 (socket says 172.21.15.32:43870) 2024-04-25T00:35:03.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.618+0000 7fe7327fc640 1 -- 172.21.15.32:0/786244742 learned_addr learned my addr 172.21.15.32:0/786244742 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:03.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.618+0000 7fe73c1dd740 1 -- 172.21.15.32:0/786244742 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5595e942f2c0 con 0x5595e94d6200 2024-04-25T00:35:03.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.618+0000 7fe73c1dd740 1 -- 172.21.15.32:0/786244742 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5595e93d7910 con 0x5595e94ccb70 2024-04-25T00:35:03.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.618+0000 7fe73c1dd740 1 -- 172.21.15.32:0/786244742 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5595e942b280 con 0x5595e94cb480 2024-04-25T00:35:03.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.619+0000 7fe71bfff640 1 -- 172.21.15.32:0/1863151233 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe72c00cb40 con 0x7fe734114e00 2024-04-25T00:35:03.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.619+0000 7fe71bfff640 1 -- 172.21.15.32:0/1863151233 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe72c007670 con 0x7fe734114e00 2024-04-25T00:35:03.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.619+0000 7fe71bfff640 1 -- 172.21.15.32:0/1863151233 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 25) ==== 138066+0+0 (secure 0 0 0) 0x7fe72c01c070 con 0x7fe734114e00 2024-04-25T00:35:03.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.619+0000 7fe7327fc640 1 --2- 172.21.15.32:0/786244742 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5595e94d6200 0x5595e94d85f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:03.622 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.620+0000 7fe71bfff640 1 --2- 172.21.15.32:0/1863151233 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe71c0a3060 0x7fe71c0a5520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:03.623 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.621+0000 7fe71bfff640 1 -- 172.21.15.32:0/1863151233 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fe72c0d0010 con 0x7fe734114e00 2024-04-25T00:35:03.627 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.623+0000 7fe7337fe640 1 --2- 172.21.15.32:0/1863151233 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe71c0a3060 0x7fe71c0a5520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:03.627 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.625+0000 7fe732ffd640 1 --2- 172.21.15.32:0/786244742 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5595e94cb480 0x5595e94cb900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:03.627 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.626+0000 7fe7337fe640 1 --2- 172.21.15.32:0/1863151233 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe71c0a3060 0x7fe71c0a5520 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7fe72c0cd3e0 tx=0x7fe72c098390 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:03.629 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.628+0000 7fe7327fc640 1 -- 172.21.15.32:0/786244742 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5595e94cb480 msgr2=0x5595e94cb900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:03.629 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.628+0000 7fe7327fc640 1 --2- 172.21.15.32:0/786244742 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5595e94cb480 0x5595e94cb900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.629 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.628+0000 7fe7327fc640 1 -- 172.21.15.32:0/786244742 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5595e94d6200 msgr2=0x5595e94d85f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:03.629 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.628+0000 7fe7327fc640 1 --2- 172.21.15.32:0/786244742 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5595e94d6200 0x5595e94d85f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.630 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.629+0000 7fe7327fc640 1 -- 172.21.15.32:0/786244742 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5595e93e8fb0 con 0x5595e94ccb70 2024-04-25T00:35:03.635 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.634+0000 7fe7327fc640 1 --2- 172.21.15.32:0/786244742 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94ccb70 0x5595e94caeb0 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7fe724005c20 tx=0x7fe7240056d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=701750ea55271e82 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:03.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:03 smithi130 ceph-mon[32525]: pgmap v55: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 1.7 KiB/s rd, 683 MiB/s wr, 1.46k op/s 2024-04-25T00:35:03.642 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.639+0000 7fe7197fa640 1 -- 172.21.15.32:0/786244742 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe724041070 con 0x5595e94ccb70 2024-04-25T00:35:03.642 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.639+0000 7fe7197fa640 1 -- 172.21.15.32:0/786244742 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe724031c20 con 0x5595e94ccb70 2024-04-25T00:35:03.643 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.639+0000 7fe7197fa640 1 -- 172.21.15.32:0/786244742 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe724035030 con 0x5595e94ccb70 2024-04-25T00:35:03.643 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.641+0000 7fe73c1dd740 1 -- 172.21.15.32:0/786244742 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94ccb70 msgr2=0x5595e94caeb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:03.643 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.641+0000 7fe73c1dd740 1 --2- 172.21.15.32:0/786244742 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94ccb70 0x5595e94caeb0 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7fe724005c20 tx=0x7fe7240056d0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.643 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.641+0000 7fe73c1dd740 1 -- 172.21.15.32:0/786244742 shutdown_connections 2024-04-25T00:35:03.643 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.641+0000 7fe73c1dd740 1 --2- 172.21.15.32:0/786244742 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5595e94d6200 0x5595e94d85f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.643 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.641+0000 7fe73c1dd740 1 --2- 172.21.15.32:0/786244742 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5595e94cb480 0x5595e94cb900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.643 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.641+0000 7fe73c1dd740 1 --2- 172.21.15.32:0/786244742 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94ccb70 0x5595e94caeb0 secure :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7fe724005c20 tx=0x7fe7240056d0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.643 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.641+0000 7fe73c1dd740 1 -- 172.21.15.32:0/786244742 >> 172.21.15.32:0/786244742 conn(0x5595e9493950 msgr2=0x5595e94c79e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:03.645 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.644+0000 7fe73c1dd740 1 -- 172.21.15.32:0/786244742 shutdown_connections 2024-04-25T00:35:03.647 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.644+0000 7fe73c1dd740 1 -- 172.21.15.32:0/786244742 wait complete. 2024-04-25T00:35:03.647 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.645+0000 7fe73c1dd740 1 -- start start 2024-04-25T00:35:03.647 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.645+0000 7fe73c1dd740 1 Processor -- start 2024-04-25T00:35:03.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.649+0000 7fe73c1dd740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94cb480 0x5595e94d3360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:03.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.649+0000 7fe73c1dd740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5595e94d6200 0x5595e94d38a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:03.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.649+0000 7fe73c1dd740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5595e94d3e30 0x5595e94e2570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:03.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.649+0000 7fe73c1dd740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5595e94d9030 con 0x5595e94d6200 2024-04-25T00:35:03.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.649+0000 7fe73c1dd740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5595e93eae90 con 0x5595e94cb480 2024-04-25T00:35:03.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.649+0000 7fe73c1dd740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5595e94d91b0 con 0x5595e94d3e30 2024-04-25T00:35:03.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.649+0000 7fe73c1dd740 10 client.0 _ll_register_callbacks cb 0x5595e9426970 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:35:03.654 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.651+0000 7fe7327fc640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94cb480 0x5595e94d3360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:03.654 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.651+0000 7fe732ffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5595e94d6200 0x5595e94d38a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:03.654 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.651+0000 7fe732ffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5595e94d6200 0x5595e94d38a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53370/0 (socket says 172.21.15.32:53370) 2024-04-25T00:35:03.654 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.651+0000 7fe732ffd640 1 -- 172.21.15.32:0/1675448673 learned_addr learned my addr 172.21.15.32:0/1675448673 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:03.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.652+0000 7fe7327fc640 1 --2- 172.21.15.32:0/1675448673 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5595e94d3e30 0x5595e94e2570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:03.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.654+0000 7fe732ffd640 1 -- 172.21.15.32:0/1675448673 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5595e94d3e30 msgr2=0x5595e94e2570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:03.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.654+0000 7fe732ffd640 1 --2- 172.21.15.32:0/1675448673 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5595e94d3e30 0x5595e94e2570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.654+0000 7fe732ffd640 1 -- 172.21.15.32:0/1675448673 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94cb480 msgr2=0x5595e94d3360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:03.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.654+0000 7fe732ffd640 1 --2- 172.21.15.32:0/1675448673 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94cb480 0x5595e94d3360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.654+0000 7fe732ffd640 1 -- 172.21.15.32:0/1675448673 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5595e920b7e0 con 0x5595e94d6200 2024-04-25T00:35:03.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.658+0000 7fe716ff5640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:35:03.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.658+0000 7fe732ffd640 1 --2- 172.21.15.32:0/1675448673 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5595e94d6200 0x5595e94d38a0 secure :-1 s=READY pgs=594 cs=0 l=1 rev1=1 crypto rx=0x7fe728009f30 tx=0x7fe72800e660 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e460945cefda3647 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:03.661 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.660+0000 7fe716ff5640 1 -- 172.21.15.32:0/1675448673 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe728017070 con 0x5595e94d6200 2024-04-25T00:35:03.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.660+0000 7fe73c1dd740 1 -- 172.21.15.32:0/1675448673 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe724031c20 con 0x5595e94d6200 2024-04-25T00:35:03.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.660+0000 7fe716ff5640 1 -- 172.21.15.32:0/1675448673 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fe72800f040 con 0x5595e94d6200 2024-04-25T00:35:03.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.662+0000 7fe716ff5640 1 -- 172.21.15.32:0/1675448673 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7fe728017070 con 0x5595e94d6200 2024-04-25T00:35:03.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.662+0000 7fe73c1dd740 10 client.16020 fetch_fsmap learned FSMap version 26 2024-04-25T00:35:03.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.662+0000 7fe73c1dd740 1 -- 172.21.15.32:0/1675448673 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x5595e926bf70 con 0x5595e94d6200 2024-04-25T00:35:03.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.665+0000 7fe716ff5640 1 -- 172.21.15.32:0/1675448673 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe724031c20 con 0x5595e94d6200 2024-04-25T00:35:03.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.665+0000 7fe716ff5640 1 -- 172.21.15.32:0/1675448673 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7fe728024080 con 0x5595e94d6200 2024-04-25T00:35:03.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.665+0000 7fe73c1dd740 10 client.16020 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:35:03.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.665+0000 7fe73c1dd740 10 client.16020 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:35:03.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.665+0000 7fe73c1dd740 20 client.16020 populate_metadata read hostname 'smithi032' 2024-04-25T00:35:03.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.665+0000 7fe73c1dd740 1 --2- 172.21.15.32:0/1675448673 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5595e94e6d80 0x5595e94e91e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:03.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.665+0000 7fe73c1dd740 4 client.16020 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:35:03.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.665+0000 7fe73c1dd740 1 -- 172.21.15.32:0/1675448673 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5595e94035a0 con 0x5595e94e6d80 2024-04-25T00:35:03.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.666+0000 7fe7327fc640 1 --2- 172.21.15.32:0/1675448673 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5595e94e6d80 0x5595e94e91e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:03.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.668+0000 7fe7327fc640 1 --2- 172.21.15.32:0/1675448673 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5595e94e6d80 0x5595e94e91e0 crc :-1 s=READY pgs=65 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5911510b1b7b7023 server_cookie=57e98e2c6d453f23 in_seq=0 out_seq=0 2024-04-25T00:35:03.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.668+0000 7fe716ff5640 10 client.16020 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:35:03.671 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.670+0000 7fe716ff5640 1 -- 172.21.15.32:0/1675448673 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x5595e94035a0 con 0x5595e94e6d80 2024-04-25T00:35:03.671 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.670+0000 7fe716ff5640 10 client.16020 handle_command_reply: tid=0 2024-04-25T00:35:03.671 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.670+0000 7fe73c1dd740 1 client.16020 shutdown 2024-04-25T00:35:03.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.672+0000 7fe73c1dd740 1 -- 172.21.15.32:0/1675448673 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5595e94d6200 msgr2=0x5595e94d38a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:03.674 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.672+0000 7fe73c1dd740 1 --2- 172.21.15.32:0/1675448673 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5595e94d6200 0x5595e94d38a0 secure :-1 s=READY pgs=594 cs=0 l=1 rev1=1 crypto rx=0x7fe728009f30 tx=0x7fe72800e660 comp rx=0 tx=0).stop 2024-04-25T00:35:03.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.674+0000 7fe73c1dd740 1 -- 172.21.15.32:0/1675448673 shutdown_connections 2024-04-25T00:35:03.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.674+0000 7fe73c1dd740 1 --2- 172.21.15.32:0/1675448673 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5595e94e6d80 0x5595e94e91e0 crc :-1 s=READY pgs=65 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.674+0000 7fe73c1dd740 1 --2- 172.21.15.32:0/1675448673 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5595e94d3e30 0x5595e94e2570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.674+0000 7fe73c1dd740 1 --2- 172.21.15.32:0/1675448673 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5595e94d6200 0x5595e94d38a0 unknown :-1 s=CLOSED pgs=594 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.674+0000 7fe73c1dd740 1 --2- 172.21.15.32:0/1675448673 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94cb480 0x5595e94d3360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.674+0000 7fe73c1dd740 1 -- 172.21.15.32:0/1675448673 >> 172.21.15.32:0/1675448673 conn(0x5595e9493950 msgr2=0x5595e94cd7e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:03.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.674+0000 7fe716ff5640 0 client.16020 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:35:03.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.674+0000 7fe73c1dd740 1 -- 172.21.15.32:0/1675448673 shutdown_connections 2024-04-25T00:35:03.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.674+0000 7fe73c1dd740 1 -- 172.21.15.32:0/1675448673 wait complete. 2024-04-25T00:35:03.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.679+0000 7fe73c1dd740 20 client.16020 trim_cache size 0 max 16384 2024-04-25T00:35:03.745 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.743+0000 7fe73c1dd740 1 Processor -- start 2024-04-25T00:35:03.745 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.743+0000 7fe73c1dd740 1 -- start start 2024-04-25T00:35:03.745 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.744+0000 7fe73c1dd740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5595e94cb480 0x5595e94eeaf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:03.746 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.744+0000 7fe73c1dd740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94d3e30 0x5595e94ef030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:03.746 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.744+0000 7fe73c1dd740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5595e94d6200 0x5595e94e1fc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:03.746 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.744+0000 7fe73c1dd740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fe720012b50 con 0x5595e94cb480 2024-04-25T00:35:03.746 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.745+0000 7fe73c1dd740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fe720010420 con 0x5595e94d3e30 2024-04-25T00:35:03.746 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.745+0000 7fe732ffd640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94d3e30 0x5595e94ef030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:03.746 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.745+0000 7fe732ffd640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94d3e30 0x5595e94ef030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:43896/0 (socket says 172.21.15.32:43896) 2024-04-25T00:35:03.746 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.745+0000 7fe732ffd640 1 -- 172.21.15.32:0/250374908 learned_addr learned my addr 172.21.15.32:0/250374908 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:03.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.745+0000 7fe73c1dd740 1 -- 172.21.15.32:0/250374908 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fe72000e210 con 0x5595e94d6200 2024-04-25T00:35:03.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.745+0000 7fe732ffd640 1 -- 172.21.15.32:0/250374908 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5595e94d6200 msgr2=0x5595e94e1fc0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-25T00:35:03.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.745+0000 7fe732ffd640 1 --2- 172.21.15.32:0/250374908 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5595e94d6200 0x5595e94e1fc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.745+0000 7fe732ffd640 1 -- 172.21.15.32:0/250374908 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5595e94cb480 msgr2=0x5595e94eeaf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:03.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.745+0000 7fe732ffd640 1 --2- 172.21.15.32:0/250374908 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5595e94cb480 0x5595e94eeaf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.745+0000 7fe732ffd640 1 -- 172.21.15.32:0/250374908 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5595e9103170 con 0x5595e94d3e30 2024-04-25T00:35:03.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:03 smithi032 ceph-mon[29357]: pgmap v55: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 1.7 KiB/s rd, 683 MiB/s wr, 1.46k op/s 2024-04-25T00:35:03.748 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.747+0000 7fe732ffd640 1 --2- 172.21.15.32:0/250374908 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94d3e30 0x5595e94ef030 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7fe728010770 tx=0x7fe728013600 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5c202947166cea44 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:03.752 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.749+0000 7fe718ff9640 1 -- 172.21.15.32:0/250374908 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe72802a070 con 0x5595e94d3e30 2024-04-25T00:35:03.752 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.749+0000 7fe718ff9640 1 -- 172.21.15.32:0/250374908 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe728013670 con 0x5595e94d3e30 2024-04-25T00:35:03.752 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.749+0000 7fe718ff9640 1 -- 172.21.15.32:0/250374908 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe728017070 con 0x5595e94d3e30 2024-04-25T00:35:03.752 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.751+0000 7fe73c1dd740 1 -- 172.21.15.32:0/250374908 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94d3e30 msgr2=0x5595e94ef030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:03.752 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.751+0000 7fe73c1dd740 1 --2- 172.21.15.32:0/250374908 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94d3e30 0x5595e94ef030 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7fe728010770 tx=0x7fe728013600 comp rx=0 tx=0).stop 2024-04-25T00:35:03.753 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.751+0000 7fe73c1dd740 1 -- 172.21.15.32:0/250374908 shutdown_connections 2024-04-25T00:35:03.753 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.751+0000 7fe73c1dd740 1 --2- 172.21.15.32:0/250374908 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5595e94d6200 0x5595e94e1fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.753 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.751+0000 7fe73c1dd740 1 --2- 172.21.15.32:0/250374908 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94d3e30 0x5595e94ef030 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.753 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.751+0000 7fe73c1dd740 1 --2- 172.21.15.32:0/250374908 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5595e94cb480 0x5595e94eeaf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.753 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.751+0000 7fe73c1dd740 1 -- 172.21.15.32:0/250374908 >> 172.21.15.32:0/250374908 conn(0x5595e9493950 msgr2=0x5595e94cd7e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:03.758 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.757+0000 7fe73c1dd740 1 -- 172.21.15.32:0/250374908 shutdown_connections 2024-04-25T00:35:03.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.757+0000 7fe73c1dd740 1 -- 172.21.15.32:0/250374908 wait complete. 2024-04-25T00:35:03.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.757+0000 7fe73c1dd740 1 -- start start 2024-04-25T00:35:03.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.757+0000 7fe73c1dd740 1 Processor -- start 2024-04-25T00:35:03.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.758+0000 7fe73c1dd740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94cb480 0x5595e94de4d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:03.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.758+0000 7fe73c1dd740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5595e94d3e30 0x5595e94dea10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:03.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.758+0000 7fe73c1dd740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5595e94d6200 0x5595e94d7330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:03.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.758+0000 7fe73c1dd740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fe72000e8e0 con 0x5595e94d6200 2024-04-25T00:35:03.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.758+0000 7fe73c1dd740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fe72000fd50 con 0x5595e94cb480 2024-04-25T00:35:03.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.758+0000 7fe73c1dd740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fe72000efb0 con 0x5595e94d3e30 2024-04-25T00:35:03.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.758+0000 7fe73c1dd740 10 client.0 _ll_register_callbacks cb 0x7fe7280092f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:35:03.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.758+0000 7fe7327fc640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94cb480 0x5595e94de4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:03.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.758+0000 7fe7327fc640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94cb480 0x5595e94de4d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:43898/0 (socket says 172.21.15.32:43898) 2024-04-25T00:35:03.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.758+0000 7fe7327fc640 1 -- 172.21.15.32:0/1111711835 learned_addr learned my addr 172.21.15.32:0/1111711835 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:03.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.759+0000 7fe732ffd640 1 --2- 172.21.15.32:0/1111711835 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5595e94d3e30 0x5595e94dea10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:03.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.759+0000 7fe7327fc640 1 -- 172.21.15.32:0/1111711835 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5595e94d3e30 msgr2=0x5595e94dea10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:03.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.759+0000 7fe7327fc640 1 --2- 172.21.15.32:0/1111711835 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5595e94d3e30 0x5595e94dea10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.759+0000 7fe7327fc640 1 -- 172.21.15.32:0/1111711835 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5595e94d6200 msgr2=0x5595e94d7330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:03.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.759+0000 7fe7327fc640 1 --2- 172.21.15.32:0/1111711835 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5595e94d6200 0x5595e94d7330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.759+0000 7fe7327fc640 1 -- 172.21.15.32:0/1111711835 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5595e94d7870 con 0x5595e94cb480 2024-04-25T00:35:03.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.759+0000 7fe7327fc640 1 --2- 172.21.15.32:0/1111711835 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94cb480 0x5595e94de4d0 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7fe724005c20 tx=0x7fe724009520 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3a5af2ede89ac1f4 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:03.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.760+0000 7fe716ff5640 10 client.0 ms_handle_connect on v2:172.21.15.121:3300/0 2024-04-25T00:35:03.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.760+0000 7fe716ff5640 1 -- 172.21.15.32:0/1111711835 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe724059020 con 0x5595e94cb480 2024-04-25T00:35:03.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.760+0000 7fe73c1dd740 1 -- 172.21.15.32:0/1111711835 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe728013670 con 0x5595e94cb480 2024-04-25T00:35:03.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.760+0000 7fe716ff5640 1 -- 172.21.15.32:0/1111711835 <== mon.2 v2:172.21.15.121:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fe724055ae0 con 0x5595e94cb480 2024-04-25T00:35:03.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.760+0000 7fe716ff5640 1 -- 172.21.15.32:0/1111711835 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7fe724061050 con 0x5595e94cb480 2024-04-25T00:35:03.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.763+0000 7fe73c1dd740 10 client.25574 fetch_fsmap learned FSMap version 26 2024-04-25T00:35:03.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.763+0000 7fe73c1dd740 1 -- 172.21.15.32:0/1111711835 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x5595e954db40 con 0x5595e94cb480 2024-04-25T00:35:03.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.763+0000 7fe716ff5640 1 -- 172.21.15.32:0/1111711835 <== mon.2 v2:172.21.15.121:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe728013670 con 0x5595e94cb480 2024-04-25T00:35:03.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.763+0000 7fe716ff5640 1 -- 172.21.15.32:0/1111711835 <== mon.2 v2:172.21.15.121:3300/0 5 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7fe724055ce0 con 0x5595e94cb480 2024-04-25T00:35:03.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.764+0000 7fe73c1dd740 10 client.25574 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:35:03.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.764+0000 7fe73c1dd740 10 client.25574 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:35:03.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.764+0000 7fe73c1dd740 20 client.25574 populate_metadata read hostname 'smithi032' 2024-04-25T00:35:03.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.764+0000 7fe73c1dd740 1 --2- 172.21.15.32:0/1111711835 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5595e954e1f0 0x5595e9550610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:03.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.764+0000 7fe73c1dd740 4 client.25574 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-25T00:35:03.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.764+0000 7fe73c1dd740 1 -- 172.21.15.32:0/1111711835 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) -- 0x5595e943c800 con 0x5595e954e1f0 2024-04-25T00:35:03.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.764+0000 7fe732ffd640 1 --2- 172.21.15.32:0/1111711835 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5595e954e1f0 0x5595e9550610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:03.772 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.771+0000 7fe732ffd640 1 --2- 172.21.15.32:0/1111711835 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5595e954e1f0 0x5595e9550610 crc :-1 s=READY pgs=66 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=cc2732a112406770 server_cookie=58fa2a14bbda4af0 in_seq=0 out_seq=0 2024-04-25T00:35:03.772 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.771+0000 7fe716ff5640 10 client.25574 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:35:03.774 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.772+0000 7fe716ff5640 1 -- 172.21.15.32:0/1111711835 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+110 (crc 0 0 0) 0x5595e943c800 con 0x5595e954e1f0 2024-04-25T00:35:03.774 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.773+0000 7fe716ff5640 10 client.25574 handle_command_reply: tid=0 2024-04-25T00:35:03.774 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.773+0000 7fe73c1dd740 1 client.25574 shutdown 2024-04-25T00:35:03.775 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.773+0000 7fe73c1dd740 1 -- 172.21.15.32:0/1111711835 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94cb480 msgr2=0x5595e94de4d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:03.775 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.773+0000 7fe73c1dd740 1 --2- 172.21.15.32:0/1111711835 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94cb480 0x5595e94de4d0 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7fe724005c20 tx=0x7fe724009520 comp rx=0 tx=0).stop 2024-04-25T00:35:03.775 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.773+0000 7fe73c1dd740 1 -- 172.21.15.32:0/1111711835 shutdown_connections 2024-04-25T00:35:03.775 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.773+0000 7fe73c1dd740 1 --2- 172.21.15.32:0/1111711835 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5595e954e1f0 0x5595e9550610 crc :-1 s=READY pgs=66 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.775 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.773+0000 7fe73c1dd740 1 --2- 172.21.15.32:0/1111711835 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5595e94d6200 0x5595e94d7330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.775 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.773+0000 7fe73c1dd740 1 --2- 172.21.15.32:0/1111711835 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5595e94d3e30 0x5595e94dea10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.775 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.773+0000 7fe73c1dd740 1 --2- 172.21.15.32:0/1111711835 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5595e94cb480 0x5595e94de4d0 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.775 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.773+0000 7fe73c1dd740 1 -- 172.21.15.32:0/1111711835 >> 172.21.15.32:0/1111711835 conn(0x5595e9493950 msgr2=0x5595e94c62b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:03.775 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.773+0000 7fe716ff5640 0 client.25574 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:35:03.778 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.777+0000 7fe73c1dd740 1 -- 172.21.15.32:0/1111711835 shutdown_connections 2024-04-25T00:35:03.778 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.777+0000 7fe73c1dd740 1 -- 172.21.15.32:0/1111711835 wait complete. 2024-04-25T00:35:03.778 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.777+0000 7fe73c1dd740 20 client.25574 trim_cache size 0 max 16384 2024-04-25T00:35:03.780 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:35:03.780 INFO:teuthology.orchestra.run.smithi032.stdout: "return_code": 0, 2024-04-25T00:35:03.780 INFO:teuthology.orchestra.run.smithi032.stdout: "scrub_tag": "dcf3868e-a5ff-4ef9-b62a-fbe3ab2421c6", 2024-04-25T00:35:03.781 INFO:teuthology.orchestra.run.smithi032.stdout: "mode": "asynchronous" 2024-04-25T00:35:03.781 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:35:03.781 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.780+0000 7fe7394d2640 1 -- 172.21.15.32:0/1863151233 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe71c0a3060 msgr2=0x7fe71c0a5520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:03.782 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.780+0000 7fe7394d2640 1 --2- 172.21.15.32:0/1863151233 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe71c0a3060 0x7fe71c0a5520 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7fe72c0cd3e0 tx=0x7fe72c098390 comp rx=0 tx=0).stop 2024-04-25T00:35:03.782 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.780+0000 7fe7394d2640 1 -- 172.21.15.32:0/1863151233 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe734114e00 msgr2=0x7fe7341152b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:03.782 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.780+0000 7fe7394d2640 1 --2- 172.21.15.32:0/1863151233 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe734114e00 0x7fe7341152b0 secure :-1 s=READY pgs=593 cs=0 l=1 rev1=1 crypto rx=0x7fe72c00b970 tx=0x7fe72c00be30 comp rx=0 tx=0).stop 2024-04-25T00:35:03.785 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.783+0000 7fe7394d2640 1 -- 172.21.15.32:0/1863151233 shutdown_connections 2024-04-25T00:35:03.786 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.783+0000 7fe7394d2640 1 --2- 172.21.15.32:0/1863151233 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe71c0a3060 0x7fe71c0a5520 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.786 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.784+0000 7fe7394d2640 1 --2- 172.21.15.32:0/1863151233 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe734114e00 0x7fe7341152b0 unknown :-1 s=CLOSED pgs=593 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.786 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.784+0000 7fe7394d2640 1 --2- 172.21.15.32:0/1863151233 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe73411af20 0x7fe7341148c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.786 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.784+0000 7fe7394d2640 1 --2- 172.21.15.32:0/1863151233 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe7341087d0 0x7fe73411a800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:03.786 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.784+0000 7fe7394d2640 1 -- 172.21.15.32:0/1863151233 >> 172.21.15.32:0/1863151233 conn(0x7fe73406c720 msgr2=0x7fe73411b300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:03.786 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.784+0000 7fe7394d2640 1 -- 172.21.15.32:0/1863151233 shutdown_connections 2024-04-25T00:35:03.786 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:03.784+0000 7fe7394d2640 1 -- 172.21.15.32:0/1863151233 wait complete. 2024-04-25T00:35:03.812 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-25T00:35:03.813 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-25T00:35:04.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.031+0000 7fc83359e640 1 -- 172.21.15.32:0/1864559997 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc83410d560 msgr2=0x7fc834115130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:04.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.031+0000 7fc83359e640 1 --2- 172.21.15.32:0/1864559997 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc83410d560 0x7fc834115130 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7fc82c005fd0 tx=0x7fc82c02b430 comp rx=0 tx=0).stop 2024-04-25T00:35:04.036 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.034+0000 7fc83359e640 1 -- 172.21.15.32:0/1864559997 shutdown_connections 2024-04-25T00:35:04.036 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.034+0000 7fc83359e640 1 --2- 172.21.15.32:0/1864559997 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc83410d560 0x7fc834115130 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.036 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.034+0000 7fc83359e640 1 --2- 172.21.15.32:0/1864559997 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc834109360 0x7fc83410ce80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.036 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.034+0000 7fc83359e640 1 --2- 172.21.15.32:0/1864559997 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc8341089b0 0x7fc834108d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.036 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.034+0000 7fc83359e640 1 -- 172.21.15.32:0/1864559997 >> 172.21.15.32:0/1864559997 conn(0x7fc83406cb20 msgr2=0x7fc83406cf30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:04.041 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.039+0000 7fc83359e640 1 -- 172.21.15.32:0/1864559997 shutdown_connections 2024-04-25T00:35:04.047 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.045+0000 7fc83359e640 1 -- 172.21.15.32:0/1864559997 wait complete. 2024-04-25T00:35:04.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.047+0000 7fc83359e640 1 Processor -- start 2024-04-25T00:35:04.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.047+0000 7fc83359e640 1 -- start start 2024-04-25T00:35:04.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.048+0000 7fc83359e640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc8341089b0 0x7fc8341a45c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:04.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.048+0000 7fc83359e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc83410d560 0x7fc8341a4b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:04.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.048+0000 7fc83359e640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc83419e680 0x7fc83419eb00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:04.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.048+0000 7fc83359e640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fc8341182a0 con 0x7fc83410d560 2024-04-25T00:35:04.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.048+0000 7fc83359e640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fc834118120 con 0x7fc83419e680 2024-04-25T00:35:04.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.049+0000 7fc83359e640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fc834118420 con 0x7fc8341089b0 2024-04-25T00:35:04.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.049+0000 7fc832d9d640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc83419e680 0x7fc83419eb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:04.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.052+0000 7fc831d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc83410d560 0x7fc8341a4b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:04.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.053+0000 7fc83259c640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc8341089b0 0x7fc8341a45c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:04.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.053+0000 7fc831d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc83410d560 0x7fc8341a4b00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53384/0 (socket says 172.21.15.32:53384) 2024-04-25T00:35:04.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.053+0000 7fc831d9b640 1 -- 172.21.15.32:0/4238051780 learned_addr learned my addr 172.21.15.32:0/4238051780 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:04.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.054+0000 7fc83259c640 1 -- 172.21.15.32:0/4238051780 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc83419e680 msgr2=0x7fc83419eb00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:04.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.054+0000 7fc83259c640 1 --2- 172.21.15.32:0/4238051780 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc83419e680 0x7fc83419eb00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.054+0000 7fc83259c640 1 -- 172.21.15.32:0/4238051780 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc83410d560 msgr2=0x7fc8341a4b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:04.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.054+0000 7fc83259c640 1 --2- 172.21.15.32:0/4238051780 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc83410d560 0x7fc8341a4b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.054+0000 7fc83259c640 1 -- 172.21.15.32:0/4238051780 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc83419f280 con 0x7fc8341089b0 2024-04-25T00:35:04.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.054+0000 7fc83259c640 1 --2- 172.21.15.32:0/4238051780 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc8341089b0 0x7fc8341a45c0 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7fc82400b3f0 tx=0x7fc82400b8b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:04.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.056+0000 7fc8237fe640 1 -- 172.21.15.32:0/4238051780 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc824015020 con 0x7fc8341089b0 2024-04-25T00:35:04.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.056+0000 7fc8237fe640 1 -- 172.21.15.32:0/4238051780 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc824011d10 con 0x7fc8341089b0 2024-04-25T00:35:04.059 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.057+0000 7fc8237fe640 1 -- 172.21.15.32:0/4238051780 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc82400f2e0 con 0x7fc8341089b0 2024-04-25T00:35:04.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.058+0000 7fc83359e640 1 -- 172.21.15.32:0/4238051780 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc8341ab360 con 0x7fc8341089b0 2024-04-25T00:35:04.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.058+0000 7fc83359e640 1 -- 172.21.15.32:0/4238051780 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc8341ab710 con 0x7fc8341089b0 2024-04-25T00:35:04.065 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.063+0000 7fc83b873740 1 Processor -- start 2024-04-25T00:35:04.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.066+0000 7fc8237fe640 1 -- 172.21.15.32:0/4238051780 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 25) ==== 138066+0+0 (secure 0 0 0) 0x7fc824018020 con 0x7fc8341089b0 2024-04-25T00:35:04.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.067+0000 7fc8237fe640 1 --2- 172.21.15.32:0/4238051780 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fc81c0a2f10 0x7fc81c0a53d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:04.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.068+0000 7fc8237fe640 1 -- 172.21.15.32:0/4238051780 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fc8240cf060 con 0x7fc8341089b0 2024-04-25T00:35:04.071 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.069+0000 7fc83b873740 1 -- start start 2024-04-25T00:35:04.071 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.069+0000 7fc83b873740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555576569010 0x5555765693f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:04.071 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.069+0000 7fc83b873740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555576569930 0x555576573ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:04.071 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.069+0000 7fc83b873740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555576574000 0x5555765763f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:04.071 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.069+0000 7fc83b873740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5555764c56b0 con 0x555576569930 2024-04-25T00:35:04.071 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.069+0000 7fc83b873740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55557646d880 con 0x555576569010 2024-04-25T00:35:04.071 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.069+0000 7fc83b873740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5555764c1670 con 0x555576574000 2024-04-25T00:35:04.073 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.070+0000 7fc831d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555576569930 0x555576573ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:04.073 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.070+0000 7fc831d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555576569930 0x555576573ac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:53390/0 (socket says 172.21.15.32:53390) 2024-04-25T00:35:04.073 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.070+0000 7fc831d9b640 1 -- 172.21.15.32:0/862246459 learned_addr learned my addr 172.21.15.32:0/862246459 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:04.073 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.071+0000 7fc831d9b640 1 -- 172.21.15.32:0/862246459 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555576574000 msgr2=0x5555765763f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:04.073 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.071+0000 7fc831d9b640 1 --2- 172.21.15.32:0/862246459 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555576574000 0x5555765763f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.074 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.071+0000 7fc831d9b640 1 -- 172.21.15.32:0/862246459 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555576569010 msgr2=0x5555765693f0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-25T00:35:04.074 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.071+0000 7fc831d9b640 1 --2- 172.21.15.32:0/862246459 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555576569010 0x5555765693f0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.074 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.071+0000 7fc831d9b640 1 -- 172.21.15.32:0/862246459 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5555762a37d0 con 0x555576569930 2024-04-25T00:35:04.074 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.071+0000 7fc831d9b640 1 --2- 172.21.15.32:0/862246459 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555576569930 0x555576573ac0 secure :-1 s=READY pgs=595 cs=0 l=1 rev1=1 crypto rx=0x7fc828009860 tx=0x7fc828009d20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=12981192b0f36092 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:04.077 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.074+0000 7fc820ff9640 1 -- 172.21.15.32:0/862246459 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc82800d950 con 0x555576569930 2024-04-25T00:35:04.077 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.074+0000 7fc820ff9640 1 -- 172.21.15.32:0/862246459 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc828014430 con 0x555576569930 2024-04-25T00:35:04.077 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.075+0000 7fc83b873740 1 -- 172.21.15.32:0/862246459 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555576569930 msgr2=0x555576573ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:04.077 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.075+0000 7fc83b873740 1 --2- 172.21.15.32:0/862246459 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555576569930 0x555576573ac0 secure :-1 s=READY pgs=595 cs=0 l=1 rev1=1 crypto rx=0x7fc828009860 tx=0x7fc828009d20 comp rx=0 tx=0).stop 2024-04-25T00:35:04.077 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.076+0000 7fc832d9d640 1 --2- 172.21.15.32:0/4238051780 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fc81c0a2f10 0x7fc81c0a53d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:04.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.077+0000 7fc83b873740 1 -- 172.21.15.32:0/862246459 shutdown_connections 2024-04-25T00:35:04.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.077+0000 7fc83b873740 1 --2- 172.21.15.32:0/862246459 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555576574000 0x5555765763f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.077+0000 7fc83b873740 1 --2- 172.21.15.32:0/862246459 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555576569930 0x555576573ac0 unknown :-1 s=CLOSED pgs=595 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.077+0000 7fc83b873740 1 --2- 172.21.15.32:0/862246459 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555576569010 0x5555765693f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.077+0000 7fc83b873740 1 -- 172.21.15.32:0/862246459 >> 172.21.15.32:0/862246459 conn(0x555576565330 msgr2=0x555576565710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:04.086 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.084+0000 7fc83b873740 1 -- 172.21.15.32:0/862246459 shutdown_connections 2024-04-25T00:35:04.086 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.084+0000 7fc83b873740 1 -- 172.21.15.32:0/862246459 wait complete. 2024-04-25T00:35:04.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.085+0000 7fc83b873740 1 -- start start 2024-04-25T00:35:04.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.086+0000 7fc83b873740 1 Processor -- start 2024-04-25T00:35:04.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.089+0000 7fc83b873740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555576569010 0x555576570f00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:04.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.089+0000 7fc83b873740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555576569930 0x555576571440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:04.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.089+0000 7fc83b873740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555576574000 0x555576571980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:04.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.089+0000 7fc83b873740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x555576576e40 con 0x555576569010 2024-04-25T00:35:04.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.089+0000 7fc83b873740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x555576480f50 con 0x555576569930 2024-04-25T00:35:04.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.089+0000 7fc83b873740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x555576576fc0 con 0x555576574000 2024-04-25T00:35:04.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.089+0000 7fc83b873740 10 client.0 _ll_register_callbacks cb 0x5555764bc000 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:35:04.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.090+0000 7fc832d9d640 1 --2- 172.21.15.32:0/4238051780 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fc81c0a2f10 0x7fc81c0a53d0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x5555765664c0 tx=0x7fc82c00d000 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:04.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.090+0000 7fc832d9d640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555576574000 0x555576571980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:04.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.090+0000 7fc832d9d640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555576574000 0x555576571980 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:54776/0 (socket says 172.21.15.32:54776) 2024-04-25T00:35:04.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.090+0000 7fc831d9b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555576569930 0x555576571440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:04.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.090+0000 7fc831d9b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555576569930 0x555576571440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:43924/0 (socket says 172.21.15.32:43924) 2024-04-25T00:35:04.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.090+0000 7fc831d9b640 1 -- 172.21.15.32:0/423925184 learned_addr learned my addr 172.21.15.32:0/423925184 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:04.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.091+0000 7fc83259c640 1 --2- 172.21.15.32:0/423925184 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555576569010 0x555576570f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:04.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.092+0000 7fc83259c640 1 -- 172.21.15.32:0/423925184 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555576574000 msgr2=0x555576571980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:04.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.092+0000 7fc83259c640 1 --2- 172.21.15.32:0/423925184 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555576574000 0x555576571980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.092+0000 7fc83259c640 1 -- 172.21.15.32:0/423925184 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555576569930 msgr2=0x555576571440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:04.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.092+0000 7fc83259c640 1 --2- 172.21.15.32:0/423925184 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555576569930 0x555576571440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.092+0000 7fc83259c640 1 -- 172.21.15.32:0/423925184 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5555761d93f0 con 0x555576569010 2024-04-25T00:35:04.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.092+0000 7fc83259c640 1 --2- 172.21.15.32:0/423925184 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555576569010 0x555576570f00 secure :-1 s=READY pgs=596 cs=0 l=1 rev1=1 crypto rx=0x7fc8240cc330 tx=0x7fc82409b030 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a464e46ba60fbf64 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:04.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.092+0000 7fc819ffb640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:35:04.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.093+0000 7fc819ffb640 1 -- 172.21.15.32:0/423925184 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc8240b3070 con 0x555576569010 2024-04-25T00:35:04.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.093+0000 7fc83b873740 1 -- 172.21.15.32:0/423925184 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x5555764d2bf0 con 0x555576569010 2024-04-25T00:35:04.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.093+0000 7fc819ffb640 1 -- 172.21.15.32:0/423925184 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fc8240976a0 con 0x555576569010 2024-04-25T00:35:04.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.094+0000 7fc819ffb640 1 -- 172.21.15.32:0/423925184 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7fc8240a3810 con 0x555576569010 2024-04-25T00:35:04.097 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.095+0000 7fc83b873740 10 client.16029 fetch_fsmap learned FSMap version 26 2024-04-25T00:35:04.097 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.095+0000 7fc83b873740 1 -- 172.21.15.32:0/423925184 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x5555761990f0 con 0x555576569010 2024-04-25T00:35:04.098 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.096+0000 7fc819ffb640 1 -- 172.21.15.32:0/423925184 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x5555764d2bf0 con 0x555576569010 2024-04-25T00:35:04.098 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.097+0000 7fc819ffb640 1 -- 172.21.15.32:0/423925184 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7fc824022060 con 0x555576569010 2024-04-25T00:35:04.106 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.105+0000 7fc83b873740 10 client.16029 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:35:04.107 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.105+0000 7fc83b873740 10 client.16029 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:35:04.107 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.105+0000 7fc83b873740 20 client.16029 populate_metadata read hostname 'smithi032' 2024-04-25T00:35:04.107 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.105+0000 7fc83b873740 1 --2- 172.21.15.32:0/423925184 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5555765604b0 0x555576562900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:04.107 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.105+0000 7fc83b873740 4 client.16029 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:35:04.107 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.105+0000 7fc83b873740 1 -- 172.21.15.32:0/423925184 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5555764ccf40 con 0x5555765604b0 2024-04-25T00:35:04.111 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.109+0000 7fc831d9b640 1 --2- 172.21.15.32:0/423925184 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5555765604b0 0x555576562900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:04.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.117+0000 7fc831d9b640 1 --2- 172.21.15.32:0/423925184 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5555765604b0 0x555576562900 crc :-1 s=READY pgs=67 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=6cd7654b2f1a6091 server_cookie=ba016eda8026a6a5 in_seq=0 out_seq=0 2024-04-25T00:35:04.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.117+0000 7fc819ffb640 10 client.16029 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:35:04.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.120+0000 7fc819ffb640 1 -- 172.21.15.32:0/423925184 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x5555764ccf40 con 0x5555765604b0 2024-04-25T00:35:04.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.120+0000 7fc819ffb640 10 client.16029 handle_command_reply: tid=0 2024-04-25T00:35:04.124 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.122+0000 7fc83b873740 1 client.16029 shutdown 2024-04-25T00:35:04.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.131+0000 7fc83b873740 1 -- 172.21.15.32:0/423925184 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555576569010 msgr2=0x555576570f00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:04.132 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.131+0000 7fc83b873740 1 --2- 172.21.15.32:0/423925184 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555576569010 0x555576570f00 secure :-1 s=READY pgs=596 cs=0 l=1 rev1=1 crypto rx=0x7fc8240cc330 tx=0x7fc82409b030 comp rx=0 tx=0).stop 2024-04-25T00:35:04.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.135+0000 7fc83b873740 1 -- 172.21.15.32:0/423925184 shutdown_connections 2024-04-25T00:35:04.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.135+0000 7fc83b873740 1 --2- 172.21.15.32:0/423925184 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5555765604b0 0x555576562900 crc :-1 s=READY pgs=67 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.135+0000 7fc83b873740 1 --2- 172.21.15.32:0/423925184 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555576574000 0x555576571980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.135+0000 7fc83b873740 1 --2- 172.21.15.32:0/423925184 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555576569930 0x555576571440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.135+0000 7fc83b873740 1 --2- 172.21.15.32:0/423925184 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555576569010 0x555576570f00 unknown :-1 s=CLOSED pgs=596 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.135+0000 7fc83b873740 1 -- 172.21.15.32:0/423925184 >> 172.21.15.32:0/423925184 conn(0x555576565330 msgr2=0x55557656b130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:04.137 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.135+0000 7fc819ffb640 0 client.16029 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:35:04.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.139+0000 7fc83b873740 1 -- 172.21.15.32:0/423925184 shutdown_connections 2024-04-25T00:35:04.143 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.142+0000 7fc83b873740 1 -- 172.21.15.32:0/423925184 wait complete. 2024-04-25T00:35:04.143 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.142+0000 7fc83b873740 20 client.16029 trim_cache size 0 max 16384 2024-04-25T00:35:04.187 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.185+0000 7fc83b873740 1 Processor -- start 2024-04-25T00:35:04.187 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.185+0000 7fc83b873740 1 -- start start 2024-04-25T00:35:04.188 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.186+0000 7fc83b873740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555576569010 0x555576573630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:04.188 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.186+0000 7fc83b873740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555576569930 0x55557657fdc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:04.188 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.186+0000 7fc83b873740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555576574000 0x55557656af60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:04.188 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.186+0000 7fc83b873740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fc8341b77a0 con 0x555576569010 2024-04-25T00:35:04.188 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.186+0000 7fc83b873740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fc8341b5070 con 0x555576569930 2024-04-25T00:35:04.188 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.187+0000 7fc83b873740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fc8341b2ec0 con 0x555576574000 2024-04-25T00:35:04.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.187+0000 7fc831d9b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555576569930 0x55557657fdc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:04.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.187+0000 7fc831d9b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555576569930 0x55557657fdc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:43932/0 (socket says 172.21.15.32:43932) 2024-04-25T00:35:04.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.187+0000 7fc831d9b640 1 -- 172.21.15.32:0/3049511740 learned_addr learned my addr 172.21.15.32:0/3049511740 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:04.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.187+0000 7fc832d9d640 1 --2- 172.21.15.32:0/3049511740 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555576574000 0x55557656af60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:04.189 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.187+0000 7fc83259c640 1 --2- 172.21.15.32:0/3049511740 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555576569010 0x555576573630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:04.190 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.188+0000 7fc831d9b640 1 -- 172.21.15.32:0/3049511740 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555576574000 msgr2=0x55557656af60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:04.190 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.188+0000 7fc831d9b640 1 --2- 172.21.15.32:0/3049511740 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555576574000 0x55557656af60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.190 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.188+0000 7fc831d9b640 1 -- 172.21.15.32:0/3049511740 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555576569010 msgr2=0x555576573630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:04.190 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.188+0000 7fc831d9b640 1 --2- 172.21.15.32:0/3049511740 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555576569010 0x555576573630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.190 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.188+0000 7fc831d9b640 1 -- 172.21.15.32:0/3049511740 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x555576568be0 con 0x555576569930 2024-04-25T00:35:04.191 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.188+0000 7fc831d9b640 1 --2- 172.21.15.32:0/3049511740 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555576569930 0x55557657fdc0 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7fc828006e70 tx=0x7fc828006de0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e63115cfa85d3abc server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:04.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.192+0000 7fc81bfff640 1 -- 172.21.15.32:0/3049511740 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc828029070 con 0x555576569930 2024-04-25T00:35:04.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.192+0000 7fc81bfff640 1 -- 172.21.15.32:0/3049511740 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc828004620 con 0x555576569930 2024-04-25T00:35:04.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.192+0000 7fc81bfff640 1 -- 172.21.15.32:0/3049511740 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc8280039c0 con 0x555576569930 2024-04-25T00:35:04.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.194+0000 7fc83b873740 1 -- 172.21.15.32:0/3049511740 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555576569930 msgr2=0x55557657fdc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:04.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.194+0000 7fc83b873740 1 --2- 172.21.15.32:0/3049511740 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555576569930 0x55557657fdc0 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7fc828006e70 tx=0x7fc828006de0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.194+0000 7fc83b873740 1 -- 172.21.15.32:0/3049511740 shutdown_connections 2024-04-25T00:35:04.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.194+0000 7fc83b873740 1 --2- 172.21.15.32:0/3049511740 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555576574000 0x55557656af60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.194+0000 7fc83b873740 1 --2- 172.21.15.32:0/3049511740 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555576569930 0x55557657fdc0 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.196 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.194+0000 7fc83b873740 1 --2- 172.21.15.32:0/3049511740 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555576569010 0x555576573630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.196 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.194+0000 7fc83b873740 1 -- 172.21.15.32:0/3049511740 >> 172.21.15.32:0/3049511740 conn(0x555576565330 msgr2=0x55557656a850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:04.197 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.196+0000 7fc83b873740 1 -- 172.21.15.32:0/3049511740 shutdown_connections 2024-04-25T00:35:04.198 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.196+0000 7fc83b873740 1 -- 172.21.15.32:0/3049511740 wait complete. 2024-04-25T00:35:04.198 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.196+0000 7fc83b873740 1 -- start start 2024-04-25T00:35:04.198 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.196+0000 7fc83b873740 1 Processor -- start 2024-04-25T00:35:04.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.197+0000 7fc83b873740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555576569010 0x555576563890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:04.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.197+0000 7fc83b873740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555576569930 0x555576563dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:04.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.197+0000 7fc83b873740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555576574000 0x555576564310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:04.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.197+0000 7fc83b873740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fc8341b3530 con 0x555576569010 2024-04-25T00:35:04.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.197+0000 7fc83b873740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fc8341b49a0 con 0x555576574000 2024-04-25T00:35:04.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.197+0000 7fc83b873740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fc8341b3c00 con 0x555576569930 2024-04-25T00:35:04.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.197+0000 7fc83b873740 10 client.0 _ll_register_callbacks cb 0x7fc82409aa00 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:35:04.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.197+0000 7fc831d9b640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555576569930 0x555576563dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:04.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.197+0000 7fc831d9b640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555576569930 0x555576563dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:54796/0 (socket says 172.21.15.32:54796) 2024-04-25T00:35:04.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.197+0000 7fc831d9b640 1 -- 172.21.15.32:0/1086962944 learned_addr learned my addr 172.21.15.32:0/1086962944 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:04.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.198+0000 7fc831d9b640 1 -- 172.21.15.32:0/1086962944 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555576574000 msgr2=0x555576564310 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-25T00:35:04.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.198+0000 7fc831d9b640 1 --2- 172.21.15.32:0/1086962944 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555576574000 0x555576564310 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.198+0000 7fc831d9b640 1 -- 172.21.15.32:0/1086962944 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555576569010 msgr2=0x555576563890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:04.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.198+0000 7fc831d9b640 1 --2- 172.21.15.32:0/1086962944 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555576569010 0x555576563890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.198+0000 7fc831d9b640 1 -- 172.21.15.32:0/1086962944 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55557657a690 con 0x555576569930 2024-04-25T00:35:04.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.198+0000 7fc831d9b640 1 --2- 172.21.15.32:0/1086962944 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555576569930 0x555576563dd0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7fc828000c00 tx=0x7fc82800d9e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d04606c509b82a12 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:04.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.201+0000 7fc819ffb640 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:35:04.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.201+0000 7fc819ffb640 1 -- 172.21.15.32:0/1086962944 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc828029070 con 0x555576569930 2024-04-25T00:35:04.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.201+0000 7fc83b873740 1 -- 172.21.15.32:0/1086962944 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x55557644a6f0 con 0x555576569930 2024-04-25T00:35:04.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.201+0000 7fc819ffb640 1 -- 172.21.15.32:0/1086962944 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fc828005990 con 0x555576569930 2024-04-25T00:35:04.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.202+0000 7fc819ffb640 1 -- 172.21.15.32:0/1086962944 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7fc82803a050 con 0x555576569930 2024-04-25T00:35:04.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.202+0000 7fc83b873740 10 client.25564 fetch_fsmap learned FSMap version 26 2024-04-25T00:35:04.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.202+0000 7fc83b873740 1 -- 172.21.15.32:0/1086962944 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x5555765f2ff0 con 0x555576569930 2024-04-25T00:35:04.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.203+0000 7fc819ffb640 1 -- 172.21.15.32:0/1086962944 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x55557644a6f0 con 0x555576569930 2024-04-25T00:35:04.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.203+0000 7fc819ffb640 1 -- 172.21.15.32:0/1086962944 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7fc828024040 con 0x555576569930 2024-04-25T00:35:04.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.203+0000 7fc83b873740 10 client.25564 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:35:04.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.203+0000 7fc83b873740 10 client.25564 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:35:04.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.203+0000 7fc83b873740 20 client.25564 populate_metadata read hostname 'smithi032' 2024-04-25T00:35:04.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.204+0000 7fc83b873740 1 --2- 172.21.15.32:0/1086962944 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5555765f3660 0x5555765f5a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:04.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.204+0000 7fc83b873740 4 client.25564 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-25T00:35:04.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.204+0000 7fc83b873740 1 -- 172.21.15.32:0/1086962944 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "scrub status"}) -- 0x555576496d50 con 0x5555765f3660 2024-04-25T00:35:04.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.204+0000 7fc83259c640 1 --2- 172.21.15.32:0/1086962944 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5555765f3660 0x5555765f5a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:04.206 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.204+0000 7fc819ffb640 10 client.25564 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:35:04.210 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.209+0000 7fc83259c640 1 --2- 172.21.15.32:0/1086962944 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5555765f3660 0x5555765f5a80 crc :-1 s=READY pgs=68 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=baec9e0f1d4a6be9 server_cookie=9f4df4f770e5e4fe in_seq=0 out_seq=0 2024-04-25T00:35:04.215 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.214+0000 7fc819ffb640 1 -- 172.21.15.32:0/1086962944 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+265 (crc 0 0 0) 0x555576496d50 con 0x5555765f3660 2024-04-25T00:35:04.215 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.214+0000 7fc819ffb640 10 client.25564 handle_command_reply: tid=0 2024-04-25T00:35:04.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.216+0000 7fc83b873740 1 client.25564 shutdown 2024-04-25T00:35:04.222 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.221+0000 7fc83b873740 1 -- 172.21.15.32:0/1086962944 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555576569930 msgr2=0x555576563dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:04.222 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.221+0000 7fc83b873740 1 --2- 172.21.15.32:0/1086962944 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555576569930 0x555576563dd0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7fc828000c00 tx=0x7fc82800d9e0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.224 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.223+0000 7fc83b873740 1 -- 172.21.15.32:0/1086962944 shutdown_connections 2024-04-25T00:35:04.224 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.223+0000 7fc83b873740 1 --2- 172.21.15.32:0/1086962944 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5555765f3660 0x5555765f5a80 crc :-1 s=READY pgs=68 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.224 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.223+0000 7fc83b873740 1 --2- 172.21.15.32:0/1086962944 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555576574000 0x555576564310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.224 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.223+0000 7fc83b873740 1 --2- 172.21.15.32:0/1086962944 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555576569930 0x555576563dd0 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.224 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.223+0000 7fc83b873740 1 --2- 172.21.15.32:0/1086962944 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555576569010 0x555576563890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.223+0000 7fc83b873740 1 -- 172.21.15.32:0/1086962944 >> 172.21.15.32:0/1086962944 conn(0x555576565330 msgr2=0x55557656a830 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:04.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.224+0000 7fc819ffb640 0 client.25564 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:35:04.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.224+0000 7fc83b873740 1 -- 172.21.15.32:0/1086962944 shutdown_connections 2024-04-25T00:35:04.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.224+0000 7fc83b873740 1 -- 172.21.15.32:0/1086962944 wait complete. 2024-04-25T00:35:04.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.229+0000 7fc83b873740 20 client.25564 trim_cache size 0 max 16384 2024-04-25T00:35:04.235 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:35:04.236 INFO:teuthology.orchestra.run.smithi032.stdout: "status": "scrub active (1 inodes in the stack)", 2024-04-25T00:35:04.236 INFO:teuthology.orchestra.run.smithi032.stdout: "scrubs": { 2024-04-25T00:35:04.236 INFO:teuthology.orchestra.run.smithi032.stdout: "dcf3868e-a5ff-4ef9-b62a-fbe3ab2421c6": { 2024-04-25T00:35:04.236 INFO:teuthology.orchestra.run.smithi032.stdout: "path": "/", 2024-04-25T00:35:04.236 INFO:teuthology.orchestra.run.smithi032.stdout: "tag": "dcf3868e-a5ff-4ef9-b62a-fbe3ab2421c6", 2024-04-25T00:35:04.236 INFO:teuthology.orchestra.run.smithi032.stdout: "options": "recursive,force" 2024-04-25T00:35:04.236 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:35:04.236 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:35:04.236 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:35:04.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.235+0000 7fc83359e640 1 -- 172.21.15.32:0/4238051780 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fc81c0a2f10 msgr2=0x7fc81c0a53d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:04.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.235+0000 7fc83359e640 1 --2- 172.21.15.32:0/4238051780 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fc81c0a2f10 0x7fc81c0a53d0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x5555765664c0 tx=0x7fc82c00d000 comp rx=0 tx=0).stop 2024-04-25T00:35:04.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.235+0000 7fc83359e640 1 -- 172.21.15.32:0/4238051780 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc8341089b0 msgr2=0x7fc8341a45c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:04.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.235+0000 7fc83359e640 1 --2- 172.21.15.32:0/4238051780 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc8341089b0 0x7fc8341a45c0 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7fc82400b3f0 tx=0x7fc82400b8b0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.236+0000 7fc83359e640 1 -- 172.21.15.32:0/4238051780 shutdown_connections 2024-04-25T00:35:04.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.236+0000 7fc83359e640 1 --2- 172.21.15.32:0/4238051780 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fc81c0a2f10 0x7fc81c0a53d0 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.236+0000 7fc83359e640 1 --2- 172.21.15.32:0/4238051780 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc83419e680 0x7fc83419eb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.236+0000 7fc83359e640 1 --2- 172.21.15.32:0/4238051780 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc83410d560 0x7fc8341a4b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.236+0000 7fc83359e640 1 --2- 172.21.15.32:0/4238051780 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc8341089b0 0x7fc8341a45c0 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:04.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.236+0000 7fc83359e640 1 -- 172.21.15.32:0/4238051780 >> 172.21.15.32:0/4238051780 conn(0x7fc83406cb20 msgr2=0x7fc8340714b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:04.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.237+0000 7fc83359e640 1 -- 172.21.15.32:0/4238051780 shutdown_connections 2024-04-25T00:35:04.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:04.237+0000 7fc83359e640 1 -- 172.21.15.32:0/4238051780 wait complete. 2024-04-25T00:35:04.272 INFO:tasks.cephfs.filesystem:scrub status for tag:dcf3868e-a5ff-4ef9-b62a-fbe3ab2421c6 - {'path': '/', 'tag': 'dcf3868e-a5ff-4ef9-b62a-fbe3ab2421c6', 'options': 'recursive,force'} 2024-04-25T00:35:05.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:05 smithi121 ceph-mon[32231]: scrub queued for path: / 2024-04-25T00:35:05.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:05 smithi121 ceph-mon[32231]: scrub summary: idle+waiting paths [/] 2024-04-25T00:35:05.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:05 smithi121 ceph-mon[32231]: scrub summary: active paths [/] 2024-04-25T00:35:05.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:05 smithi121 ceph-mon[32231]: pgmap v56: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 2.1 KiB/s rd, 860 MiB/s wr, 1.73k op/s 2024-04-25T00:35:05.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:05 smithi130 ceph-mon[32525]: scrub queued for path: / 2024-04-25T00:35:05.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:05 smithi130 ceph-mon[32525]: scrub summary: idle+waiting paths [/] 2024-04-25T00:35:05.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:05 smithi130 ceph-mon[32525]: scrub summary: active paths [/] 2024-04-25T00:35:05.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:05 smithi130 ceph-mon[32525]: pgmap v56: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 2.1 KiB/s rd, 860 MiB/s wr, 1.73k op/s 2024-04-25T00:35:05.756 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:05 smithi032 ceph-mon[29357]: scrub queued for path: / 2024-04-25T00:35:05.756 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:05 smithi032 ceph-mon[29357]: scrub summary: idle+waiting paths [/] 2024-04-25T00:35:05.756 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:05 smithi032 ceph-mon[29357]: scrub summary: active paths [/] 2024-04-25T00:35:05.756 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:05 smithi032 ceph-mon[29357]: pgmap v56: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 2.1 KiB/s rd, 860 MiB/s wr, 1.73k op/s 2024-04-25T00:35:07.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:07 smithi130 ceph-mon[32525]: pgmap v57: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 836 MiB/s wr, 1.60k op/s 2024-04-25T00:35:07.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:07 smithi032 ceph-mon[29357]: pgmap v57: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 836 MiB/s wr, 1.60k op/s 2024-04-25T00:35:07.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:07 smithi121 ceph-mon[32231]: pgmap v57: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 836 MiB/s wr, 1.60k op/s 2024-04-25T00:35:09.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:09 smithi130 ceph-mon[32525]: pgmap v58: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 2.4 KiB/s rd, 651 MiB/s wr, 1.14k op/s 2024-04-25T00:35:09.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:09 smithi032 ceph-mon[29357]: pgmap v58: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 2.4 KiB/s rd, 651 MiB/s wr, 1.14k op/s 2024-04-25T00:35:09.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:09 smithi121 ceph-mon[32231]: pgmap v58: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 2.4 KiB/s rd, 651 MiB/s wr, 1.14k op/s 2024-04-25T00:35:11.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:11 smithi130 ceph-mon[32525]: pgmap v59: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 2.8 KiB/s rd, 840 MiB/s wr, 1.93k op/s 2024-04-25T00:35:11.751 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:11 smithi032 ceph-mon[29357]: pgmap v59: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 2.8 KiB/s rd, 840 MiB/s wr, 1.93k op/s 2024-04-25T00:35:11.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:11 smithi121 ceph-mon[32231]: pgmap v59: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 2.8 KiB/s rd, 840 MiB/s wr, 1.93k op/s 2024-04-25T00:35:13.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:13 smithi130 ceph-mon[32525]: pgmap v60: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 637 MiB/s wr, 1.68k op/s 2024-04-25T00:35:13.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:13 smithi032 ceph-mon[29357]: pgmap v60: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 637 MiB/s wr, 1.68k op/s 2024-04-25T00:35:13.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:13 smithi121 ceph-mon[32231]: pgmap v60: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 637 MiB/s wr, 1.68k op/s 2024-04-25T00:35:15.753 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:15 smithi032 ceph-mon[29357]: pgmap v61: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 784 MiB/s wr, 1.74k op/s 2024-04-25T00:35:15.753 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:15 smithi032 ceph-mon[29357]: mgrmap e26: x(active, since 116s), standbys: z, y 2024-04-25T00:35:15.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:15 smithi121 ceph-mon[32231]: pgmap v61: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 784 MiB/s wr, 1.74k op/s 2024-04-25T00:35:15.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:15 smithi121 ceph-mon[32231]: mgrmap e26: x(active, since 116s), standbys: z, y 2024-04-25T00:35:15.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:15 smithi130 ceph-mon[32525]: pgmap v61: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 784 MiB/s wr, 1.74k op/s 2024-04-25T00:35:15.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:15 smithi130 ceph-mon[32525]: mgrmap e26: x(active, since 116s), standbys: z, y 2024-04-25T00:35:17.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:17 smithi032 ceph-mon[29357]: pgmap v62: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 731 MiB/s wr, 1.57k op/s 2024-04-25T00:35:17.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:17 smithi121 ceph-mon[32231]: pgmap v62: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 731 MiB/s wr, 1.57k op/s 2024-04-25T00:35:17.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:17 smithi130 ceph-mon[32525]: pgmap v62: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 731 MiB/s wr, 1.57k op/s 2024-04-25T00:35:19.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:19 smithi032 ceph-mon[29357]: pgmap v63: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 627 MiB/s wr, 1.32k op/s 2024-04-25T00:35:19.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:19 smithi121 ceph-mon[32231]: pgmap v63: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 627 MiB/s wr, 1.32k op/s 2024-04-25T00:35:19.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:19 smithi130 ceph-mon[32525]: pgmap v63: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 627 MiB/s wr, 1.32k op/s 2024-04-25T00:35:21.764 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:21 smithi032 ceph-mon[29357]: pgmap v64: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 842 MiB/s wr, 1.84k op/s 2024-04-25T00:35:21.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:21 smithi121 ceph-mon[32231]: pgmap v64: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 842 MiB/s wr, 1.84k op/s 2024-04-25T00:35:21.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:21 smithi130 ceph-mon[32525]: pgmap v64: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 842 MiB/s wr, 1.84k op/s 2024-04-25T00:35:23.752 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:23 smithi032 ceph-mon[29357]: pgmap v65: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 753 MiB/s wr, 1.29k op/s 2024-04-25T00:35:23.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:23 smithi121 ceph-mon[32231]: pgmap v65: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 753 MiB/s wr, 1.29k op/s 2024-04-25T00:35:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:23 smithi130 ceph-mon[32525]: pgmap v65: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 753 MiB/s wr, 1.29k op/s 2024-04-25T00:35:25.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:25 smithi032 ceph-mon[29357]: pgmap v66: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 878 MiB/s wr, 1.23k op/s 2024-04-25T00:35:25.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:25 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:35:25.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:25 smithi121 ceph-mon[32231]: pgmap v66: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 878 MiB/s wr, 1.23k op/s 2024-04-25T00:35:25.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:25 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:35:25.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:25 smithi130 ceph-mon[32525]: pgmap v66: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 878 MiB/s wr, 1.23k op/s 2024-04-25T00:35:25.895 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:25 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:35:26.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:26 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:35:26.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:26 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:35:26.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:26 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:35:26.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:26 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:35:26.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:26 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:35:26.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:26 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:35:26.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:26 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:35:26.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:26 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:35:26.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:26 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:35:27.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:27 smithi032 ceph-mon[29357]: pgmap v67: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 398 KiB/s rd, 746 MiB/s wr, 1.18k op/s 2024-04-25T00:35:27.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:27 smithi121 ceph-mon[32231]: pgmap v67: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 398 KiB/s rd, 746 MiB/s wr, 1.18k op/s 2024-04-25T00:35:27.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:27 smithi130 ceph-mon[32525]: pgmap v67: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 398 KiB/s rd, 746 MiB/s wr, 1.18k op/s 2024-04-25T00:35:29.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:29 smithi032 ceph-mon[29357]: pgmap v68: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 297 KiB/s rd, 689 MiB/s wr, 1.23k op/s 2024-04-25T00:35:29.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:29 smithi121 ceph-mon[32231]: pgmap v68: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 297 KiB/s rd, 689 MiB/s wr, 1.23k op/s 2024-04-25T00:35:29.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:29 smithi130 ceph-mon[32525]: pgmap v68: 129 pgs: 129 active+clean; 10 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 297 KiB/s rd, 689 MiB/s wr, 1.23k op/s 2024-04-25T00:35:31.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:31 smithi121 ceph-mon[32231]: pgmap v69: 129 pgs: 129 active+clean; 10 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 1.8 MiB/s rd, 835 MiB/s wr, 1.83k op/s 2024-04-25T00:35:31.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:31 smithi130 ceph-mon[32525]: pgmap v69: 129 pgs: 129 active+clean; 10 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 1.8 MiB/s rd, 835 MiB/s wr, 1.83k op/s 2024-04-25T00:35:31.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:31 smithi032 ceph-mon[29357]: pgmap v69: 129 pgs: 129 active+clean; 10 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 1.8 MiB/s rd, 835 MiB/s wr, 1.83k op/s 2024-04-25T00:35:33.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:33 smithi121 ceph-mon[32231]: pgmap v70: 129 pgs: 129 active+clean; 10 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 1.8 MiB/s rd, 686 MiB/s wr, 1.60k op/s 2024-04-25T00:35:33.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:33 smithi130 ceph-mon[32525]: pgmap v70: 129 pgs: 129 active+clean; 10 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 1.8 MiB/s rd, 686 MiB/s wr, 1.60k op/s 2024-04-25T00:35:33.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:33 smithi032 ceph-mon[29357]: pgmap v70: 129 pgs: 129 active+clean; 10 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 1.8 MiB/s rd, 686 MiB/s wr, 1.60k op/s 2024-04-25T00:35:34.273 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-25T00:35:34.274 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-25T00:35:34.615 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.612+0000 7f7f028d2640 1 -- 172.21.15.32:0/3441739580 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7efc109060 msgr2=0x7f7efc114350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:34.615 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.612+0000 7f7f028d2640 1 --2- 172.21.15.32:0/3441739580 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7efc109060 0x7f7efc114350 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f7ef8009900 tx=0x7f7ef802a410 comp rx=0 tx=0).stop 2024-04-25T00:35:34.615 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.613+0000 7f7f028d2640 1 -- 172.21.15.32:0/3441739580 shutdown_connections 2024-04-25T00:35:34.615 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.613+0000 7f7f028d2640 1 --2- 172.21.15.32:0/3441739580 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7efc114890 0x7f7efc116c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.615 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.613+0000 7f7f028d2640 1 --2- 172.21.15.32:0/3441739580 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7efc109060 0x7f7efc114350 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.615 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.613+0000 7f7f028d2640 1 --2- 172.21.15.32:0/3441739580 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7efc108740 0x7f7efc108b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.615 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.613+0000 7f7f028d2640 1 -- 172.21.15.32:0/3441739580 >> 172.21.15.32:0/3441739580 conn(0x7f7efc06c6c0 msgr2=0x7f7efc06cad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:34.615 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.614+0000 7f7f028d2640 1 -- 172.21.15.32:0/3441739580 shutdown_connections 2024-04-25T00:35:34.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.615+0000 7f7f028d2640 1 -- 172.21.15.32:0/3441739580 wait complete. 2024-04-25T00:35:34.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.616+0000 7f7f028d2640 1 Processor -- start 2024-04-25T00:35:34.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.616+0000 7f7f028d2640 1 -- start start 2024-04-25T00:35:34.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.616+0000 7f7f028d2640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7efc108740 0x7f7efc113470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:34.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.616+0000 7f7f028d2640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7efc114890 0x7f7efc1139b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:34.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.616+0000 7f7f028d2640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7efc1a7000 0x7f7efc1a74b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:34.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.616+0000 7f7f028d2640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f7efc11a280 con 0x7f7efc108740 2024-04-25T00:35:34.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.616+0000 7f7f028d2640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f7efc11a100 con 0x7f7efc1a7000 2024-04-25T00:35:34.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.616+0000 7f7f028d2640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f7efc11a400 con 0x7f7efc114890 2024-04-25T00:35:34.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.616+0000 7f7f020d1640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7efc1a7000 0x7f7efc1a74b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:34.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.616+0000 7f7f020d1640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7efc1a7000 0x7f7efc1a74b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:32858/0 (socket says 172.21.15.32:32858) 2024-04-25T00:35:34.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.616+0000 7f7f020d1640 1 -- 172.21.15.32:0/1505912433 learned_addr learned my addr 172.21.15.32:0/1505912433 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:34.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.617+0000 7f7f020d1640 1 -- 172.21.15.32:0/1505912433 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7efc114890 msgr2=0x7f7efc1139b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:34.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.617+0000 7f7f020d1640 1 --2- 172.21.15.32:0/1505912433 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7efc114890 0x7f7efc1139b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.617+0000 7f7f020d1640 1 -- 172.21.15.32:0/1505912433 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7efc108740 msgr2=0x7f7efc113470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:34.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.617+0000 7f7f020d1640 1 --2- 172.21.15.32:0/1505912433 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7efc108740 0x7f7efc113470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.617+0000 7f7f020d1640 1 -- 172.21.15.32:0/1505912433 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7efc1a7af0 con 0x7f7efc1a7000 2024-04-25T00:35:34.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.618+0000 7f7f020d1640 1 --2- 172.21.15.32:0/1505912433 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7efc1a7000 0x7f7efc1a74b0 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f7eec00d6a0 tx=0x7f7eec00db60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:34.628 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.626+0000 7f7ef2ffd640 1 -- 172.21.15.32:0/1505912433 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7eec017070 con 0x7f7efc1a7000 2024-04-25T00:35:34.628 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.627+0000 7f7f028d2640 1 -- 172.21.15.32:0/1505912433 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7efc1a7de0 con 0x7f7efc1a7000 2024-04-25T00:35:34.628 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.627+0000 7f7f028d2640 1 -- 172.21.15.32:0/1505912433 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7efc079cc0 con 0x7f7efc1a7000 2024-04-25T00:35:34.629 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.629+0000 7f7ef2ffd640 1 -- 172.21.15.32:0/1505912433 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7eec009820 con 0x7f7efc1a7000 2024-04-25T00:35:34.629 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.629+0000 7f7ef2ffd640 1 -- 172.21.15.32:0/1505912433 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7eec004d00 con 0x7f7efc1a7000 2024-04-25T00:35:34.639 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.636+0000 7f7f06841740 1 Processor -- start 2024-04-25T00:35:34.639 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.637+0000 7f7f06841740 1 -- start start 2024-04-25T00:35:34.640 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.637+0000 7f7f06841740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56370d398820 0x56370d398c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:34.640 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.638+0000 7f7f06841740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56370d399140 0x56370d3a32d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:34.640 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.638+0000 7f7f06841740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56370d3a3810 0x56370d3a5c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:34.640 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.638+0000 7f7f06841740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x56370d2fd6b0 con 0x56370d399140 2024-04-25T00:35:34.640 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.638+0000 7f7f06841740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x56370d2a5880 con 0x56370d398820 2024-04-25T00:35:34.641 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.638+0000 7f7f06841740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x56370d2f9670 con 0x56370d3a3810 2024-04-25T00:35:34.641 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.638+0000 7f7ef2ffd640 1 -- 172.21.15.32:0/1505912433 <== mon.2 v2:172.21.15.121:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f7eec010460 con 0x7f7efc1a7000 2024-04-25T00:35:34.641 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.638+0000 7f7f010cf640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56370d398820 0x56370d398c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:34.641 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.638+0000 7f7f010cf640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56370d398820 0x56370d398c00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:32872/0 (socket says 172.21.15.32:32872) 2024-04-25T00:35:34.642 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.638+0000 7f7f010cf640 1 -- 172.21.15.32:0/2588020566 learned_addr learned my addr 172.21.15.32:0/2588020566 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:34.644 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.639+0000 7f7f018d0640 1 --2- 172.21.15.32:0/2588020566 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56370d399140 0x56370d3a32d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:34.644 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.639+0000 7f7f010cf640 1 --2- 172.21.15.32:0/2588020566 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56370d3a3810 0x56370d3a5c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:34.645 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.640+0000 7f7f010cf640 1 -- 172.21.15.32:0/2588020566 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56370d398820 msgr2=0x56370d398c00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:34.645 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.640+0000 7f7f010cf640 1 --2- 172.21.15.32:0/2588020566 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56370d398820 0x56370d398c00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.645 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.640+0000 7f7f010cf640 1 -- 172.21.15.32:0/2588020566 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56370d399140 msgr2=0x56370d3a32d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:34.645 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.640+0000 7f7f010cf640 1 --2- 172.21.15.32:0/2588020566 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56370d399140 0x56370d3a32d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.645 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.640+0000 7f7f010cf640 1 -- 172.21.15.32:0/2588020566 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56370d0db7d0 con 0x56370d3a3810 2024-04-25T00:35:34.645 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.640+0000 7f7ef2ffd640 1 --2- 172.21.15.32:0/1505912433 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7ee80a2f10 0x7f7ee80a53d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:34.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.641+0000 7f7ef2ffd640 1 -- 172.21.15.32:0/1505912433 <== mon.2 v2:172.21.15.121:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f7eec0d01e0 con 0x7f7efc1a7000 2024-04-25T00:35:34.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.641+0000 7f7f018d0640 1 --2- 172.21.15.32:0/1505912433 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7ee80a2f10 0x7f7ee80a53d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:34.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.641+0000 7f7f010cf640 1 --2- 172.21.15.32:0/2588020566 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56370d3a3810 0x56370d3a5c00 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f7ef802aa00 tx=0x7f7ef8005fd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c9ce38005c47d123 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:34.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.642+0000 7f7f018d0640 1 --2- 172.21.15.32:0/1505912433 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7ee80a2f10 0x7f7ee80a53d0 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x56370d399de0 tx=0x7f7ef400d040 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:34.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.643+0000 7f7ee7fff640 1 -- 172.21.15.32:0/2588020566 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7ef8035030 con 0x56370d3a3810 2024-04-25T00:35:34.647 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.643+0000 7f7ee7fff640 1 -- 172.21.15.32:0/2588020566 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7ef8003cd0 con 0x56370d3a3810 2024-04-25T00:35:34.647 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.643+0000 7f7ee7fff640 1 -- 172.21.15.32:0/2588020566 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7ef8031c90 con 0x56370d3a3810 2024-04-25T00:35:34.650 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.646+0000 7f7f06841740 1 -- 172.21.15.32:0/2588020566 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56370d3a3810 msgr2=0x56370d3a5c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:34.650 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.646+0000 7f7f06841740 1 --2- 172.21.15.32:0/2588020566 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56370d3a3810 0x56370d3a5c00 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f7ef802aa00 tx=0x7f7ef8005fd0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.650 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.646+0000 7f7f06841740 1 -- 172.21.15.32:0/2588020566 shutdown_connections 2024-04-25T00:35:34.650 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.646+0000 7f7f06841740 1 --2- 172.21.15.32:0/2588020566 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56370d3a3810 0x56370d3a5c00 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.650 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.646+0000 7f7f06841740 1 --2- 172.21.15.32:0/2588020566 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56370d399140 0x56370d3a32d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.650 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.646+0000 7f7f06841740 1 --2- 172.21.15.32:0/2588020566 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56370d398820 0x56370d398c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.651 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.646+0000 7f7f06841740 1 -- 172.21.15.32:0/2588020566 >> 172.21.15.32:0/2588020566 conn(0x56370d394b40 msgr2=0x56370d394f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:34.651 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.648+0000 7f7f06841740 1 -- 172.21.15.32:0/2588020566 shutdown_connections 2024-04-25T00:35:34.651 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.648+0000 7f7f06841740 1 -- 172.21.15.32:0/2588020566 wait complete. 2024-04-25T00:35:34.651 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.649+0000 7f7f06841740 1 -- start start 2024-04-25T00:35:34.651 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.649+0000 7f7f06841740 1 Processor -- start 2024-04-25T00:35:34.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.650+0000 7f7f06841740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56370d398820 0x56370d3a0810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:34.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.650+0000 7f7f06841740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56370d399140 0x56370d3a0d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:34.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.652+0000 7f7f06841740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56370d3a3810 0x56370d3a1290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:34.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.652+0000 7f7f06841740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x56370d3a65a0 con 0x56370d398820 2024-04-25T00:35:34.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.653+0000 7f7f06841740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x56370d2b8f50 con 0x56370d3a3810 2024-04-25T00:35:34.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.653+0000 7f7f06841740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x56370d3a6720 con 0x56370d399140 2024-04-25T00:35:34.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.653+0000 7f7f010cf640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56370d399140 0x56370d3a0d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:34.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.653+0000 7f7f010cf640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56370d399140 0x56370d3a0d50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:54820/0 (socket says 172.21.15.32:54820) 2024-04-25T00:35:34.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.653+0000 7f7f010cf640 1 -- 172.21.15.32:0/136900016 learned_addr learned my addr 172.21.15.32:0/136900016 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:34.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.653+0000 7f7f020d1640 1 --2- 172.21.15.32:0/136900016 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56370d3a3810 0x56370d3a1290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:34.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.653+0000 7f7f06841740 10 client.0 _ll_register_callbacks cb 0x56370d2f4000 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:35:34.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.654+0000 7f7f020d1640 1 -- 172.21.15.32:0/136900016 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56370d399140 msgr2=0x56370d3a0d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:34.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.654+0000 7f7f020d1640 1 --2- 172.21.15.32:0/136900016 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56370d399140 0x56370d3a0d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.654+0000 7f7f020d1640 1 -- 172.21.15.32:0/136900016 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56370d398820 msgr2=0x56370d3a0810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:34.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.654+0000 7f7f020d1640 1 --2- 172.21.15.32:0/136900016 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56370d398820 0x56370d3a0810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.654+0000 7f7f020d1640 1 -- 172.21.15.32:0/136900016 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56370d0113f0 con 0x56370d3a3810 2024-04-25T00:35:34.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.654+0000 7f7f020d1640 1 --2- 172.21.15.32:0/136900016 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56370d3a3810 0x56370d3a1290 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f7eec0cd520 tx=0x7f7eec0a06a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5c9b67364abddd4 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:34.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.654+0000 7f7ee57fa640 10 client.0 ms_handle_connect on v2:172.21.15.121:3300/0 2024-04-25T00:35:34.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.655+0000 7f7ee57fa640 1 -- 172.21.15.32:0/136900016 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7eec020040 con 0x56370d3a3810 2024-04-25T00:35:34.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.655+0000 7f7ee57fa640 1 -- 172.21.15.32:0/136900016 <== mon.2 v2:172.21.15.121:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f7eec098960 con 0x56370d3a3810 2024-04-25T00:35:34.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.656+0000 7f7f06841740 1 -- 172.21.15.32:0/136900016 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f7ef8003cd0 con 0x56370d3a3810 2024-04-25T00:35:34.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.658+0000 7f7ee57fa640 1 -- 172.21.15.32:0/136900016 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7f7eec096e60 con 0x56370d3a3810 2024-04-25T00:35:34.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.659+0000 7f7f06841740 10 client.25595 fetch_fsmap learned FSMap version 26 2024-04-25T00:35:34.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.661+0000 7f7f06841740 1 -- 172.21.15.32:0/136900016 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x56370cfd10f0 con 0x56370d3a3810 2024-04-25T00:35:34.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.663+0000 7f7ee57fa640 1 -- 172.21.15.32:0/136900016 <== mon.2 v2:172.21.15.121:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7eec004780 con 0x56370d3a3810 2024-04-25T00:35:34.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.663+0000 7f7ee57fa640 1 -- 172.21.15.32:0/136900016 <== mon.2 v2:172.21.15.121:3300/0 5 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7f7eec0a0200 con 0x56370d3a3810 2024-04-25T00:35:34.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.663+0000 7f7f06841740 10 client.25595 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:35:34.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.663+0000 7f7f06841740 10 client.25595 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:35:34.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.663+0000 7f7f06841740 20 client.25595 populate_metadata read hostname 'smithi032' 2024-04-25T00:35:34.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.663+0000 7f7f06841740 1 --2- 172.21.15.32:0/136900016 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x56370d3b39b0 0x56370d3b5e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:34.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.664+0000 7f7f06841740 4 client.25595 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:35:34.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.664+0000 7f7f06841740 1 -- 172.21.15.32:0/136900016 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x7f7ef8003cd0 con 0x56370d3b39b0 2024-04-25T00:35:34.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.666+0000 7f7f018d0640 1 --2- 172.21.15.32:0/136900016 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x56370d3b39b0 0x56370d3b5e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:34.671 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.669+0000 7f7ee57fa640 10 client.25595 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:35:34.671 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.669+0000 7f7f018d0640 1 --2- 172.21.15.32:0/136900016 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x56370d3b39b0 0x56370d3b5e00 crc :-1 s=READY pgs=69 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7ea031269376f468 server_cookie=e1795d0c47b5c2e2 in_seq=0 out_seq=0 2024-04-25T00:35:34.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.678+0000 7f7ee57fa640 1 -- 172.21.15.32:0/136900016 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x7f7ef8003cd0 con 0x56370d3b39b0 2024-04-25T00:35:34.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.678+0000 7f7ee57fa640 10 client.25595 handle_command_reply: tid=0 2024-04-25T00:35:34.679 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.678+0000 7f7f06841740 1 client.25595 shutdown 2024-04-25T00:35:34.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.679+0000 7f7f06841740 1 -- 172.21.15.32:0/136900016 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56370d3a3810 msgr2=0x56370d3a1290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:34.680 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.679+0000 7f7f06841740 1 --2- 172.21.15.32:0/136900016 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56370d3a3810 0x56370d3a1290 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f7eec0cd520 tx=0x7f7eec0a06a0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.683 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.682+0000 7f7f06841740 1 -- 172.21.15.32:0/136900016 shutdown_connections 2024-04-25T00:35:34.683 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.682+0000 7f7f06841740 1 --2- 172.21.15.32:0/136900016 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x56370d3b39b0 0x56370d3b5e00 crc :-1 s=READY pgs=69 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.683 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.682+0000 7f7f06841740 1 --2- 172.21.15.32:0/136900016 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56370d3a3810 0x56370d3a1290 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.683 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.682+0000 7f7f06841740 1 --2- 172.21.15.32:0/136900016 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56370d399140 0x56370d3a0d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.683 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.682+0000 7f7f06841740 1 --2- 172.21.15.32:0/136900016 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56370d398820 0x56370d3a0810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.684 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.682+0000 7f7f06841740 1 -- 172.21.15.32:0/136900016 >> 172.21.15.32:0/136900016 conn(0x56370d394b40 msgr2=0x56370d393d00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:34.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.685+0000 7f7ee57fa640 0 client.25595 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:35:34.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.686+0000 7f7f06841740 1 -- 172.21.15.32:0/136900016 shutdown_connections 2024-04-25T00:35:34.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.686+0000 7f7f06841740 1 -- 172.21.15.32:0/136900016 wait complete. 2024-04-25T00:35:34.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.686+0000 7f7f06841740 20 client.25595 trim_cache size 0 max 16384 2024-04-25T00:35:34.744 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.744+0000 7f7f06841740 1 Processor -- start 2024-04-25T00:35:34.745 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.744+0000 7f7f06841740 1 -- start start 2024-04-25T00:35:34.745 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.745+0000 7f7f06841740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56370d398820 0x56370d3af4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:34.745 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.745+0000 7f7f06841740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56370d399140 0x56370d3a10b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:34.746 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.745+0000 7f7f06841740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56370d3a3810 0x56370d3bb6d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:34.746 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.745+0000 7f7f06841740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f7ee0012940 con 0x56370d399140 2024-04-25T00:35:34.746 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.745+0000 7f7f06841740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f7ee0010210 con 0x56370d3a3810 2024-04-25T00:35:34.746 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.745+0000 7f7f06841740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f7ee000e060 con 0x56370d398820 2024-04-25T00:35:34.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.746+0000 7f7f020d1640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56370d3a3810 0x56370d3bb6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:34.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.746+0000 7f7f018d0640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56370d398820 0x56370d3af4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:34.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.746+0000 7f7f010cf640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56370d399140 0x56370d3a10b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:34.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.746+0000 7f7f010cf640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56370d399140 0x56370d3a10b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39668/0 (socket says 172.21.15.32:39668) 2024-04-25T00:35:34.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.746+0000 7f7f010cf640 1 -- 172.21.15.32:0/2452709769 learned_addr learned my addr 172.21.15.32:0/2452709769 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:34.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.746+0000 7f7f010cf640 1 -- 172.21.15.32:0/2452709769 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56370d398820 msgr2=0x56370d3af4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:34.748 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.746+0000 7f7f010cf640 1 --2- 172.21.15.32:0/2452709769 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56370d398820 0x56370d3af4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.748 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.746+0000 7f7f010cf640 1 -- 172.21.15.32:0/2452709769 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56370d3a3810 msgr2=0x56370d3bb6d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:34.748 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.746+0000 7f7f010cf640 1 --2- 172.21.15.32:0/2452709769 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56370d3a3810 0x56370d3bb6d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.748 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.746+0000 7f7f010cf640 1 -- 172.21.15.32:0/2452709769 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56370d39ed60 con 0x56370d399140 2024-04-25T00:35:34.748 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.747+0000 7f7f010cf640 1 --2- 172.21.15.32:0/2452709769 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56370d399140 0x56370d3a10b0 secure :-1 s=READY pgs=597 cs=0 l=1 rev1=1 crypto rx=0x7f7ef8009900 tx=0x7f7ef8004010 comp rx=0 tx=0).ready entity=mon.0 client_cookie=59fe2c301416b8c8 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:34.748 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.748+0000 7f7ee77fe640 1 -- 172.21.15.32:0/2452709769 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7ef804c020 con 0x56370d399140 2024-04-25T00:35:34.748 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.748+0000 7f7ee77fe640 1 -- 172.21.15.32:0/2452709769 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7ef8003a00 con 0x56370d399140 2024-04-25T00:35:34.749 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.748+0000 7f7ee77fe640 1 -- 172.21.15.32:0/2452709769 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7ef8035040 con 0x56370d399140 2024-04-25T00:35:34.750 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.749+0000 7f7f06841740 1 -- 172.21.15.32:0/2452709769 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56370d399140 msgr2=0x56370d3a10b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:34.750 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.749+0000 7f7f06841740 1 --2- 172.21.15.32:0/2452709769 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56370d399140 0x56370d3a10b0 secure :-1 s=READY pgs=597 cs=0 l=1 rev1=1 crypto rx=0x7f7ef8009900 tx=0x7f7ef8004010 comp rx=0 tx=0).stop 2024-04-25T00:35:34.753 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.752+0000 7f7f06841740 1 -- 172.21.15.32:0/2452709769 shutdown_connections 2024-04-25T00:35:34.753 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.752+0000 7f7f06841740 1 --2- 172.21.15.32:0/2452709769 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56370d3a3810 0x56370d3bb6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.753 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.752+0000 7f7f06841740 1 --2- 172.21.15.32:0/2452709769 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56370d399140 0x56370d3a10b0 unknown :-1 s=CLOSED pgs=597 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.753 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.752+0000 7f7f06841740 1 --2- 172.21.15.32:0/2452709769 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56370d398820 0x56370d3af4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.753 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.752+0000 7f7f06841740 1 -- 172.21.15.32:0/2452709769 >> 172.21.15.32:0/2452709769 conn(0x56370d394b40 msgr2=0x56370d39baf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:34.753 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.752+0000 7f7f06841740 1 -- 172.21.15.32:0/2452709769 shutdown_connections 2024-04-25T00:35:34.753 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.752+0000 7f7f06841740 1 -- 172.21.15.32:0/2452709769 wait complete. 2024-04-25T00:35:34.754 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.753+0000 7f7f06841740 1 -- start start 2024-04-25T00:35:34.754 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.753+0000 7f7f06841740 1 Processor -- start 2024-04-25T00:35:34.755 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.754+0000 7f7f06841740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56370d398820 0x56370d3ab6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:34.755 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.754+0000 7f7f06841740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56370d399140 0x56370d3abbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:34.755 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.754+0000 7f7f06841740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56370d3a3810 0x56370d3a6b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:34.755 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.754+0000 7f7f06841740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f7ee000e6d0 con 0x56370d398820 2024-04-25T00:35:34.755 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.754+0000 7f7f06841740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f7ee000fb40 con 0x56370d399140 2024-04-25T00:35:34.755 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.755+0000 7f7f06841740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f7ee000eda0 con 0x56370d3a3810 2024-04-25T00:35:34.755 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.755+0000 7f7f010cf640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56370d399140 0x56370d3abbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:34.755 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.755+0000 7f7f010cf640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56370d399140 0x56370d3abbe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:32902/0 (socket says 172.21.15.32:32902) 2024-04-25T00:35:34.755 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.755+0000 7f7f010cf640 1 -- 172.21.15.32:0/1980131723 learned_addr learned my addr 172.21.15.32:0/1980131723 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:35:34.755 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.755+0000 7f7f06841740 10 client.0 _ll_register_callbacks cb 0x7f7eec09c0c0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:35:34.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.755+0000 7f7f020d1640 1 --2- 172.21.15.32:0/1980131723 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56370d3a3810 0x56370d3a6b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:34.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.755+0000 7f7f010cf640 1 -- 172.21.15.32:0/1980131723 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56370d3a3810 msgr2=0x56370d3a6b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:34.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.755+0000 7f7f010cf640 1 --2- 172.21.15.32:0/1980131723 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56370d3a3810 0x56370d3a6b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.755+0000 7f7f010cf640 1 -- 172.21.15.32:0/1980131723 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56370d398820 msgr2=0x56370d3ab6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:35:34.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.755+0000 7f7f010cf640 1 --2- 172.21.15.32:0/1980131723 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56370d398820 0x56370d3ab6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.755+0000 7f7f010cf640 1 -- 172.21.15.32:0/1980131723 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56370d3a7250 con 0x56370d399140 2024-04-25T00:35:34.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.756+0000 7f7f010cf640 1 --2- 172.21.15.32:0/1980131723 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56370d399140 0x56370d3abbe0 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f7ef802aa00 tx=0x7f7ef8032ca0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=551586741c079767 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:35:34.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.758+0000 7f7ee57fa640 10 client.0 ms_handle_connect on v2:172.21.15.121:3300/0 2024-04-25T00:35:34.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.758+0000 7f7ee57fa640 1 -- 172.21.15.32:0/1980131723 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7ef8035070 con 0x56370d399140 2024-04-25T00:35:34.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.758+0000 7f7f06841740 1 -- 172.21.15.32:0/1980131723 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f7ef8003a00 con 0x56370d399140 2024-04-25T00:35:34.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.758+0000 7f7ee57fa640 1 -- 172.21.15.32:0/1980131723 <== mon.2 v2:172.21.15.121:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f7ef803d680 con 0x56370d399140 2024-04-25T00:35:34.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.759+0000 7f7ee57fa640 1 -- 172.21.15.32:0/1980131723 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7f7ef8062050 con 0x56370d399140 2024-04-25T00:35:34.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.759+0000 7f7f06841740 10 client.25598 fetch_fsmap learned FSMap version 26 2024-04-25T00:35:34.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.759+0000 7f7f06841740 1 -- 172.21.15.32:0/1980131723 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x56370d422120 con 0x56370d399140 2024-04-25T00:35:34.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.760+0000 7f7ee57fa640 1 -- 172.21.15.32:0/1980131723 <== mon.2 v2:172.21.15.121:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7ef8003a00 con 0x56370d399140 2024-04-25T00:35:34.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.761+0000 7f7ee57fa640 1 -- 172.21.15.32:0/1980131723 <== mon.2 v2:172.21.15.121:3300/0 5 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7f7ef803a050 con 0x56370d399140 2024-04-25T00:35:34.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.761+0000 7f7f06841740 10 client.25598 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:35:34.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.761+0000 7f7f06841740 10 client.25598 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:35:34.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.761+0000 7f7f06841740 20 client.25598 populate_metadata read hostname 'smithi032' 2024-04-25T00:35:34.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.761+0000 7f7f06841740 1 --2- 172.21.15.32:0/1980131723 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x56370d4226d0 0x56370d424af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:35:34.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.761+0000 7f7f06841740 4 client.25598 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-25T00:35:34.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.761+0000 7f7f06841740 1 -- 172.21.15.32:0/1980131723 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "scrub status"}) -- 0x56370d30abf0 con 0x56370d4226d0 2024-04-25T00:35:34.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.768+0000 7f7f018d0640 1 --2- 172.21.15.32:0/1980131723 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x56370d4226d0 0x56370d424af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:35:34.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.778+0000 7f7f018d0640 1 --2- 172.21.15.32:0/1980131723 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x56370d4226d0 0x56370d424af0 crc :-1 s=READY pgs=70 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b29e7112b5acb04 server_cookie=2c48b79f02fc21af in_seq=0 out_seq=0 2024-04-25T00:35:34.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.778+0000 7f7ee57fa640 10 client.25598 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:35:34.781 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.779+0000 7f7ee57fa640 1 -- 172.21.15.32:0/1980131723 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+265 (crc 0 0 0) 0x56370d30abf0 con 0x56370d4226d0 2024-04-25T00:35:34.781 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.779+0000 7f7ee57fa640 10 client.25598 handle_command_reply: tid=0 2024-04-25T00:35:34.781 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.779+0000 7f7f06841740 1 client.25598 shutdown 2024-04-25T00:35:34.786 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.781+0000 7f7f06841740 1 -- 172.21.15.32:0/1980131723 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56370d399140 msgr2=0x56370d3abbe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:34.786 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.781+0000 7f7f06841740 1 --2- 172.21.15.32:0/1980131723 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56370d399140 0x56370d3abbe0 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f7ef802aa00 tx=0x7f7ef8032ca0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.787+0000 7f7f06841740 1 -- 172.21.15.32:0/1980131723 shutdown_connections 2024-04-25T00:35:34.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.787+0000 7f7f06841740 1 --2- 172.21.15.32:0/1980131723 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x56370d4226d0 0x56370d424af0 crc :-1 s=READY pgs=70 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.787+0000 7f7f06841740 1 --2- 172.21.15.32:0/1980131723 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56370d3a3810 0x56370d3a6b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.787+0000 7f7f06841740 1 --2- 172.21.15.32:0/1980131723 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56370d399140 0x56370d3abbe0 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.787+0000 7f7f06841740 1 --2- 172.21.15.32:0/1980131723 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56370d398820 0x56370d3ab6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.787+0000 7f7f06841740 1 -- 172.21.15.32:0/1980131723 >> 172.21.15.32:0/1980131723 conn(0x56370d394b40 msgr2=0x56370d393470 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:34.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.787+0000 7f7ee57fa640 0 client.25598 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:35:34.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.791+0000 7f7f06841740 1 -- 172.21.15.32:0/1980131723 shutdown_connections 2024-04-25T00:35:34.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.791+0000 7f7f06841740 1 -- 172.21.15.32:0/1980131723 wait complete. 2024-04-25T00:35:34.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.793+0000 7f7f06841740 20 client.25598 trim_cache size 0 max 16384 2024-04-25T00:35:34.795 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:35:34.795 INFO:teuthology.orchestra.run.smithi032.stdout: "status": "scrub active (1 inodes in the stack)", 2024-04-25T00:35:34.795 INFO:teuthology.orchestra.run.smithi032.stdout: "scrubs": { 2024-04-25T00:35:34.795 INFO:teuthology.orchestra.run.smithi032.stdout: "dcf3868e-a5ff-4ef9-b62a-fbe3ab2421c6": { 2024-04-25T00:35:34.795 INFO:teuthology.orchestra.run.smithi032.stdout: "path": "/", 2024-04-25T00:35:34.795 INFO:teuthology.orchestra.run.smithi032.stdout: "tag": "dcf3868e-a5ff-4ef9-b62a-fbe3ab2421c6", 2024-04-25T00:35:34.795 INFO:teuthology.orchestra.run.smithi032.stdout: "options": "recursive,force" 2024-04-25T00:35:34.795 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:35:34.795 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:35:34.795 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:35:34.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.796+0000 7f7f028d2640 1 -- 172.21.15.32:0/1505912433 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7ee80a2f10 msgr2=0x7f7ee80a53d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:34.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.796+0000 7f7f028d2640 1 --2- 172.21.15.32:0/1505912433 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7ee80a2f10 0x7f7ee80a53d0 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x56370d399de0 tx=0x7f7ef400d040 comp rx=0 tx=0).stop 2024-04-25T00:35:34.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.796+0000 7f7f028d2640 1 -- 172.21.15.32:0/1505912433 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7efc1a7000 msgr2=0x7f7efc1a74b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:35:34.797 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.796+0000 7f7f028d2640 1 --2- 172.21.15.32:0/1505912433 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7efc1a7000 0x7f7efc1a74b0 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f7eec00d6a0 tx=0x7f7eec00db60 comp rx=0 tx=0).stop 2024-04-25T00:35:34.799 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.799+0000 7f7f028d2640 1 -- 172.21.15.32:0/1505912433 shutdown_connections 2024-04-25T00:35:34.800 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.799+0000 7f7f028d2640 1 --2- 172.21.15.32:0/1505912433 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7ee80a2f10 0x7f7ee80a53d0 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.800 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.799+0000 7f7f028d2640 1 --2- 172.21.15.32:0/1505912433 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7efc1a7000 0x7f7efc1a74b0 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.800 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.799+0000 7f7f028d2640 1 --2- 172.21.15.32:0/1505912433 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7efc114890 0x7f7efc1139b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.800 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.799+0000 7f7f028d2640 1 --2- 172.21.15.32:0/1505912433 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7efc108740 0x7f7efc113470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:35:34.800 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.799+0000 7f7f028d2640 1 -- 172.21.15.32:0/1505912433 >> 172.21.15.32:0/1505912433 conn(0x7f7efc06c6c0 msgr2=0x7f7efc0700c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:35:34.802 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.801+0000 7f7f028d2640 1 -- 172.21.15.32:0/1505912433 shutdown_connections 2024-04-25T00:35:34.802 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:35:34.801+0000 7f7f028d2640 1 -- 172.21.15.32:0/1505912433 wait complete. 2024-04-25T00:35:34.828 INFO:tasks.cephfs.filesystem:scrub status for tag:dcf3868e-a5ff-4ef9-b62a-fbe3ab2421c6 - {'path': '/', 'tag': 'dcf3868e-a5ff-4ef9-b62a-fbe3ab2421c6', 'options': 'recursive,force'} 2024-04-25T00:35:35.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:35 smithi121 ceph-mon[32231]: pgmap v71: 129 pgs: 129 active+clean; 11 GiB data, 24 GiB used, 1.0 TiB / 1.0 TiB avail; 1.8 MiB/s rd, 698 MiB/s wr, 2.01k op/s 2024-04-25T00:35:35.896 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:35 smithi130 ceph-mon[32525]: pgmap v71: 129 pgs: 129 active+clean; 11 GiB data, 24 GiB used, 1.0 TiB / 1.0 TiB avail; 1.8 MiB/s rd, 698 MiB/s wr, 2.01k op/s 2024-04-25T00:35:36.000 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:35 smithi032 ceph-mon[29357]: pgmap v71: 129 pgs: 129 active+clean; 11 GiB data, 24 GiB used, 1.0 TiB / 1.0 TiB avail; 1.8 MiB/s rd, 698 MiB/s wr, 2.01k op/s 2024-04-25T00:35:37.799 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:37 smithi032 ceph-mon[29357]: pgmap v72: 129 pgs: 129 active+clean; 12 GiB data, 25 GiB used, 1.0 TiB / 1.0 TiB avail; 2.0 MiB/s rd, 579 MiB/s wr, 2.44k op/s 2024-04-25T00:35:37.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:37 smithi121 ceph-mon[32231]: pgmap v72: 129 pgs: 129 active+clean; 12 GiB data, 25 GiB used, 1.0 TiB / 1.0 TiB avail; 2.0 MiB/s rd, 579 MiB/s wr, 2.44k op/s 2024-04-25T00:35:37.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:37 smithi130 ceph-mon[32525]: pgmap v72: 129 pgs: 129 active+clean; 12 GiB data, 25 GiB used, 1.0 TiB / 1.0 TiB avail; 2.0 MiB/s rd, 579 MiB/s wr, 2.44k op/s 2024-04-25T00:35:39.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:39 smithi121 ceph-mon[32231]: pgmap v73: 129 pgs: 129 active+clean; 12 GiB data, 25 GiB used, 1.0 TiB / 1.0 TiB avail; 2.2 MiB/s rd, 603 MiB/s wr, 2.85k op/s 2024-04-25T00:35:39.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:39 smithi130 ceph-mon[32525]: pgmap v73: 129 pgs: 129 active+clean; 12 GiB data, 25 GiB used, 1.0 TiB / 1.0 TiB avail; 2.2 MiB/s rd, 603 MiB/s wr, 2.85k op/s 2024-04-25T00:35:39.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:39 smithi032 ceph-mon[29357]: pgmap v73: 129 pgs: 129 active+clean; 12 GiB data, 25 GiB used, 1.0 TiB / 1.0 TiB avail; 2.2 MiB/s rd, 603 MiB/s wr, 2.85k op/s 2024-04-25T00:35:41.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:41 smithi121 ceph-mon[32231]: pgmap v74: 129 pgs: 129 active+clean; 13 GiB data, 27 GiB used, 1.0 TiB / 1.0 TiB avail; 2.6 MiB/s rd, 712 MiB/s wr, 3.33k op/s 2024-04-25T00:35:41.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:41 smithi130 ceph-mon[32525]: pgmap v74: 129 pgs: 129 active+clean; 13 GiB data, 27 GiB used, 1.0 TiB / 1.0 TiB avail; 2.6 MiB/s rd, 712 MiB/s wr, 3.33k op/s 2024-04-25T00:35:41.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:41 smithi032 ceph-mon[29357]: pgmap v74: 129 pgs: 129 active+clean; 13 GiB data, 27 GiB used, 1.0 TiB / 1.0 TiB avail; 2.6 MiB/s rd, 712 MiB/s wr, 3.33k op/s 2024-04-25T00:35:43.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:43 smithi121 ceph-mon[32231]: pgmap v75: 129 pgs: 129 active+clean; 14 GiB data, 29 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 588 MiB/s wr, 2.87k op/s 2024-04-25T00:35:43.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:43 smithi130 ceph-mon[32525]: pgmap v75: 129 pgs: 129 active+clean; 14 GiB data, 29 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 588 MiB/s wr, 2.87k op/s 2024-04-25T00:35:43.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:43 smithi032 ceph-mon[29357]: pgmap v75: 129 pgs: 129 active+clean; 14 GiB data, 29 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 588 MiB/s wr, 2.87k op/s 2024-04-25T00:35:45.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:45 smithi130 ceph-mon[32525]: pgmap v76: 129 pgs: 129 active+clean; 15 GiB data, 31 GiB used, 1.0 TiB / 1.0 TiB avail; 1.7 MiB/s rd, 660 MiB/s wr, 2.95k op/s 2024-04-25T00:35:45.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:45 smithi032 ceph-mon[29357]: pgmap v76: 129 pgs: 129 active+clean; 15 GiB data, 31 GiB used, 1.0 TiB / 1.0 TiB avail; 1.7 MiB/s rd, 660 MiB/s wr, 2.95k op/s 2024-04-25T00:35:46.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:45 smithi121 ceph-mon[32231]: pgmap v76: 129 pgs: 129 active+clean; 15 GiB data, 31 GiB used, 1.0 TiB / 1.0 TiB avail; 1.7 MiB/s rd, 660 MiB/s wr, 2.95k op/s 2024-04-25T00:35:47.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:47 smithi130 ceph-mon[32525]: pgmap v77: 129 pgs: 129 active+clean; 15 GiB data, 32 GiB used, 1.0 TiB / 1.0 TiB avail; 1.7 MiB/s rd, 645 MiB/s wr, 2.59k op/s 2024-04-25T00:35:47.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:47 smithi032 ceph-mon[29357]: pgmap v77: 129 pgs: 129 active+clean; 15 GiB data, 32 GiB used, 1.0 TiB / 1.0 TiB avail; 1.7 MiB/s rd, 645 MiB/s wr, 2.59k op/s 2024-04-25T00:35:48.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:47 smithi121 ceph-mon[32231]: pgmap v77: 129 pgs: 129 active+clean; 15 GiB data, 32 GiB used, 1.0 TiB / 1.0 TiB avail; 1.7 MiB/s rd, 645 MiB/s wr, 2.59k op/s 2024-04-25T00:35:49.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:49 smithi130 ceph-mon[32525]: pgmap v78: 129 pgs: 129 active+clean; 16 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 616 MiB/s wr, 1.98k op/s 2024-04-25T00:35:49.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:49 smithi130 ceph-mon[32525]: scrub summary: active paths [/] 2024-04-25T00:35:49.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:49 smithi130 ceph-mon[32525]: scrub summary: idle+waiting paths [/] 2024-04-25T00:35:49.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:49 smithi032 ceph-mon[29357]: pgmap v78: 129 pgs: 129 active+clean; 16 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 616 MiB/s wr, 1.98k op/s 2024-04-25T00:35:49.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:49 smithi032 ceph-mon[29357]: scrub summary: active paths [/] 2024-04-25T00:35:49.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:49 smithi032 ceph-mon[29357]: scrub summary: idle+waiting paths [/] 2024-04-25T00:35:50.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:49 smithi121 ceph-mon[32231]: pgmap v78: 129 pgs: 129 active+clean; 16 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 616 MiB/s wr, 1.98k op/s 2024-04-25T00:35:50.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:49 smithi121 ceph-mon[32231]: scrub summary: active paths [/] 2024-04-25T00:35:50.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:49 smithi121 ceph-mon[32231]: scrub summary: idle+waiting paths [/] 2024-04-25T00:35:51.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:51 smithi130 ceph-mon[32525]: pgmap v79: 129 pgs: 129 active+clean; 18 GiB data, 37 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 812 MiB/s wr, 1.87k op/s 2024-04-25T00:35:51.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:51 smithi032 ceph-mon[29357]: pgmap v79: 129 pgs: 129 active+clean; 18 GiB data, 37 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 812 MiB/s wr, 1.87k op/s 2024-04-25T00:35:52.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:51 smithi121 ceph-mon[32231]: pgmap v79: 129 pgs: 129 active+clean; 18 GiB data, 37 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 812 MiB/s wr, 1.87k op/s 2024-04-25T00:35:53.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:53 smithi130 ceph-mon[32525]: pgmap v80: 129 pgs: 129 active+clean; 18 GiB data, 38 GiB used, 1.0 TiB / 1.0 TiB avail; 756 KiB/s rd, 678 MiB/s wr, 1.33k op/s 2024-04-25T00:35:53.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:53 smithi032 ceph-mon[29357]: pgmap v80: 129 pgs: 129 active+clean; 18 GiB data, 38 GiB used, 1.0 TiB / 1.0 TiB avail; 756 KiB/s rd, 678 MiB/s wr, 1.33k op/s 2024-04-25T00:35:54.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:53 smithi121 ceph-mon[32231]: pgmap v80: 129 pgs: 129 active+clean; 18 GiB data, 38 GiB used, 1.0 TiB / 1.0 TiB avail; 756 KiB/s rd, 678 MiB/s wr, 1.33k op/s 2024-04-25T00:35:55.002 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:54 smithi032 ceph-mon[29357]: scrub summary: idle+waiting paths [/] 2024-04-25T00:35:55.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:54 smithi121 ceph-mon[32231]: scrub summary: idle+waiting paths [/] 2024-04-25T00:35:55.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:54 smithi130 ceph-mon[32525]: scrub summary: idle+waiting paths [/] 2024-04-25T00:35:55.987 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:55 smithi121 ceph-mon[32231]: pgmap v81: 129 pgs: 129 active+clean; 18 GiB data, 38 GiB used, 1.0 TiB / 1.0 TiB avail; 1.7 MiB/s rd, 768 MiB/s wr, 1.41k op/s 2024-04-25T00:35:55.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:55 smithi032 ceph-mon[29357]: pgmap v81: 129 pgs: 129 active+clean; 18 GiB data, 38 GiB used, 1.0 TiB / 1.0 TiB avail; 1.7 MiB/s rd, 768 MiB/s wr, 1.41k op/s 2024-04-25T00:35:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:55 smithi130 ceph-mon[32525]: pgmap v81: 129 pgs: 129 active+clean; 18 GiB data, 38 GiB used, 1.0 TiB / 1.0 TiB avail; 1.7 MiB/s rd, 768 MiB/s wr, 1.41k op/s 2024-04-25T00:35:57.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:57 smithi032 ceph-mon[29357]: pgmap v82: 129 pgs: 129 active+clean; 18 GiB data, 38 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 713 MiB/s wr, 1.18k op/s 2024-04-25T00:35:57.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:57 smithi032 ceph-mon[29357]: scrub summary: idle 2024-04-25T00:35:57.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:57 smithi032 ceph-mon[29357]: scrub summary: idle 2024-04-25T00:35:58.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:57 smithi121 ceph-mon[32231]: pgmap v82: 129 pgs: 129 active+clean; 18 GiB data, 38 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 713 MiB/s wr, 1.18k op/s 2024-04-25T00:35:58.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:57 smithi121 ceph-mon[32231]: scrub summary: idle 2024-04-25T00:35:58.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:57 smithi121 ceph-mon[32231]: scrub summary: idle 2024-04-25T00:35:58.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:57 smithi130 ceph-mon[32525]: pgmap v82: 129 pgs: 129 active+clean; 18 GiB data, 38 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 713 MiB/s wr, 1.18k op/s 2024-04-25T00:35:58.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:57 smithi130 ceph-mon[32525]: scrub summary: idle 2024-04-25T00:35:58.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:57 smithi130 ceph-mon[32525]: scrub summary: idle 2024-04-25T00:35:59.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:35:59 smithi032 ceph-mon[29357]: pgmap v83: 129 pgs: 129 active+clean; 18 GiB data, 38 GiB used, 1.0 TiB / 1.0 TiB avail; 1.6 MiB/s rd, 670 MiB/s wr, 1.07k op/s 2024-04-25T00:36:00.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:35:59 smithi121 ceph-mon[32231]: pgmap v83: 129 pgs: 129 active+clean; 18 GiB data, 38 GiB used, 1.0 TiB / 1.0 TiB avail; 1.6 MiB/s rd, 670 MiB/s wr, 1.07k op/s 2024-04-25T00:36:00.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:35:59 smithi130 ceph-mon[32525]: pgmap v83: 129 pgs: 129 active+clean; 18 GiB data, 38 GiB used, 1.0 TiB / 1.0 TiB avail; 1.6 MiB/s rd, 670 MiB/s wr, 1.07k op/s 2024-04-25T00:36:01.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:01 smithi032 ceph-mon[29357]: pgmap v84: 129 pgs: 129 active+clean; 18 GiB data, 38 GiB used, 1.0 TiB / 1.0 TiB avail; 2.0 MiB/s rd, 786 MiB/s wr, 1.82k op/s 2024-04-25T00:36:02.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:01 smithi121 ceph-mon[32231]: pgmap v84: 129 pgs: 129 active+clean; 18 GiB data, 38 GiB used, 1.0 TiB / 1.0 TiB avail; 2.0 MiB/s rd, 786 MiB/s wr, 1.82k op/s 2024-04-25T00:36:02.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:01 smithi130 ceph-mon[32525]: pgmap v84: 129 pgs: 129 active+clean; 18 GiB data, 38 GiB used, 1.0 TiB / 1.0 TiB avail; 2.0 MiB/s rd, 786 MiB/s wr, 1.82k op/s 2024-04-25T00:36:03.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:03 smithi032 ceph-mon[29357]: pgmap v85: 129 pgs: 129 active+clean; 18 GiB data, 38 GiB used, 1.0 TiB / 1.0 TiB avail; 1.8 MiB/s rd, 536 MiB/s wr, 1.55k op/s 2024-04-25T00:36:04.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:03 smithi121 ceph-mon[32231]: pgmap v85: 129 pgs: 129 active+clean; 18 GiB data, 38 GiB used, 1.0 TiB / 1.0 TiB avail; 1.8 MiB/s rd, 536 MiB/s wr, 1.55k op/s 2024-04-25T00:36:04.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:03 smithi130 ceph-mon[32525]: pgmap v85: 129 pgs: 129 active+clean; 18 GiB data, 38 GiB used, 1.0 TiB / 1.0 TiB avail; 1.8 MiB/s rd, 536 MiB/s wr, 1.55k op/s 2024-04-25T00:36:04.828 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-25T00:36:04.829 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-25T00:36:05.145 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.139+0000 7f992bab3640 1 -- 172.21.15.32:0/3473001647 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9924108740 msgr2=0x7f9924108b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:05.146 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.139+0000 7f992bab3640 1 --2- 172.21.15.32:0/3473001647 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9924108740 0x7f9924108b20 secure :-1 s=READY pgs=598 cs=0 l=1 rev1=1 crypto rx=0x7f991c009ca0 tx=0x7f991c02a8e0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.148 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.143+0000 7f992bab3640 1 -- 172.21.15.32:0/3473001647 shutdown_connections 2024-04-25T00:36:05.149 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.143+0000 7f992bab3640 1 --2- 172.21.15.32:0/3473001647 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9924114890 0x7f9924116c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.150 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.143+0000 7f992bab3640 1 --2- 172.21.15.32:0/3473001647 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9924109060 0x7f9924114350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.143+0000 7f992bab3640 1 --2- 172.21.15.32:0/3473001647 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9924108740 0x7f9924108b20 unknown :-1 s=CLOSED pgs=598 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.143+0000 7f992bab3640 1 -- 172.21.15.32:0/3473001647 >> 172.21.15.32:0/3473001647 conn(0x7f992406c720 msgr2=0x7f992406cb30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:05.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.148+0000 7f992bab3640 1 -- 172.21.15.32:0/3473001647 shutdown_connections 2024-04-25T00:36:05.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.148+0000 7f992bab3640 1 -- 172.21.15.32:0/3473001647 wait complete. 2024-04-25T00:36:05.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.149+0000 7f992bab3640 1 Processor -- start 2024-04-25T00:36:05.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.149+0000 7f992bab3640 1 -- start start 2024-04-25T00:36:05.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.150+0000 7f992bab3640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9924108740 0x7f99241142e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:05.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.150+0000 7f992bab3640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9924109060 0x7f99241113d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:05.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.150+0000 7f992bab3640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9924114890 0x7f9924111b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:05.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.150+0000 7f992bab3640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f992410bce0 con 0x7f9924108740 2024-04-25T00:36:05.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.150+0000 7f992bab3640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f992410bb60 con 0x7f9924114890 2024-04-25T00:36:05.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.150+0000 7f992bab3640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f992410be60 con 0x7f9924109060 2024-04-25T00:36:05.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.150+0000 7f992904e640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9924109060 0x7f99241113d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:05.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.150+0000 7f992904e640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9924109060 0x7f99241113d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:57998/0 (socket says 172.21.15.32:57998) 2024-04-25T00:36:05.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.150+0000 7f992904e640 1 -- 172.21.15.32:0/806661221 learned_addr learned my addr 172.21.15.32:0/806661221 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:05.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.151+0000 7f992904e640 1 -- 172.21.15.32:0/806661221 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9924114890 msgr2=0x7f9924111b40 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-04-25T00:36:05.153 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.151+0000 7f992904e640 1 --2- 172.21.15.32:0/806661221 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9924114890 0x7f9924111b40 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.151+0000 7f992904e640 1 -- 172.21.15.32:0/806661221 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9924108740 msgr2=0x7f99241142e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:05.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.151+0000 7f992904e640 1 --2- 172.21.15.32:0/806661221 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9924108740 0x7f99241142e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.151+0000 7f992904e640 1 -- 172.21.15.32:0/806661221 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f992411ca10 con 0x7f9924109060 2024-04-25T00:36:05.154 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.151+0000 7f992904e640 1 --2- 172.21.15.32:0/806661221 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9924109060 0x7f99241113d0 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f991400d980 tx=0x7f991400de40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:05.156 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.155+0000 7f991affd640 1 -- 172.21.15.32:0/806661221 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9914017070 con 0x7f9924109060 2024-04-25T00:36:05.156 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.155+0000 7f991affd640 1 -- 172.21.15.32:0/806661221 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9914009c30 con 0x7f9924109060 2024-04-25T00:36:05.156 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.155+0000 7f991affd640 1 -- 172.21.15.32:0/806661221 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f991400f620 con 0x7f9924109060 2024-04-25T00:36:05.156 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.156+0000 7f992bab3640 1 -- 172.21.15.32:0/806661221 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f992411cd00 con 0x7f9924109060 2024-04-25T00:36:05.157 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.156+0000 7f992bab3640 1 -- 172.21.15.32:0/806661221 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f992411d200 con 0x7f9924109060 2024-04-25T00:36:05.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.160+0000 7f992e7c2740 1 Processor -- start 2024-04-25T00:36:05.169 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.163+0000 7f992e7c2740 1 -- start start 2024-04-25T00:36:05.169 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.163+0000 7f992e7c2740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556ccbb8b820 0x556ccbb8bc00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:05.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.163+0000 7f992e7c2740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556ccbb8c140 0x556ccbb962d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:05.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.163+0000 7f992e7c2740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556ccbb96810 0x556ccbb98c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:05.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.163+0000 7f992e7c2740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x556ccbaf06b0 con 0x556ccbb96810 2024-04-25T00:36:05.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.163+0000 7f992e7c2740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x556ccba98880 con 0x556ccbb8c140 2024-04-25T00:36:05.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.163+0000 7f992e7c2740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x556ccbaec670 con 0x556ccbb8b820 2024-04-25T00:36:05.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.164+0000 7f992904e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556ccbb96810 0x556ccbb98c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:05.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.164+0000 7f992904e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556ccbb96810 0x556ccbb98c00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:58100/0 (socket says 172.21.15.32:58100) 2024-04-25T00:36:05.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.164+0000 7f992904e640 1 -- 172.21.15.32:0/2850002356 learned_addr learned my addr 172.21.15.32:0/2850002356 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:05.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.165+0000 7f991affd640 1 -- 172.21.15.32:0/806661221 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f991400f7c0 con 0x7f9924109060 2024-04-25T00:36:05.170 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.165+0000 7f992984f640 1 --2- 172.21.15.32:0/2850002356 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556ccbb8c140 0x556ccbb962d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:05.171 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.165+0000 7f992904e640 1 -- 172.21.15.32:0/2850002356 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556ccbb8b820 msgr2=0x556ccbb8bc00 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-25T00:36:05.171 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.165+0000 7f992904e640 1 --2- 172.21.15.32:0/2850002356 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556ccbb8b820 0x556ccbb8bc00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.171 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.165+0000 7f992904e640 1 -- 172.21.15.32:0/2850002356 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556ccbb8c140 msgr2=0x556ccbb962d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:05.171 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.165+0000 7f992904e640 1 --2- 172.21.15.32:0/2850002356 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556ccbb8c140 0x556ccbb962d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.171 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.166+0000 7f992904e640 1 -- 172.21.15.32:0/2850002356 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x556ccb8ce7d0 con 0x556ccbb96810 2024-04-25T00:36:05.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.166+0000 7f992904e640 1 --2- 172.21.15.32:0/2850002356 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556ccbb96810 0x556ccbb98c00 secure :-1 s=READY pgs=599 cs=0 l=1 rev1=1 crypto rx=0x7f99140cc6b0 tx=0x7f99140d5b20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6a1f171595ce61d0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:05.178 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.176+0000 7f991affd640 1 --2- 172.21.15.32:0/806661221 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f99100a2f10 0x7f99100a53d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:05.181 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.177+0000 7f990ffff640 1 -- 172.21.15.32:0/2850002356 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f99140e2070 con 0x556ccbb96810 2024-04-25T00:36:05.181 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.177+0000 7f990ffff640 1 -- 172.21.15.32:0/2850002356 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f991409bc40 con 0x556ccbb96810 2024-04-25T00:36:05.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.178+0000 7f992e7c2740 1 -- 172.21.15.32:0/2850002356 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556ccbb96810 msgr2=0x556ccbb98c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:05.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.178+0000 7f992e7c2740 1 --2- 172.21.15.32:0/2850002356 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556ccbb96810 0x556ccbb98c00 secure :-1 s=READY pgs=599 cs=0 l=1 rev1=1 crypto rx=0x7f99140cc6b0 tx=0x7f99140d5b20 comp rx=0 tx=0).stop 2024-04-25T00:36:05.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.179+0000 7f992e7c2740 1 -- 172.21.15.32:0/2850002356 shutdown_connections 2024-04-25T00:36:05.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.180+0000 7f992e7c2740 1 --2- 172.21.15.32:0/2850002356 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556ccbb96810 0x556ccbb98c00 unknown :-1 s=CLOSED pgs=599 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.180+0000 7f992e7c2740 1 --2- 172.21.15.32:0/2850002356 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556ccbb8c140 0x556ccbb962d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.180+0000 7f992e7c2740 1 --2- 172.21.15.32:0/2850002356 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556ccbb8b820 0x556ccbb8bc00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.180+0000 7f992e7c2740 1 -- 172.21.15.32:0/2850002356 >> 172.21.15.32:0/2850002356 conn(0x556ccbb87b40 msgr2=0x556ccbb87f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:05.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.181+0000 7f991affd640 1 -- 172.21.15.32:0/806661221 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f99140cf430 con 0x7f9924109060 2024-04-25T00:36:05.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.183+0000 7f992a050640 1 --2- 172.21.15.32:0/806661221 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f99100a2f10 0x7f99100a53d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:05.197 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.184+0000 7f992a050640 1 --2- 172.21.15.32:0/806661221 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f99100a2f10 0x7f99100a53d0 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f992406ac90 tx=0x7f9920005b80 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:05.197 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.185+0000 7f992e7c2740 1 -- 172.21.15.32:0/2850002356 shutdown_connections 2024-04-25T00:36:05.198 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.185+0000 7f992e7c2740 1 -- 172.21.15.32:0/2850002356 wait complete. 2024-04-25T00:36:05.198 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.185+0000 7f992e7c2740 1 -- start start 2024-04-25T00:36:05.198 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.185+0000 7f992e7c2740 1 Processor -- start 2024-04-25T00:36:05.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.186+0000 7f992e7c2740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556ccbb8b820 0x556ccbb938f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:05.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.186+0000 7f992e7c2740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556ccbb8c140 0x556ccbb93e30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:05.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.186+0000 7f992e7c2740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556ccbb96810 0x556ccbb94370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:05.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.186+0000 7f992e7c2740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x556ccbb996a0 con 0x556ccbb8b820 2024-04-25T00:36:05.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.186+0000 7f992e7c2740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x556ccbaabf50 con 0x556ccbb96810 2024-04-25T00:36:05.199 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.186+0000 7f992e7c2740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x556ccbb99820 con 0x556ccbb8c140 2024-04-25T00:36:05.200 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.186+0000 7f992e7c2740 10 client.0 _ll_register_callbacks cb 0x556ccbae7000 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:36:05.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.187+0000 7f992984f640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556ccbb8b820 0x556ccbb938f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:05.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.187+0000 7f992984f640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556ccbb8b820 0x556ccbb938f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:58112/0 (socket says 172.21.15.32:58112) 2024-04-25T00:36:05.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.187+0000 7f992984f640 1 -- 172.21.15.32:0/274671655 learned_addr learned my addr 172.21.15.32:0/274671655 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:05.201 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.187+0000 7f992904e640 1 --2- 172.21.15.32:0/274671655 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556ccbb8c140 0x556ccbb93e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:05.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.188+0000 7f992904e640 1 -- 172.21.15.32:0/274671655 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556ccbb96810 msgr2=0x556ccbb94370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:05.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.188+0000 7f992904e640 1 --2- 172.21.15.32:0/274671655 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556ccbb96810 0x556ccbb94370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.188+0000 7f992904e640 1 -- 172.21.15.32:0/274671655 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556ccbb8b820 msgr2=0x556ccbb938f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:05.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.188+0000 7f992904e640 1 --2- 172.21.15.32:0/274671655 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556ccbb8b820 0x556ccbb938f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.188+0000 7f992904e640 1 -- 172.21.15.32:0/274671655 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x556ccb8043f0 con 0x556ccbb8c140 2024-04-25T00:36:05.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.188+0000 7f992904e640 1 --2- 172.21.15.32:0/274671655 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556ccbb8c140 0x556ccbb93e30 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f991409bdf0 tx=0x7f9914005bb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a3c0f6a013fddf69 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:05.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.188+0000 7f990d7fa640 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:36:05.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.189+0000 7f990d7fa640 1 -- 172.21.15.32:0/274671655 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9914020070 con 0x556ccbb8c140 2024-04-25T00:36:05.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.189+0000 7f992e7c2740 1 -- 172.21.15.32:0/274671655 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x556ccbafdbf0 con 0x556ccbb8c140 2024-04-25T00:36:05.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.191+0000 7f990d7fa640 1 -- 172.21.15.32:0/274671655 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f99140cf150 con 0x556ccbb8c140 2024-04-25T00:36:05.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.192+0000 7f990d7fa640 1 -- 172.21.15.32:0/274671655 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7f991409aea0 con 0x556ccbb8c140 2024-04-25T00:36:05.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.193+0000 7f992e7c2740 10 client.25579 fetch_fsmap learned FSMap version 26 2024-04-25T00:36:05.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.193+0000 7f992e7c2740 1 -- 172.21.15.32:0/274671655 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x556ccb7c40f0 con 0x556ccbb8c140 2024-04-25T00:36:05.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.194+0000 7f990d7fa640 1 -- 172.21.15.32:0/274671655 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x556ccbafdbf0 con 0x556ccbb8c140 2024-04-25T00:36:05.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.194+0000 7f990d7fa640 1 -- 172.21.15.32:0/274671655 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7f99140cc9f0 con 0x556ccbb8c140 2024-04-25T00:36:05.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.194+0000 7f992e7c2740 10 client.25579 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:36:05.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.194+0000 7f992e7c2740 10 client.25579 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:36:05.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.194+0000 7f992e7c2740 20 client.25579 populate_metadata read hostname 'smithi032' 2024-04-25T00:36:05.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.194+0000 7f992e7c2740 1 --2- 172.21.15.32:0/274671655 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x556ccbba6da0 0x556ccbba91c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:05.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.194+0000 7f992e7c2740 4 client.25579 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:36:05.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.194+0000 7f992e7c2740 1 -- 172.21.15.32:0/274671655 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x556ccbaf7f40 con 0x556ccbba6da0 2024-04-25T00:36:05.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.198+0000 7f992984f640 1 --2- 172.21.15.32:0/274671655 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x556ccbba6da0 0x556ccbba91c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:05.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.216+0000 7f992984f640 1 --2- 172.21.15.32:0/274671655 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x556ccbba6da0 0x556ccbba91c0 crc :-1 s=READY pgs=71 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=61ae6ab57d076568 server_cookie=c9d93c15028d7c9b in_seq=0 out_seq=0 2024-04-25T00:36:05.218 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.216+0000 7f990d7fa640 10 client.25579 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:05.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.223+0000 7f990d7fa640 1 -- 172.21.15.32:0/274671655 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x556ccbaf7f40 con 0x556ccbba6da0 2024-04-25T00:36:05.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.223+0000 7f990d7fa640 10 client.25579 handle_command_reply: tid=0 2024-04-25T00:36:05.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.223+0000 7f992e7c2740 1 client.25579 shutdown 2024-04-25T00:36:05.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.224+0000 7f992e7c2740 1 -- 172.21.15.32:0/274671655 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556ccbb8c140 msgr2=0x556ccbb93e30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:05.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.224+0000 7f992e7c2740 1 --2- 172.21.15.32:0/274671655 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556ccbb8c140 0x556ccbb93e30 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f991409bdf0 tx=0x7f9914005bb0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.224+0000 7f992e7c2740 1 -- 172.21.15.32:0/274671655 shutdown_connections 2024-04-25T00:36:05.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.224+0000 7f992e7c2740 1 --2- 172.21.15.32:0/274671655 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x556ccbba6da0 0x556ccbba91c0 crc :-1 s=READY pgs=71 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.224+0000 7f992e7c2740 1 --2- 172.21.15.32:0/274671655 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556ccbb96810 0x556ccbb94370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.224+0000 7f992e7c2740 1 --2- 172.21.15.32:0/274671655 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556ccbb8c140 0x556ccbb93e30 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.224+0000 7f992e7c2740 1 --2- 172.21.15.32:0/274671655 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556ccbb8b820 0x556ccbb938f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.224+0000 7f990d7fa640 0 client.25579 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:05.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.225+0000 7f992e7c2740 1 -- 172.21.15.32:0/274671655 >> 172.21.15.32:0/274671655 conn(0x556ccbb87b40 msgr2=0x556ccbb8d940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:05.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.225+0000 7f992e7c2740 1 -- 172.21.15.32:0/274671655 shutdown_connections 2024-04-25T00:36:05.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.229+0000 7f992e7c2740 1 -- 172.21.15.32:0/274671655 wait complete. 2024-04-25T00:36:05.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.231+0000 7f992e7c2740 20 client.25579 trim_cache size 0 max 16384 2024-04-25T00:36:05.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.291+0000 7f992e7c2740 1 Processor -- start 2024-04-25T00:36:05.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.293+0000 7f992e7c2740 1 -- start start 2024-04-25T00:36:05.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.293+0000 7f992e7c2740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556ccbb8b820 0x556ccbb96020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:05.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.293+0000 7f992e7c2740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556ccbb8c140 0x556ccbba25e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:05.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.293+0000 7f992e7c2740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556ccbb96810 0x556ccbbaea90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:05.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.293+0000 7f992e7c2740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f99241c8970 con 0x556ccbb8b820 2024-04-25T00:36:05.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.293+0000 7f992e7c2740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f99241c6240 con 0x556ccbb96810 2024-04-25T00:36:05.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.293+0000 7f992e7c2740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f9924068480 con 0x556ccbb8c140 2024-04-25T00:36:05.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.294+0000 7f992a050640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556ccbb96810 0x556ccbbaea90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:05.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.294+0000 7f992a050640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556ccbb96810 0x556ccbbaea90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:54800/0 (socket says 172.21.15.32:54800) 2024-04-25T00:36:05.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.294+0000 7f992a050640 1 -- 172.21.15.32:0/3475729540 learned_addr learned my addr 172.21.15.32:0/3475729540 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:05.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.294+0000 7f992a050640 1 -- 172.21.15.32:0/3475729540 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556ccbb8c140 msgr2=0x556ccbba25e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:05.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.294+0000 7f992a050640 1 --2- 172.21.15.32:0/3475729540 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556ccbb8c140 0x556ccbba25e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.294+0000 7f992a050640 1 -- 172.21.15.32:0/3475729540 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556ccbb8b820 msgr2=0x556ccbb96020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:05.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.294+0000 7f992a050640 1 --2- 172.21.15.32:0/3475729540 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556ccbb8b820 0x556ccbb96020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.294+0000 7f992a050640 1 -- 172.21.15.32:0/3475729540 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x556ccbb8f2e0 con 0x556ccbb96810 2024-04-25T00:36:05.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.296+0000 7f992a050640 1 --2- 172.21.15.32:0/3475729540 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556ccbb96810 0x556ccbbaea90 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f99200041d0 tx=0x7f99200046c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6d6a6c82164b9c0e server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:05.297 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.296+0000 7f990f7fe640 1 -- 172.21.15.32:0/3475729540 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9920016070 con 0x556ccbb96810 2024-04-25T00:36:05.297 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.297+0000 7f990f7fe640 1 -- 172.21.15.32:0/3475729540 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9920005500 con 0x556ccbb96810 2024-04-25T00:36:05.298 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.298+0000 7f992e7c2740 1 -- 172.21.15.32:0/3475729540 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556ccbb96810 msgr2=0x556ccbbaea90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:05.298 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.298+0000 7f992e7c2740 1 --2- 172.21.15.32:0/3475729540 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556ccbb96810 0x556ccbbaea90 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f99200041d0 tx=0x7f99200046c0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.299 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.298+0000 7f992e7c2740 1 -- 172.21.15.32:0/3475729540 shutdown_connections 2024-04-25T00:36:05.299 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.298+0000 7f992e7c2740 1 --2- 172.21.15.32:0/3475729540 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556ccbb96810 0x556ccbbaea90 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.299 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.298+0000 7f992e7c2740 1 --2- 172.21.15.32:0/3475729540 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556ccbb8c140 0x556ccbba25e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.299 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.298+0000 7f992e7c2740 1 --2- 172.21.15.32:0/3475729540 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556ccbb8b820 0x556ccbb96020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.299 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.298+0000 7f992e7c2740 1 -- 172.21.15.32:0/3475729540 >> 172.21.15.32:0/3475729540 conn(0x556ccbb87b40 msgr2=0x556ccbb8ebd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:05.300 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.300+0000 7f992e7c2740 1 -- 172.21.15.32:0/3475729540 shutdown_connections 2024-04-25T00:36:05.304 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.304+0000 7f992e7c2740 1 -- 172.21.15.32:0/3475729540 wait complete. 2024-04-25T00:36:05.305 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.304+0000 7f992e7c2740 1 -- start start 2024-04-25T00:36:05.305 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.304+0000 7f992e7c2740 1 Processor -- start 2024-04-25T00:36:05.306 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.305+0000 7f992e7c2740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556ccbb8b820 0x556ccbb99ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:05.306 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.305+0000 7f992e7c2740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556ccbb8c140 0x556ccbb99ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:05.306 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.305+0000 7f992e7c2740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556ccbb9a530 0x556ccbc0b2f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:05.306 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.305+0000 7f992e7c2740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f99241c48b0 con 0x556ccbb8c140 2024-04-25T00:36:05.306 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.305+0000 7f992e7c2740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f99241c5b70 con 0x556ccbb8b820 2024-04-25T00:36:05.306 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.305+0000 7f992e7c2740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f99241c4dd0 con 0x556ccbb9a530 2024-04-25T00:36:05.306 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.305+0000 7f992e7c2740 10 client.0 _ll_register_callbacks cb 0x7f99140e0070 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:36:05.307 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.306+0000 7f992a050640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556ccbb9a530 0x556ccbc0b2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:05.307 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.306+0000 7f992904e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556ccbb8c140 0x556ccbb99ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:05.307 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.306+0000 7f992904e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556ccbb8c140 0x556ccbb99ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:58132/0 (socket says 172.21.15.32:58132) 2024-04-25T00:36:05.307 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.306+0000 7f992904e640 1 -- 172.21.15.32:0/1341023809 learned_addr learned my addr 172.21.15.32:0/1341023809 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:05.307 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.306+0000 7f992984f640 1 --2- 172.21.15.32:0/1341023809 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556ccbb8b820 0x556ccbb99ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:05.307 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.307+0000 7f992904e640 1 -- 172.21.15.32:0/1341023809 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556ccbb9a530 msgr2=0x556ccbc0b2f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:05.307 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.307+0000 7f992904e640 1 --2- 172.21.15.32:0/1341023809 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556ccbb9a530 0x556ccbc0b2f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.308 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.307+0000 7f992904e640 1 -- 172.21.15.32:0/1341023809 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556ccbb8b820 msgr2=0x556ccbb99ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:05.308 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.307+0000 7f992904e640 1 --2- 172.21.15.32:0/1341023809 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556ccbb8b820 0x556ccbb99ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.308 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.307+0000 7f992904e640 1 -- 172.21.15.32:0/1341023809 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x556ccbc0b9b0 con 0x556ccbb8c140 2024-04-25T00:36:05.308 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.308+0000 7f992904e640 1 --2- 172.21.15.32:0/1341023809 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556ccbb8c140 0x556ccbb99ff0 secure :-1 s=READY pgs=600 cs=0 l=1 rev1=1 crypto rx=0x7f991409b240 tx=0x7f9914005bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a9ff246bb3664523 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:05.308 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.308+0000 7f990d7fa640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:36:05.309 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.308+0000 7f990d7fa640 1 -- 172.21.15.32:0/1341023809 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9914020070 con 0x556ccbb8c140 2024-04-25T00:36:05.309 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.308+0000 7f992e7c2740 1 -- 172.21.15.32:0/1341023809 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x556ccba756f0 con 0x556ccbb8c140 2024-04-25T00:36:05.309 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.309+0000 7f990d7fa640 1 -- 172.21.15.32:0/1341023809 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f99140d1d20 con 0x556ccbb8c140 2024-04-25T00:36:05.310 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.309+0000 7f990d7fa640 1 -- 172.21.15.32:0/1341023809 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7f991409b310 con 0x556ccbb8c140 2024-04-25T00:36:05.310 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.309+0000 7f992e7c2740 10 client.16053 fetch_fsmap learned FSMap version 26 2024-04-25T00:36:05.310 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.309+0000 7f992e7c2740 1 -- 172.21.15.32:0/1341023809 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x556ccbc0cd50 con 0x556ccbb8c140 2024-04-25T00:36:05.311 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.310+0000 7f990d7fa640 1 -- 172.21.15.32:0/1341023809 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x556ccba756f0 con 0x556ccbb8c140 2024-04-25T00:36:05.311 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.310+0000 7f990d7fa640 1 -- 172.21.15.32:0/1341023809 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7f99140a3d20 con 0x556ccbb8c140 2024-04-25T00:36:05.311 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.310+0000 7f992e7c2740 10 client.16053 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:36:05.311 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.310+0000 7f992e7c2740 10 client.16053 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:36:05.311 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.310+0000 7f992e7c2740 20 client.16053 populate_metadata read hostname 'smithi032' 2024-04-25T00:36:05.311 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.310+0000 7f992e7c2740 1 --2- 172.21.15.32:0/1341023809 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x556ccbc0d370 0x556ccbc0f790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:05.311 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.310+0000 7f992e7c2740 4 client.16053 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-25T00:36:05.311 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.310+0000 7f992e7c2740 1 -- 172.21.15.32:0/1341023809 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "scrub status"}) -- 0x556ccbac1d50 con 0x556ccbc0d370 2024-04-25T00:36:05.312 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.311+0000 7f992984f640 1 --2- 172.21.15.32:0/1341023809 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x556ccbc0d370 0x556ccbc0f790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:05.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.312+0000 7f990d7fa640 10 client.16053 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:05.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.313+0000 7f992984f640 1 --2- 172.21.15.32:0/1341023809 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x556ccbc0d370 0x556ccbc0f790 crc :-1 s=READY pgs=72 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=55b653ec93a7a661 server_cookie=1ecae9ac3c3b329a in_seq=0 out_seq=0 2024-04-25T00:36:05.318 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.318+0000 7f990d7fa640 1 -- 172.21.15.32:0/1341023809 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+63 (crc 0 0 0) 0x556ccbac1d50 con 0x556ccbc0d370 2024-04-25T00:36:05.318 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.318+0000 7f990d7fa640 10 client.16053 handle_command_reply: tid=0 2024-04-25T00:36:05.320 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.320+0000 7f992e7c2740 1 client.16053 shutdown 2024-04-25T00:36:05.324 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.324+0000 7f992e7c2740 1 -- 172.21.15.32:0/1341023809 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556ccbb8c140 msgr2=0x556ccbb99ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:05.324 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.324+0000 7f992e7c2740 1 --2- 172.21.15.32:0/1341023809 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556ccbb8c140 0x556ccbb99ff0 secure :-1 s=READY pgs=600 cs=0 l=1 rev1=1 crypto rx=0x7f991409b240 tx=0x7f9914005bb0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.329+0000 7f992e7c2740 1 -- 172.21.15.32:0/1341023809 shutdown_connections 2024-04-25T00:36:05.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.329+0000 7f992e7c2740 1 --2- 172.21.15.32:0/1341023809 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x556ccbc0d370 0x556ccbc0f790 crc :-1 s=READY pgs=72 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.330 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.329+0000 7f992e7c2740 1 --2- 172.21.15.32:0/1341023809 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556ccbb9a530 0x556ccbc0b2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.330 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.329+0000 7f992e7c2740 1 --2- 172.21.15.32:0/1341023809 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x556ccbb8c140 0x556ccbb99ff0 unknown :-1 s=CLOSED pgs=600 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.330 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.329+0000 7f992e7c2740 1 --2- 172.21.15.32:0/1341023809 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x556ccbb8b820 0x556ccbb99ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.330 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.329+0000 7f992e7c2740 1 -- 172.21.15.32:0/1341023809 >> 172.21.15.32:0/1341023809 conn(0x556ccbb87b40 msgr2=0x556ccbb8cfb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:05.330 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.330+0000 7f990d7fa640 0 client.16053 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:05.337 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.337+0000 7f992e7c2740 1 -- 172.21.15.32:0/1341023809 shutdown_connections 2024-04-25T00:36:05.343 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:36:05.343 INFO:teuthology.orchestra.run.smithi032.stdout: "status": "no active scrubs running", 2024-04-25T00:36:05.343 INFO:teuthology.orchestra.run.smithi032.stdout: "scrubs": {} 2024-04-25T00:36:05.343 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:36:05.343 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.339+0000 7f992e7c2740 1 -- 172.21.15.32:0/1341023809 wait complete. 2024-04-25T00:36:05.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.339+0000 7f992e7c2740 20 client.16053 trim_cache size 0 max 16384 2024-04-25T00:36:05.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.340+0000 7f992bab3640 1 -- 172.21.15.32:0/806661221 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f99100a2f10 msgr2=0x7f99100a53d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:05.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.340+0000 7f992bab3640 1 --2- 172.21.15.32:0/806661221 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f99100a2f10 0x7f99100a53d0 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f992406ac90 tx=0x7f9920005b80 comp rx=0 tx=0).stop 2024-04-25T00:36:05.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.340+0000 7f992bab3640 1 -- 172.21.15.32:0/806661221 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9924109060 msgr2=0x7f99241113d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:05.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.340+0000 7f992bab3640 1 --2- 172.21.15.32:0/806661221 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9924109060 0x7f99241113d0 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f991400d980 tx=0x7f991400de40 comp rx=0 tx=0).stop 2024-04-25T00:36:05.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.340+0000 7f992bab3640 1 -- 172.21.15.32:0/806661221 shutdown_connections 2024-04-25T00:36:05.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.340+0000 7f992bab3640 1 --2- 172.21.15.32:0/806661221 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f99100a2f10 0x7f99100a53d0 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.340+0000 7f992bab3640 1 --2- 172.21.15.32:0/806661221 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9924114890 0x7f9924111b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.340+0000 7f992bab3640 1 --2- 172.21.15.32:0/806661221 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9924109060 0x7f99241113d0 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.340+0000 7f992bab3640 1 --2- 172.21.15.32:0/806661221 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9924108740 0x7f99241142e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.340+0000 7f992bab3640 1 -- 172.21.15.32:0/806661221 >> 172.21.15.32:0/806661221 conn(0x7f992406c720 msgr2=0x7f9924071300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:05.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.341+0000 7f992bab3640 1 -- 172.21.15.32:0/806661221 shutdown_connections 2024-04-25T00:36:05.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.341+0000 7f992bab3640 1 -- 172.21.15.32:0/806661221 wait complete. 2024-04-25T00:36:05.365 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-25T00:36:05.366 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:36:05.633 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.631+0000 7f7db759e640 1 -- 172.21.15.32:0/1959368441 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7db810e340 msgr2=0x7f7db811cf30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:05.633 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.631+0000 7f7db759e640 1 --2- 172.21.15.32:0/1959368441 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7db810e340 0x7f7db811cf30 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f7dac009900 tx=0x7f7dac02a3f0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.634 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.633+0000 7f7db759e640 1 -- 172.21.15.32:0/1959368441 shutdown_connections 2024-04-25T00:36:05.634 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.633+0000 7f7db759e640 1 --2- 172.21.15.32:0/1959368441 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7db810e340 0x7f7db811cf30 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.634 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.633+0000 7f7db759e640 1 --2- 172.21.15.32:0/1959368441 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7db810d820 0x7f7db810dca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.634 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.633+0000 7f7db759e640 1 --2- 172.21.15.32:0/1959368441 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7db810fcc0 0x7f7db81100a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.634 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.633+0000 7f7db759e640 1 -- 172.21.15.32:0/1959368441 >> 172.21.15.32:0/1959368441 conn(0x7f7db806c650 msgr2=0x7f7db806ca60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:05.635 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.634+0000 7f7db759e640 1 -- 172.21.15.32:0/1959368441 shutdown_connections 2024-04-25T00:36:05.635 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.634+0000 7f7db759e640 1 -- 172.21.15.32:0/1959368441 wait complete. 2024-04-25T00:36:05.636 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.635+0000 7f7db759e640 1 Processor -- start 2024-04-25T00:36:05.637 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.636+0000 7f7db759e640 1 -- start start 2024-04-25T00:36:05.637 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.637+0000 7f7db759e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7db810d820 0x7f7db811c4c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:05.637 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.637+0000 7f7db759e640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7db810e340 0x7f7db811ca00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:05.637 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.637+0000 7f7db759e640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7db810fcc0 0x7f7db8115720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:05.637 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.637+0000 7f7db759e640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f7db81206d0 con 0x7f7db810d820 2024-04-25T00:36:05.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.637+0000 7f7db759e640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f7db8120550 con 0x7f7db810e340 2024-04-25T00:36:05.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.637+0000 7f7db759e640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f7db8120850 con 0x7f7db810fcc0 2024-04-25T00:36:05.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.637+0000 7f7db5d9b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7db810e340 0x7f7db811ca00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:05.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.638+0000 7f7db6d9d640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7db810fcc0 0x7f7db8115720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:05.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.638+0000 7f7db5d9b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7db810e340 0x7f7db811ca00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:54824/0 (socket says 172.21.15.32:54824) 2024-04-25T00:36:05.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.638+0000 7f7db5d9b640 1 -- 172.21.15.32:0/1823055582 learned_addr learned my addr 172.21.15.32:0/1823055582 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:05.638 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.638+0000 7f7db659c640 1 --2- 172.21.15.32:0/1823055582 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7db810d820 0x7f7db811c4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:05.639 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.638+0000 7f7db659c640 1 -- 172.21.15.32:0/1823055582 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7db810fcc0 msgr2=0x7f7db8115720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:05.639 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.638+0000 7f7db659c640 1 --2- 172.21.15.32:0/1823055582 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7db810fcc0 0x7f7db8115720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.639 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.638+0000 7f7db659c640 1 -- 172.21.15.32:0/1823055582 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7db810e340 msgr2=0x7f7db811ca00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:05.639 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.638+0000 7f7db659c640 1 --2- 172.21.15.32:0/1823055582 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7db810e340 0x7f7db811ca00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:05.639 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.638+0000 7f7db659c640 1 -- 172.21.15.32:0/1823055582 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7db8115fe0 con 0x7f7db810d820 2024-04-25T00:36:05.642 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.638+0000 7f7db6d9d640 1 --2- 172.21.15.32:0/1823055582 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7db810fcc0 0x7f7db8115720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:36:05.642 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.638+0000 7f7db659c640 1 --2- 172.21.15.32:0/1823055582 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7db810d820 0x7f7db811c4c0 secure :-1 s=READY pgs=601 cs=0 l=1 rev1=1 crypto rx=0x7f7db000cce0 tx=0x7f7db000a620 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:05.642 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.641+0000 7f7da77fe640 1 -- 172.21.15.32:0/1823055582 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7db000f450 con 0x7f7db810d820 2024-04-25T00:36:05.642 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.641+0000 7f7da77fe640 1 -- 172.21.15.32:0/1823055582 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7db000fad0 con 0x7f7db810d820 2024-04-25T00:36:05.642 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.641+0000 7f7da77fe640 1 -- 172.21.15.32:0/1823055582 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7db0018c10 con 0x7f7db810d820 2024-04-25T00:36:05.643 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.641+0000 7f7db759e640 1 -- 172.21.15.32:0/1823055582 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7db81162d0 con 0x7f7db810d820 2024-04-25T00:36:05.644 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.641+0000 7f7db759e640 1 -- 172.21.15.32:0/1823055582 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7db811a220 con 0x7f7db810d820 2024-04-25T00:36:05.647 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.644+0000 7f7db759e640 1 -- 172.21.15.32:0/1823055582 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7db806d8f0 con 0x7f7db810d820 2024-04-25T00:36:05.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.649+0000 7f7da77fe640 1 -- 172.21.15.32:0/1823055582 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f7db000fc80 con 0x7f7db810d820 2024-04-25T00:36:05.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.650+0000 7f7da77fe640 1 --2- 172.21.15.32:0/1823055582 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7da00a2f00 0x7f7da00a53c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:05.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.650+0000 7f7db5d9b640 1 --2- 172.21.15.32:0/1823055582 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7da00a2f00 0x7f7da00a53c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:05.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.650+0000 7f7da77fe640 1 -- 172.21.15.32:0/1823055582 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f7db00cfa30 con 0x7f7db810d820 2024-04-25T00:36:05.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.651+0000 7f7db5d9b640 1 --2- 172.21.15.32:0/1823055582 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7da00a2f00 0x7f7da00a53c0 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f7db8116d00 tx=0x7f7da800b480 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:05.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:05.661+0000 7f7da77fe640 1 -- 172.21.15.32:0/1823055582 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f7db0014030 con 0x7f7db810d820 2024-04-25T00:36:05.682 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:05 smithi032 ceph-mon[29357]: pgmap v86: 129 pgs: 129 active+clean; 19 GiB data, 39 GiB used, 1.0 TiB / 1.0 TiB avail; 2.1 MiB/s rd, 553 MiB/s wr, 1.53k op/s 2024-04-25T00:36:06.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:05 smithi121 ceph-mon[32231]: pgmap v86: 129 pgs: 129 active+clean; 19 GiB data, 39 GiB used, 1.0 TiB / 1.0 TiB avail; 2.1 MiB/s rd, 553 MiB/s wr, 1.53k op/s 2024-04-25T00:36:06.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:05 smithi130 ceph-mon[32525]: pgmap v86: 129 pgs: 129 active+clean; 19 GiB data, 39 GiB used, 1.0 TiB / 1.0 TiB avail; 2.1 MiB/s rd, 553 MiB/s wr, 1.53k op/s 2024-04-25T00:36:06.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.201+0000 7f7db759e640 1 -- 172.21.15.32:0/1823055582 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f7db811a580 con 0x7f7db810d820 2024-04-25T00:36:06.210 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:36:06.210 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":26,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14643,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6839/187492674","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6838","nonce":187492674},{"type":"v1","addr":"172.21.15.32:6839","nonce":187492674}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14655,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6841/1948785545","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6840","nonce":1948785545},{"type":"v1","addr":"172.21.15.32:6841","nonce":1948785545}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":26,"flags":306,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":true},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-25T00:30:47.866875+0000","modified":"2024-04-25T00:34:57.347792+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24506,"mds_1":24481,"mds_2":24476},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24443":{"gid":24443,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24445":{"gid":24445,"name":"f","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24457":{"gid":24457,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24476":{"gid":24476,"name":"h","rank":2,"incarnation":22,"state":"up:active","state_seq":12,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24481":{"gid":24481,"name":"i","rank":1,"incarnation":20,"state":"up:active","state_seq":11,"addr":"172.21.15.130:6839/4267125809","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":4267125809},{"type":"v1","addr":"172.21.15.130:6839","nonce":4267125809}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24506":{"gid":24506,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.121:6839/258444853","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6838","nonce":258444853},{"type":"v1","addr":"172.21.15.121:6839","nonce":258444853}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24506,"qdb_cluster":[24481,24506,24476]},"id":1}]} 2024-04-25T00:36:06.211 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.206+0000 7f7da77fe640 1 -- 172.21.15.32:0/1823055582 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 26 v26) ==== 94+0+11877 (secure 0 0 0) 0x7f7db009c2c0 con 0x7f7db810d820 2024-04-25T00:36:06.211 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 26 2024-04-25T00:36:06.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.215+0000 7f7db759e640 1 -- 172.21.15.32:0/1823055582 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7da00a2f00 msgr2=0x7f7da00a53c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:06.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.215+0000 7f7db759e640 1 --2- 172.21.15.32:0/1823055582 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7da00a2f00 0x7f7da00a53c0 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f7db8116d00 tx=0x7f7da800b480 comp rx=0 tx=0).stop 2024-04-25T00:36:06.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.216+0000 7f7db759e640 1 -- 172.21.15.32:0/1823055582 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7db810d820 msgr2=0x7f7db811c4c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:06.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.216+0000 7f7db759e640 1 --2- 172.21.15.32:0/1823055582 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7db810d820 0x7f7db811c4c0 secure :-1 s=READY pgs=601 cs=0 l=1 rev1=1 crypto rx=0x7f7db000cce0 tx=0x7f7db000a620 comp rx=0 tx=0).stop 2024-04-25T00:36:06.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.216+0000 7f7db759e640 1 -- 172.21.15.32:0/1823055582 shutdown_connections 2024-04-25T00:36:06.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.216+0000 7f7db759e640 1 --2- 172.21.15.32:0/1823055582 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7da00a2f00 0x7f7da00a53c0 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.216+0000 7f7db759e640 1 --2- 172.21.15.32:0/1823055582 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7db810fcc0 0x7f7db8115720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.216+0000 7f7db759e640 1 --2- 172.21.15.32:0/1823055582 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7db810e340 0x7f7db811ca00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.216+0000 7f7db759e640 1 --2- 172.21.15.32:0/1823055582 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7db810d820 0x7f7db811c4c0 unknown :-1 s=CLOSED pgs=601 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.216+0000 7f7db759e640 1 -- 172.21.15.32:0/1823055582 >> 172.21.15.32:0/1823055582 conn(0x7f7db806c650 msgr2=0x7f7db8070980 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:06.219 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.218+0000 7f7db759e640 1 -- 172.21.15.32:0/1823055582 shutdown_connections 2024-04-25T00:36:06.219 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.219+0000 7f7db759e640 1 -- 172.21.15.32:0/1823055582 wait complete. 2024-04-25T00:36:06.238 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-25T00:36:06.398 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.396+0000 7fa27c67b640 1 -- 172.21.15.32:0/4183270443 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa27410ff90 msgr2=0x7fa27410da80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:06.398 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.396+0000 7fa27c67b640 1 --2- 172.21.15.32:0/4183270443 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa27410ff90 0x7fa27410da80 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7fa27001a940 tx=0x7fa27003b470 comp rx=0 tx=0).stop 2024-04-25T00:36:06.398 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.396+0000 7fa27c67b640 1 -- 172.21.15.32:0/4183270443 shutdown_connections 2024-04-25T00:36:06.398 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.396+0000 7fa27c67b640 1 --2- 172.21.15.32:0/4183270443 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa27411af20 0x7fa27411d310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.398 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.396+0000 7fa27c67b640 1 --2- 172.21.15.32:0/4183270443 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa27410dfc0 0x7fa27410e440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.398 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.396+0000 7fa27c67b640 1 --2- 172.21.15.32:0/4183270443 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa27410ff90 0x7fa27410da80 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.398 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.396+0000 7fa27c67b640 1 -- 172.21.15.32:0/4183270443 >> 172.21.15.32:0/4183270443 conn(0x7fa27406c850 msgr2=0x7fa27406cc60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:06.399 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.399+0000 7fa27c67b640 1 -- 172.21.15.32:0/4183270443 shutdown_connections 2024-04-25T00:36:06.400 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.399+0000 7fa27c67b640 1 -- 172.21.15.32:0/4183270443 wait complete. 2024-04-25T00:36:06.401 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.400+0000 7fa27c67b640 1 Processor -- start 2024-04-25T00:36:06.401 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.400+0000 7fa27c67b640 1 -- start start 2024-04-25T00:36:06.401 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.400+0000 7fa27c67b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa27410dfc0 0x7fa27411a640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.403 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.400+0000 7fa27c67b640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa27410ff90 0x7fa2741166d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.403 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.400+0000 7fa27c67b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa27411af20 0x7fa274116c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.403 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.400+0000 7fa27c67b640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fa2741130e0 con 0x7fa27410dfc0 2024-04-25T00:36:06.403 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.400+0000 7fa27c67b640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fa274112f60 con 0x7fa27411af20 2024-04-25T00:36:06.403 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.400+0000 7fa27c67b640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fa274113260 con 0x7fa27410ff90 2024-04-25T00:36:06.404 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.401+0000 7fa27a417640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa27410dfc0 0x7fa27411a640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.404 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.401+0000 7fa27a417640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa27410dfc0 0x7fa27411a640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:58158/0 (socket says 172.21.15.32:58158) 2024-04-25T00:36:06.404 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.401+0000 7fa27a417640 1 -- 172.21.15.32:0/1241019098 learned_addr learned my addr 172.21.15.32:0/1241019098 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:06.404 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.401+0000 7fa279c16640 1 --2- 172.21.15.32:0/1241019098 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa27410ff90 0x7fa2741166d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.404 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.401+0000 7fa27a417640 1 -- 172.21.15.32:0/1241019098 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa27410ff90 msgr2=0x7fa2741166d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:06.404 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.401+0000 7fa27a417640 1 --2- 172.21.15.32:0/1241019098 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa27410ff90 0x7fa2741166d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.404 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.401+0000 7fa27a417640 1 -- 172.21.15.32:0/1241019098 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa27411af20 msgr2=0x7fa274116c10 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-04-25T00:36:06.404 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.401+0000 7fa27a417640 1 --2- 172.21.15.32:0/1241019098 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa27411af20 0x7fa274116c10 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.404 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.401+0000 7fa27a417640 1 -- 172.21.15.32:0/1241019098 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa274117310 con 0x7fa27410dfc0 2024-04-25T00:36:06.404 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.402+0000 7fa27a417640 1 --2- 172.21.15.32:0/1241019098 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa27410dfc0 0x7fa27411a640 secure :-1 s=READY pgs=602 cs=0 l=1 rev1=1 crypto rx=0x7fa27001a910 tx=0x7fa270004cd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:06.407 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.405+0000 7fa26b7fe640 1 -- 172.21.15.32:0/1241019098 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa270051070 con 0x7fa27410dfc0 2024-04-25T00:36:06.407 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.405+0000 7fa26b7fe640 1 -- 172.21.15.32:0/1241019098 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa270003680 con 0x7fa27410dfc0 2024-04-25T00:36:06.407 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.405+0000 7fa26b7fe640 1 -- 172.21.15.32:0/1241019098 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa270042440 con 0x7fa27410dfc0 2024-04-25T00:36:06.407 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.405+0000 7fa27c67b640 1 -- 172.21.15.32:0/1241019098 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa2741aacb0 con 0x7fa27410dfc0 2024-04-25T00:36:06.407 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.406+0000 7fa27c67b640 1 -- 172.21.15.32:0/1241019098 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa2741ab060 con 0x7fa27410dfc0 2024-04-25T00:36:06.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.414+0000 7fa26b7fe640 1 -- 172.21.15.32:0/1241019098 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7fa270054070 con 0x7fa27410dfc0 2024-04-25T00:36:06.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.415+0000 7fa26b7fe640 1 --2- 172.21.15.32:0/1241019098 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa2600a2f00 0x7fa2600a53c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.416+0000 7fa26b7fe640 1 -- 172.21.15.32:0/1241019098 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fa2701009e0 con 0x7fa27410dfc0 2024-04-25T00:36:06.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.416+0000 7fa279c16640 1 --2- 172.21.15.32:0/1241019098 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa2600a2f00 0x7fa2600a53c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.420 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.420+0000 7fa27f381740 1 Processor -- start 2024-04-25T00:36:06.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.420+0000 7fa27f381740 1 -- start start 2024-04-25T00:36:06.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.421+0000 7fa27f381740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c0489f9580 0x55c0489f9960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.421+0000 7fa27f381740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c0489f9ea0 0x55c048a04030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.421+0000 7fa27f381740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c048a04570 0x55c048a06960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.422+0000 7fa27f381740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55c04895e670 con 0x55c048a04570 2024-04-25T00:36:06.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.422+0000 7fa27f381740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55c048906970 con 0x55c0489f9580 2024-04-25T00:36:06.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.422+0000 7fa27f381740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55c04895a630 con 0x55c0489f9ea0 2024-04-25T00:36:06.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.422+0000 7fa279c16640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c0489f9ea0 0x55c048a04030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.422+0000 7fa279c16640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c0489f9ea0 0x55c048a04030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:58082/0 (socket says 172.21.15.32:58082) 2024-04-25T00:36:06.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.422+0000 7fa279c16640 1 -- 172.21.15.32:0/3808298593 learned_addr learned my addr 172.21.15.32:0/3808298593 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:06.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.422+0000 7fa27ac18640 1 --2- 172.21.15.32:0/3808298593 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c048a04570 0x55c048a06960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.424 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.423+0000 7fa27a417640 1 --2- 172.21.15.32:0/3808298593 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c0489f9580 0x55c0489f9960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.424 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.423+0000 7fa27ac18640 1 -- 172.21.15.32:0/3808298593 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c0489f9ea0 msgr2=0x55c048a04030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:06.424 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.423+0000 7fa27ac18640 1 --2- 172.21.15.32:0/3808298593 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c0489f9ea0 0x55c048a04030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.424 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.423+0000 7fa27ac18640 1 -- 172.21.15.32:0/3808298593 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c0489f9580 msgr2=0x55c0489f9960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:06.424 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.423+0000 7fa27ac18640 1 --2- 172.21.15.32:0/3808298593 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c0489f9580 0x55c0489f9960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.424 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.423+0000 7fa27ac18640 1 -- 172.21.15.32:0/3808298593 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c04892a400 con 0x55c048a04570 2024-04-25T00:36:06.425 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.424+0000 7fa27ac18640 1 --2- 172.21.15.32:0/3808298593 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c048a04570 0x55c048a06960 secure :-1 s=READY pgs=603 cs=0 l=1 rev1=1 crypto rx=0x7fa264009ad0 tx=0x7fa264009f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dbe4c7eda276976 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:06.425 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.425+0000 7fa268ff9640 1 -- 172.21.15.32:0/3808298593 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa26400ec90 con 0x55c048a04570 2024-04-25T00:36:06.425 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.425+0000 7fa268ff9640 1 -- 172.21.15.32:0/3808298593 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa26400ee30 con 0x55c048a04570 2024-04-25T00:36:06.426 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.426+0000 7fa27f381740 1 -- 172.21.15.32:0/3808298593 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c048a04570 msgr2=0x55c048a06960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:06.427 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.426+0000 7fa27f381740 1 --2- 172.21.15.32:0/3808298593 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c048a04570 0x55c048a06960 secure :-1 s=READY pgs=603 cs=0 l=1 rev1=1 crypto rx=0x7fa264009ad0 tx=0x7fa264009f90 comp rx=0 tx=0).stop 2024-04-25T00:36:06.427 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.427+0000 7fa27f381740 1 -- 172.21.15.32:0/3808298593 shutdown_connections 2024-04-25T00:36:06.428 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.427+0000 7fa27f381740 1 --2- 172.21.15.32:0/3808298593 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c048a04570 0x55c048a06960 unknown :-1 s=CLOSED pgs=603 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.428 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.427+0000 7fa27f381740 1 --2- 172.21.15.32:0/3808298593 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c0489f9ea0 0x55c048a04030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.428 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.427+0000 7fa27f381740 1 --2- 172.21.15.32:0/3808298593 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c0489f9580 0x55c0489f9960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.428 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.427+0000 7fa27f381740 1 -- 172.21.15.32:0/3808298593 >> 172.21.15.32:0/3808298593 conn(0x55c0489f58a0 msgr2=0x55c0489f5c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:06.433 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.432+0000 7fa27f381740 1 -- 172.21.15.32:0/3808298593 shutdown_connections 2024-04-25T00:36:06.433 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.432+0000 7fa279c16640 1 --2- 172.21.15.32:0/1241019098 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa2600a2f00 0x7fa2600a53c0 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7fa26c0148c0 tx=0x7fa26c014cc0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:06.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.435+0000 7fa27f381740 1 -- 172.21.15.32:0/3808298593 wait complete. 2024-04-25T00:36:06.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.435+0000 7fa27f381740 1 -- start start 2024-04-25T00:36:06.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.435+0000 7fa27f381740 1 Processor -- start 2024-04-25T00:36:06.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.438+0000 7fa27f381740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c0489f9580 0x55c048a03e50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.438+0000 7fa27f381740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c0489f9ea0 0x55c048a01140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.438+0000 7fa27f381740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c048a04570 0x55c048a01680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.443 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.438+0000 7fa27f381740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55c048a073b0 con 0x55c048a04570 2024-04-25T00:36:06.443 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.439+0000 7fa27f381740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55c04891a050 con 0x55c0489f9580 2024-04-25T00:36:06.443 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.439+0000 7fa27f381740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55c048a07530 con 0x55c0489f9ea0 2024-04-25T00:36:06.443 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.439+0000 7fa27f381740 10 client.0 _ll_register_callbacks cb 0x55c048671be0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:36:06.443 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.440+0000 7fa279c16640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c0489f9ea0 0x55c048a01140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.443 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.441+0000 7fa279c16640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c0489f9ea0 0x55c048a01140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:58086/0 (socket says 172.21.15.32:58086) 2024-04-25T00:36:06.443 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.441+0000 7fa279c16640 1 -- 172.21.15.32:0/2339510476 learned_addr learned my addr 172.21.15.32:0/2339510476 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:06.443 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.442+0000 7fa27a417640 1 --2- 172.21.15.32:0/2339510476 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c0489f9580 0x55c048a03e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.445 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.444+0000 7fa27ac18640 1 --2- 172.21.15.32:0/2339510476 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c048a04570 0x55c048a01680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.445 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.444+0000 7fa27a417640 1 -- 172.21.15.32:0/2339510476 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c0489f9ea0 msgr2=0x55c048a01140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:06.445 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.444+0000 7fa27a417640 1 --2- 172.21.15.32:0/2339510476 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c0489f9ea0 0x55c048a01140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.445 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.444+0000 7fa27a417640 1 -- 172.21.15.32:0/2339510476 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c048a04570 msgr2=0x55c048a01680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:06.445 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.444+0000 7fa27a417640 1 --2- 172.21.15.32:0/2339510476 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c048a04570 0x55c048a01680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.445 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.444+0000 7fa27a417640 1 -- 172.21.15.32:0/2339510476 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c0488ebdc0 con 0x55c0489f9580 2024-04-25T00:36:06.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.449+0000 7fa27a417640 1 --2- 172.21.15.32:0/2339510476 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c0489f9580 0x55c048a03e50 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7fa2700fdd20 tx=0x7fa2700fe290 comp rx=0 tx=0).ready entity=mon.2 client_cookie=463d6e09bd11f9e4 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:06.455 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.455+0000 7fa25dffb640 10 client.0 ms_handle_connect on v2:172.21.15.121:3300/0 2024-04-25T00:36:06.456 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.455+0000 7fa25dffb640 1 -- 172.21.15.32:0/2339510476 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa27005b040 con 0x55c0489f9580 2024-04-25T00:36:06.456 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.455+0000 7fa27f381740 1 -- 172.21.15.32:0/2339510476 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa26400ee30 con 0x55c0489f9580 2024-04-25T00:36:06.456 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.455+0000 7fa25dffb640 1 -- 172.21.15.32:0/2339510476 <== mon.2 v2:172.21.15.121:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fa2700cbf40 con 0x55c0489f9580 2024-04-25T00:36:06.460 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.459+0000 7fa25dffb640 1 -- 172.21.15.32:0/2339510476 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7fa2700ccaf0 con 0x55c0489f9580 2024-04-25T00:36:06.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.461+0000 7fa27f381740 10 client.25625 fetch_fsmap learned FSMap version 26 2024-04-25T00:36:06.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.462+0000 7fa27f381740 1 -- 172.21.15.32:0/2339510476 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x55c0486723f0 con 0x55c0489f9580 2024-04-25T00:36:06.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.463+0000 7fa25dffb640 1 -- 172.21.15.32:0/2339510476 <== mon.2 v2:172.21.15.121:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa26400ee30 con 0x55c0489f9580 2024-04-25T00:36:06.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.463+0000 7fa25dffb640 1 -- 172.21.15.32:0/2339510476 <== mon.2 v2:172.21.15.121:3300/0 5 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7fa2700cbbb0 con 0x55c0489f9580 2024-04-25T00:36:06.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.463+0000 7fa27f381740 10 client.25625 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:36:06.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.463+0000 7fa27f381740 10 client.25625 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-25T00:36:06.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.463+0000 7fa27f381740 20 client.25625 populate_metadata read hostname 'smithi032' 2024-04-25T00:36:06.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.463+0000 7fa27f381740 1 --2- 172.21.15.32:0/2339510476 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x55c048a14590 0x55c048a16a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.463+0000 7fa27f381740 4 client.25625 mds_command: new command op to 24476 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:36:06.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.463+0000 7fa27f381740 1 -- 172.21.15.32:0/2339510476 --> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55c04896bbb0 con 0x55c048a14590 2024-04-25T00:36:06.475 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.473+0000 7fa279c16640 1 --2- 172.21.15.32:0/2339510476 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x55c048a14590 0x55c048a16a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.481+0000 7fa279c16640 1 --2- 172.21.15.32:0/2339510476 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x55c048a14590 0x55c048a16a10 crc :-1 s=READY pgs=40 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=b2ae68fa66254c62 server_cookie=5a1833696d4f3c02 in_seq=0 out_seq=0 2024-04-25T00:36:06.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.481+0000 7fa25dffb640 10 client.25625 ms_handle_connect on v2:172.21.15.121:6836/3456022860 2024-04-25T00:36:06.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.485+0000 7fa25dffb640 1 -- 172.21.15.32:0/2339510476 <== mds.2 v2:172.21.15.121:6836/3456022860 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x55c04896bbb0 con 0x55c048a14590 2024-04-25T00:36:06.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.485+0000 7fa25dffb640 10 client.25625 handle_command_reply: tid=0 2024-04-25T00:36:06.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.485+0000 7fa27f381740 1 client.25625 shutdown 2024-04-25T00:36:06.490 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.489+0000 7fa27f381740 1 -- 172.21.15.32:0/2339510476 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c0489f9580 msgr2=0x55c048a03e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:06.490 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.489+0000 7fa27f381740 1 --2- 172.21.15.32:0/2339510476 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c0489f9580 0x55c048a03e50 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7fa2700fdd20 tx=0x7fa2700fe290 comp rx=0 tx=0).stop 2024-04-25T00:36:06.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.493+0000 7fa27f381740 1 -- 172.21.15.32:0/2339510476 shutdown_connections 2024-04-25T00:36:06.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.493+0000 7fa27f381740 1 --2- 172.21.15.32:0/2339510476 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x55c048a14590 0x55c048a16a10 crc :-1 s=READY pgs=40 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.493+0000 7fa27f381740 1 --2- 172.21.15.32:0/2339510476 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c048a04570 0x55c048a01680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.493+0000 7fa27f381740 1 --2- 172.21.15.32:0/2339510476 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c0489f9ea0 0x55c048a01140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.493+0000 7fa27f381740 1 --2- 172.21.15.32:0/2339510476 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c0489f9580 0x55c048a03e50 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.493+0000 7fa25dffb640 0 client.25625 ms_handle_reset on v2:172.21.15.121:6836/3456022860 2024-04-25T00:36:06.494 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.493+0000 7fa27f381740 1 -- 172.21.15.32:0/2339510476 >> 172.21.15.32:0/2339510476 conn(0x55c0489f58a0 msgr2=0x55c0489fb6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:06.498 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.497+0000 7fa27f381740 1 -- 172.21.15.32:0/2339510476 shutdown_connections 2024-04-25T00:36:06.501 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.501+0000 7fa27f381740 1 -- 172.21.15.32:0/2339510476 wait complete. 2024-04-25T00:36:06.506 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.504+0000 7fa27f381740 20 client.25625 trim_cache size 0 max 16384 2024-04-25T00:36:06.561 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.556+0000 7fa27f381740 1 Processor -- start 2024-04-25T00:36:06.561 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.560+0000 7fa27f381740 1 -- start start 2024-04-25T00:36:06.562 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.561+0000 7fa27f381740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c0489f9580 0x55c048a10350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.562 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.561+0000 7fa27f381740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c0489f9ea0 0x55c048a1c2e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.562 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.561+0000 7fa27f381740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c048a04570 0x55c048a1c820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.562 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.561+0000 7fa27f381740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fa258012940 con 0x55c0489f9ea0 2024-04-25T00:36:06.562 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.561+0000 7fa27f381740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fa258010210 con 0x55c048a04570 2024-04-25T00:36:06.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.561+0000 7fa27f381740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fa25800e060 con 0x55c0489f9580 2024-04-25T00:36:06.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.561+0000 7fa279c16640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c0489f9ea0 0x55c048a1c2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.561+0000 7fa279c16640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c0489f9ea0 0x55c048a1c2e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:58184/0 (socket says 172.21.15.32:58184) 2024-04-25T00:36:06.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.561+0000 7fa279c16640 1 -- 172.21.15.32:0/2052347926 learned_addr learned my addr 172.21.15.32:0/2052347926 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:06.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.562+0000 7fa27ac18640 1 --2- 172.21.15.32:0/2052347926 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c048a04570 0x55c048a1c820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.563+0000 7fa279c16640 1 -- 172.21.15.32:0/2052347926 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c0489f9580 msgr2=0x55c048a10350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:06.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.563+0000 7fa279c16640 1 --2- 172.21.15.32:0/2052347926 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c0489f9580 0x55c048a10350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.563+0000 7fa279c16640 1 -- 172.21.15.32:0/2052347926 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c048a04570 msgr2=0x55c048a1c820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:06.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.563+0000 7fa279c16640 1 --2- 172.21.15.32:0/2052347926 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c048a04570 0x55c048a1c820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.563+0000 7fa279c16640 1 -- 172.21.15.32:0/2052347926 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c04873c7d0 con 0x55c0489f9ea0 2024-04-25T00:36:06.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.566+0000 7fa279c16640 1 --2- 172.21.15.32:0/2052347926 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c0489f9ea0 0x55c048a1c2e0 secure :-1 s=READY pgs=604 cs=0 l=1 rev1=1 crypto rx=0x55c048a009a0 tx=0x7fa26c009500 comp rx=0 tx=0).ready entity=mon.0 client_cookie=efa6a99a8efade1e server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:06.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.567+0000 7fa25ffff640 1 -- 172.21.15.32:0/2052347926 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa26c028540 con 0x55c0489f9ea0 2024-04-25T00:36:06.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.567+0000 7fa25ffff640 1 -- 172.21.15.32:0/2052347926 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa26c0286e0 con 0x55c0489f9ea0 2024-04-25T00:36:06.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.567+0000 7fa25ffff640 1 -- 172.21.15.32:0/2052347926 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa26c021650 con 0x55c0489f9ea0 2024-04-25T00:36:06.570 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.569+0000 7fa27f381740 1 -- 172.21.15.32:0/2052347926 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c0489f9ea0 msgr2=0x55c048a1c2e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:06.570 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.569+0000 7fa27f381740 1 --2- 172.21.15.32:0/2052347926 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c0489f9ea0 0x55c048a1c2e0 secure :-1 s=READY pgs=604 cs=0 l=1 rev1=1 crypto rx=0x55c048a009a0 tx=0x7fa26c009500 comp rx=0 tx=0).stop 2024-04-25T00:36:06.574 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.572+0000 7fa27f381740 1 -- 172.21.15.32:0/2052347926 shutdown_connections 2024-04-25T00:36:06.574 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.572+0000 7fa27f381740 1 --2- 172.21.15.32:0/2052347926 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c048a04570 0x55c048a1c820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.574 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.572+0000 7fa27f381740 1 --2- 172.21.15.32:0/2052347926 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c0489f9ea0 0x55c048a1c2e0 unknown :-1 s=CLOSED pgs=604 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.574 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.572+0000 7fa27f381740 1 --2- 172.21.15.32:0/2052347926 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c0489f9580 0x55c048a10350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.574 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.572+0000 7fa27f381740 1 -- 172.21.15.32:0/2052347926 >> 172.21.15.32:0/2052347926 conn(0x55c0489f58a0 msgr2=0x55c0489f4b40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:06.576 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.575+0000 7fa27f381740 1 -- 172.21.15.32:0/2052347926 shutdown_connections 2024-04-25T00:36:06.578 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.576+0000 7fa27f381740 1 -- 172.21.15.32:0/2052347926 wait complete. 2024-04-25T00:36:06.578 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.577+0000 7fa27f381740 1 -- start start 2024-04-25T00:36:06.578 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.577+0000 7fa27f381740 1 Processor -- start 2024-04-25T00:36:06.579 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.577+0000 7fa27f381740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c0489f9580 0x55c048a0cce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.578+0000 7fa27f381740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c0489f9ea0 0x55c048a0a070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.578+0000 7fa27a417640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c0489f9580 0x55c048a0cce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.578+0000 7fa27a417640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c0489f9580 0x55c048a0cce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:54888/0 (socket says 172.21.15.32:54888) 2024-04-25T00:36:06.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.578+0000 7fa27a417640 1 -- 172.21.15.32:0/1320105982 learned_addr learned my addr 172.21.15.32:0/1320105982 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:06.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.578+0000 7fa27f381740 1 --2- 172.21.15.32:0/1320105982 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c048a04570 0x55c048a0a610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.578+0000 7fa27f381740 1 -- 172.21.15.32:0/1320105982 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fa25800e6d0 con 0x55c0489f9ea0 2024-04-25T00:36:06.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.579+0000 7fa27f381740 1 -- 172.21.15.32:0/1320105982 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fa25800fb40 con 0x55c0489f9580 2024-04-25T00:36:06.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.579+0000 7fa27ac18640 1 --2- 172.21.15.32:0/1320105982 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c048a04570 0x55c048a0a610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.579+0000 7fa27f381740 1 -- 172.21.15.32:0/1320105982 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fa25800eda0 con 0x55c048a04570 2024-04-25T00:36:06.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.579+0000 7fa27f381740 10 client.0 _ll_register_callbacks cb 0x7fa27001a2b0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:36:06.582 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.579+0000 7fa279c16640 1 --2- 172.21.15.32:0/1320105982 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c0489f9ea0 0x55c048a0a070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.582 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.580+0000 7fa279c16640 1 -- 172.21.15.32:0/1320105982 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c048a04570 msgr2=0x55c048a0a610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:06.582 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.580+0000 7fa279c16640 1 --2- 172.21.15.32:0/1320105982 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c048a04570 0x55c048a0a610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.582 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.580+0000 7fa279c16640 1 -- 172.21.15.32:0/1320105982 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c0489f9580 msgr2=0x55c048a0cce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:06.582 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.580+0000 7fa279c16640 1 --2- 172.21.15.32:0/1320105982 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c0489f9580 0x55c048a0cce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.582 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.580+0000 7fa279c16640 1 -- 172.21.15.32:0/1320105982 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c048a0ae40 con 0x55c0489f9ea0 2024-04-25T00:36:06.582 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.581+0000 7fa279c16640 1 --2- 172.21.15.32:0/1320105982 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c0489f9ea0 0x55c048a0a070 secure :-1 s=READY pgs=605 cs=0 l=1 rev1=1 crypto rx=0x55c048a07570 tx=0x7fa26c0096c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=54e00f86b0b7efaf server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:06.583 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.582+0000 7fa25dffb640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:36:06.583 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.582+0000 7fa25dffb640 1 -- 172.21.15.32:0/1320105982 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa26c036020 con 0x55c0489f9ea0 2024-04-25T00:36:06.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.582+0000 7fa25dffb640 1 -- 172.21.15.32:0/1320105982 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fa26c028950 con 0x55c0489f9ea0 2024-04-25T00:36:06.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.582+0000 7fa27f381740 1 -- 172.21.15.32:0/1320105982 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa26c0286e0 con 0x55c0489f9ea0 2024-04-25T00:36:06.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.582+0000 7fa27a417640 1 --2- 172.21.15.32:0/1320105982 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c0489f9580 0x55c048a0cce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:36:06.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.582+0000 7fa27ac18640 1 --2- 172.21.15.32:0/1320105982 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c048a04570 0x55c048a0a610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:36:06.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.584+0000 7fa25dffb640 1 -- 172.21.15.32:0/1320105982 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7fa26c004a40 con 0x55c0489f9ea0 2024-04-25T00:36:06.586 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.586+0000 7fa27f381740 10 client.16074 fetch_fsmap learned FSMap version 26 2024-04-25T00:36:06.587 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.586+0000 7fa27f381740 1 -- 172.21.15.32:0/1320105982 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x55c048a7ac30 con 0x55c0489f9ea0 2024-04-25T00:36:06.591 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.591+0000 7fa25dffb640 1 -- 172.21.15.32:0/1320105982 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa26c0286e0 con 0x55c0489f9ea0 2024-04-25T00:36:06.592 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.591+0000 7fa25dffb640 1 -- 172.21.15.32:0/1320105982 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7fa26c028020 con 0x55c0489f9ea0 2024-04-25T00:36:06.593 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.593+0000 7fa27f381740 10 client.16074 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:36:06.593 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.593+0000 7fa27f381740 10 client.16074 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-25T00:36:06.594 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.593+0000 7fa27f381740 20 client.16074 populate_metadata read hostname 'smithi032' 2024-04-25T00:36:06.594 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.593+0000 7fa27f381740 1 --2- 172.21.15.32:0/1320105982 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x55c048a7b250 0x55c048a7d640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.594 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.593+0000 7fa27f381740 4 client.16074 mds_command: new command op to 24476 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-25T00:36:06.594 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.593+0000 7fa27f381740 1 -- 172.21.15.32:0/1320105982 --> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55c048965f00 con 0x55c048a7b250 2024-04-25T00:36:06.594 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.593+0000 7fa27a417640 1 --2- 172.21.15.32:0/1320105982 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x55c048a7b250 0x55c048a7d640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.597 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.597+0000 7fa27a417640 1 --2- 172.21.15.32:0/1320105982 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x55c048a7b250 0x55c048a7d640 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=3f4d5ef3e3f512c3 server_cookie=36fbbef1b77bc3a7 in_seq=0 out_seq=0 2024-04-25T00:36:06.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.597+0000 7fa25dffb640 10 client.16074 ms_handle_connect on v2:172.21.15.121:6836/3456022860 2024-04-25T00:36:06.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.597+0000 7fa25dffb640 1 -- 172.21.15.32:0/1320105982 <== mds.2 v2:172.21.15.121:6836/3456022860 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55c048965f00 con 0x55c048a7b250 2024-04-25T00:36:06.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.597+0000 7fa25dffb640 10 client.16074 handle_command_reply: tid=0 2024-04-25T00:36:06.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.597+0000 7fa27f381740 1 client.16074 shutdown 2024-04-25T00:36:06.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.598+0000 7fa27f381740 1 -- 172.21.15.32:0/1320105982 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c0489f9ea0 msgr2=0x55c048a0a070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:06.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.598+0000 7fa27f381740 1 --2- 172.21.15.32:0/1320105982 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c0489f9ea0 0x55c048a0a070 secure :-1 s=READY pgs=605 cs=0 l=1 rev1=1 crypto rx=0x55c048a07570 tx=0x7fa26c0096c0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.600+0000 7fa27f381740 1 -- 172.21.15.32:0/1320105982 shutdown_connections 2024-04-25T00:36:06.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.600+0000 7fa27f381740 1 --2- 172.21.15.32:0/1320105982 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x55c048a7b250 0x55c048a7d640 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.600+0000 7fa27f381740 1 --2- 172.21.15.32:0/1320105982 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c048a04570 0x55c048a0a610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.600+0000 7fa27f381740 1 --2- 172.21.15.32:0/1320105982 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c0489f9ea0 0x55c048a0a070 unknown :-1 s=CLOSED pgs=605 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.600+0000 7fa27f381740 1 --2- 172.21.15.32:0/1320105982 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c0489f9580 0x55c048a0cce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.600+0000 7fa27f381740 1 -- 172.21.15.32:0/1320105982 >> 172.21.15.32:0/1320105982 conn(0x55c0489f58a0 msgr2=0x55c048a0f340 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:06.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.601+0000 7fa25dffb640 0 client.16074 ms_handle_reset on v2:172.21.15.121:6836/3456022860 2024-04-25T00:36:06.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.601+0000 7fa27f381740 1 -- 172.21.15.32:0/1320105982 shutdown_connections 2024-04-25T00:36:06.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.601+0000 7fa27f381740 1 -- 172.21.15.32:0/1320105982 wait complete. 2024-04-25T00:36:06.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.601+0000 7fa27f381740 20 client.16074 trim_cache size 0 max 16384 2024-04-25T00:36:06.604 INFO:teuthology.orchestra.run.smithi032.stdout:[] 2024-04-25T00:36:06.608 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.608+0000 7fa27c67b640 1 -- 172.21.15.32:0/1241019098 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa2600a2f00 msgr2=0x7fa2600a53c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:06.609 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.608+0000 7fa27c67b640 1 --2- 172.21.15.32:0/1241019098 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa2600a2f00 0x7fa2600a53c0 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7fa26c0148c0 tx=0x7fa26c014cc0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.609 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.608+0000 7fa27c67b640 1 -- 172.21.15.32:0/1241019098 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa27410dfc0 msgr2=0x7fa27411a640 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:06.609 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.608+0000 7fa27c67b640 1 --2- 172.21.15.32:0/1241019098 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa27410dfc0 0x7fa27411a640 secure :-1 s=READY pgs=602 cs=0 l=1 rev1=1 crypto rx=0x7fa27001a910 tx=0x7fa270004cd0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.613 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.611+0000 7fa27c67b640 1 -- 172.21.15.32:0/1241019098 shutdown_connections 2024-04-25T00:36:06.613 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.611+0000 7fa27c67b640 1 --2- 172.21.15.32:0/1241019098 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa2600a2f00 0x7fa2600a53c0 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.611+0000 7fa27c67b640 1 --2- 172.21.15.32:0/1241019098 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa27411af20 0x7fa274116c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.611+0000 7fa27c67b640 1 --2- 172.21.15.32:0/1241019098 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa27410ff90 0x7fa2741166d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.611+0000 7fa27c67b640 1 --2- 172.21.15.32:0/1241019098 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa27410dfc0 0x7fa27411a640 unknown :-1 s=CLOSED pgs=602 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.611+0000 7fa27c67b640 1 -- 172.21.15.32:0/1241019098 >> 172.21.15.32:0/1241019098 conn(0x7fa27406c850 msgr2=0x7fa27411c660 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:06.615 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.613+0000 7fa27c67b640 1 -- 172.21.15.32:0/1241019098 shutdown_connections 2024-04-25T00:36:06.615 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.613+0000 7fa27c67b640 1 -- 172.21.15.32:0/1241019098 wait complete. 2024-04-25T00:36:06.643 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-25T00:36:06.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.935+0000 7fd238658640 1 -- 172.21.15.32:0/566850972 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd23010ff90 msgr2=0x7fd23010da80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:06.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.935+0000 7fd238658640 1 --2- 172.21.15.32:0/566850972 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd23010ff90 0x7fd23010da80 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7fd220009860 tx=0x7fd22002a360 comp rx=0 tx=0).stop 2024-04-25T00:36:06.940 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.937+0000 7fd238658640 1 -- 172.21.15.32:0/566850972 shutdown_connections 2024-04-25T00:36:06.940 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.937+0000 7fd238658640 1 --2- 172.21.15.32:0/566850972 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd23011af20 0x7fd23011d310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.940 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.937+0000 7fd238658640 1 --2- 172.21.15.32:0/566850972 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd23010dfc0 0x7fd23010e440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.937+0000 7fd238658640 1 --2- 172.21.15.32:0/566850972 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd23010ff90 0x7fd23010da80 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.938+0000 7fd238658640 1 -- 172.21.15.32:0/566850972 >> 172.21.15.32:0/566850972 conn(0x7fd23006c850 msgr2=0x7fd23006cc60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:06.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.939+0000 7fd238658640 1 -- 172.21.15.32:0/566850972 shutdown_connections 2024-04-25T00:36:06.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.940+0000 7fd238658640 1 -- 172.21.15.32:0/566850972 wait complete. 2024-04-25T00:36:06.945 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.941+0000 7fd238658640 1 Processor -- start 2024-04-25T00:36:06.945 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.941+0000 7fd238658640 1 -- start start 2024-04-25T00:36:06.945 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.942+0000 7fd238658640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd23010dfc0 0x7fd2301a6e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.945 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.942+0000 7fd238658640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd23010ff90 0x7fd2301a7380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.945 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.942+0000 7fd238658640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd23011af20 0x7fd2301ab700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.946 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.942+0000 7fd238658640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fd2301208e0 con 0x7fd23011af20 2024-04-25T00:36:06.946 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.942+0000 7fd238658640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fd230120760 con 0x7fd23010ff90 2024-04-25T00:36:06.946 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.942+0000 7fd238658640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fd230120a60 con 0x7fd23010dfc0 2024-04-25T00:36:06.946 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.942+0000 7fd2363f4640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd23010dfc0 0x7fd2301a6e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.946 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.942+0000 7fd2363f4640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd23010dfc0 0x7fd2301a6e40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:58116/0 (socket says 172.21.15.32:58116) 2024-04-25T00:36:06.946 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.942+0000 7fd2363f4640 1 -- 172.21.15.32:0/1720125441 learned_addr learned my addr 172.21.15.32:0/1720125441 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:06.946 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.943+0000 7fd236bf5640 1 --2- 172.21.15.32:0/1720125441 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd23011af20 0x7fd2301ab700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.946 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.943+0000 7fd235bf3640 1 --2- 172.21.15.32:0/1720125441 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd23010ff90 0x7fd2301a7380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.946 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.943+0000 7fd2363f4640 1 -- 172.21.15.32:0/1720125441 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd23010ff90 msgr2=0x7fd2301a7380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:06.946 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.943+0000 7fd2363f4640 1 --2- 172.21.15.32:0/1720125441 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd23010ff90 0x7fd2301a7380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.947 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.943+0000 7fd2363f4640 1 -- 172.21.15.32:0/1720125441 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd23011af20 msgr2=0x7fd2301ab700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:06.947 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.943+0000 7fd2363f4640 1 --2- 172.21.15.32:0/1720125441 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd23011af20 0x7fd2301ab700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.947 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.943+0000 7fd2363f4640 1 -- 172.21.15.32:0/1720125441 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd2301abe30 con 0x7fd23010dfc0 2024-04-25T00:36:06.947 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.944+0000 7fd2363f4640 1 --2- 172.21.15.32:0/1720125441 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd23010dfc0 0x7fd2301a6e40 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7fd22002a8d0 tx=0x7fd220033d10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:06.947 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.945+0000 7fd2277fe640 1 -- 172.21.15.32:0/1720125441 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd220037040 con 0x7fd23010dfc0 2024-04-25T00:36:06.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.945+0000 7fd238658640 1 -- 172.21.15.32:0/1720125441 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd2301ac0c0 con 0x7fd23010dfc0 2024-04-25T00:36:06.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.945+0000 7fd238658640 1 -- 172.21.15.32:0/1720125441 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd230112710 con 0x7fd23010dfc0 2024-04-25T00:36:06.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.945+0000 7fd2277fe640 1 -- 172.21.15.32:0/1720125441 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd220040070 con 0x7fd23010dfc0 2024-04-25T00:36:06.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.946+0000 7fd2277fe640 1 -- 172.21.15.32:0/1720125441 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd2200046c0 con 0x7fd23010dfc0 2024-04-25T00:36:06.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.949+0000 7fd23b366740 1 Processor -- start 2024-04-25T00:36:06.950 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.950+0000 7fd2277fe640 1 -- 172.21.15.32:0/1720125441 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7fd220004860 con 0x7fd23010dfc0 2024-04-25T00:36:06.951 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.950+0000 7fd23b366740 1 -- start start 2024-04-25T00:36:06.951 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.950+0000 7fd23b366740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c1775a8580 0x55c1775a8960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.950+0000 7fd23b366740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c1775a8ea0 0x55c1775b3030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.950+0000 7fd23b366740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c1775b3570 0x55c1775b5960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.950+0000 7fd23b366740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55c17750d670 con 0x55c1775b3570 2024-04-25T00:36:06.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.950+0000 7fd23b366740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55c1774b5970 con 0x55c1775a8ea0 2024-04-25T00:36:06.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.950+0000 7fd23b366740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55c177509630 con 0x55c1775a8580 2024-04-25T00:36:06.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.951+0000 7fd2363f4640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c1775a8ea0 0x55c1775b3030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.951+0000 7fd2363f4640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c1775a8ea0 0x55c1775b3030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:54924/0 (socket says 172.21.15.32:54924) 2024-04-25T00:36:06.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.951+0000 7fd2363f4640 1 -- 172.21.15.32:0/1530171546 learned_addr learned my addr 172.21.15.32:0/1530171546 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:06.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.951+0000 7fd236bf5640 1 --2- 172.21.15.32:0/1530171546 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c1775a8580 0x55c1775a8960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.953 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.951+0000 7fd2363f4640 1 -- 172.21.15.32:0/1530171546 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c1775a8580 msgr2=0x55c1775a8960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:06.953 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.951+0000 7fd2363f4640 1 --2- 172.21.15.32:0/1530171546 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c1775a8580 0x55c1775a8960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.953 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.951+0000 7fd2363f4640 1 -- 172.21.15.32:0/1530171546 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c1775b3570 msgr2=0x55c1775b5960 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-25T00:36:06.953 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.951+0000 7fd2363f4640 1 --2- 172.21.15.32:0/1530171546 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c1775b3570 0x55c1775b5960 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.954 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.951+0000 7fd2363f4640 1 -- 172.21.15.32:0/1530171546 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c1774d9400 con 0x55c1775a8ea0 2024-04-25T00:36:06.954 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.952+0000 7fd2363f4640 1 --2- 172.21.15.32:0/1530171546 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c1775a8ea0 0x55c1775b3030 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7fd2200ec3a0 tx=0x7fd2200f70e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2b536020b04b1aa2 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:06.956 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.956+0000 7fd224bfe640 1 -- 172.21.15.32:0/1530171546 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd2200f7cc0 con 0x55c1775a8ea0 2024-04-25T00:36:06.957 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.956+0000 7fd2277fe640 1 --2- 172.21.15.32:0/1720125441 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd21c0a2f00 0x7fd21c0a53c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.957 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.957+0000 7fd2277fe640 1 -- 172.21.15.32:0/1720125441 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fd2200ef010 con 0x7fd23010dfc0 2024-04-25T00:36:06.960 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.956+0000 7fd224bfe640 1 -- 172.21.15.32:0/1530171546 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd2200fc070 con 0x55c1775a8ea0 2024-04-25T00:36:06.960 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.958+0000 7fd224bfe640 1 -- 172.21.15.32:0/1530171546 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd2200ef820 con 0x55c1775a8ea0 2024-04-25T00:36:06.960 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.959+0000 7fd23b366740 1 -- 172.21.15.32:0/1530171546 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c1775a8ea0 msgr2=0x55c1775b3030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:06.960 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.959+0000 7fd23b366740 1 --2- 172.21.15.32:0/1530171546 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c1775a8ea0 0x55c1775b3030 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7fd2200ec3a0 tx=0x7fd2200f70e0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.961 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.961+0000 7fd236bf5640 1 --2- 172.21.15.32:0/1720125441 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd21c0a2f00 0x7fd21c0a53c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.962 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.961+0000 7fd23b366740 1 -- 172.21.15.32:0/1530171546 shutdown_connections 2024-04-25T00:36:06.962 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.961+0000 7fd23b366740 1 --2- 172.21.15.32:0/1530171546 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c1775b3570 0x55c1775b5960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.962 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.961+0000 7fd23b366740 1 --2- 172.21.15.32:0/1530171546 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c1775a8ea0 0x55c1775b3030 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.962 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.961+0000 7fd23b366740 1 --2- 172.21.15.32:0/1530171546 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c1775a8580 0x55c1775a8960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.962 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.961+0000 7fd23b366740 1 -- 172.21.15.32:0/1530171546 >> 172.21.15.32:0/1530171546 conn(0x55c1775a48a0 msgr2=0x55c1775a4c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:06.962 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.962+0000 7fd23b366740 1 -- 172.21.15.32:0/1530171546 shutdown_connections 2024-04-25T00:36:06.965 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.962+0000 7fd236bf5640 1 --2- 172.21.15.32:0/1720125441 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd21c0a2f00 0x7fd21c0a53c0 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7fd228005fd0 tx=0x7fd2280096e0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:06.966 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.965+0000 7fd23b366740 1 -- 172.21.15.32:0/1530171546 wait complete. 2024-04-25T00:36:06.966 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.965+0000 7fd23b366740 1 -- start start 2024-04-25T00:36:06.966 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.965+0000 7fd23b366740 1 Processor -- start 2024-04-25T00:36:06.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.966+0000 7fd23b366740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c1775a8580 0x55c1775b2f00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.966+0000 7fd23b366740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c1775a8ea0 0x55c1775b01f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.966+0000 7fd23b366740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c1775b3570 0x55c1775b0770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.966+0000 7fd23b366740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55c1775b6120 con 0x55c1775a8580 2024-04-25T00:36:06.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.966+0000 7fd23b366740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55c1774c9050 con 0x55c1775b3570 2024-04-25T00:36:06.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.966+0000 7fd23b366740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55c1775b62a0 con 0x55c1775a8ea0 2024-04-25T00:36:06.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.966+0000 7fd23b366740 10 client.0 _ll_register_callbacks cb 0x55c177220be0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:36:06.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.968+0000 7fd235bf3640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c1775a8ea0 0x55c1775b01f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.968+0000 7fd2363f4640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c1775a8580 0x55c1775b2f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.968+0000 7fd2363f4640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c1775a8580 0x55c1775b2f00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:58226/0 (socket says 172.21.15.32:58226) 2024-04-25T00:36:06.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.968+0000 7fd2363f4640 1 -- 172.21.15.32:0/566062833 learned_addr learned my addr 172.21.15.32:0/566062833 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:06.974 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.971+0000 7fd235bf3640 1 -- 172.21.15.32:0/566062833 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c1775b3570 msgr2=0x55c1775b0770 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-25T00:36:06.974 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.971+0000 7fd235bf3640 1 --2- 172.21.15.32:0/566062833 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c1775b3570 0x55c1775b0770 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.974 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.971+0000 7fd235bf3640 1 -- 172.21.15.32:0/566062833 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c1775a8580 msgr2=0x55c1775b2f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:06.974 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.971+0000 7fd235bf3640 1 --2- 172.21.15.32:0/566062833 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c1775a8580 0x55c1775b2f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.974 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.971+0000 7fd235bf3640 1 -- 172.21.15.32:0/566062833 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c17749adc0 con 0x55c1775a8ea0 2024-04-25T00:36:06.975 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.972+0000 7fd235bf3640 1 --2- 172.21.15.32:0/566062833 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c1775a8ea0 0x55c1775b01f0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7fd23011cd30 tx=0x7fd22c00f730 comp rx=0 tx=0).ready entity=mon.1 client_cookie=fca0a9b73d4dcad0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:06.975 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.972+0000 7fd219ffb640 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:36:06.975 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.972+0000 7fd219ffb640 1 -- 172.21.15.32:0/566062833 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd22c015070 con 0x55c1775a8ea0 2024-04-25T00:36:06.975 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.972+0000 7fd23b366740 1 -- 172.21.15.32:0/566062833 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd2200fc070 con 0x55c1775a8ea0 2024-04-25T00:36:06.975 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.972+0000 7fd219ffb640 1 -- 172.21.15.32:0/566062833 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fd22c01a660 con 0x55c1775a8ea0 2024-04-25T00:36:06.975 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.974+0000 7fd219ffb640 1 -- 172.21.15.32:0/566062833 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7fd22c023050 con 0x55c1775a8ea0 2024-04-25T00:36:06.975 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.974+0000 7fd23b366740 10 client.25612 fetch_fsmap learned FSMap version 26 2024-04-25T00:36:06.975 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.974+0000 7fd23b366740 1 -- 172.21.15.32:0/566062833 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x55c1772213f0 con 0x55c1775a8ea0 2024-04-25T00:36:06.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.981+0000 7fd219ffb640 1 -- 172.21.15.32:0/566062833 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd2200fc070 con 0x55c1775a8ea0 2024-04-25T00:36:06.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.981+0000 7fd219ffb640 1 -- 172.21.15.32:0/566062833 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7fd22c011b40 con 0x55c1775a8ea0 2024-04-25T00:36:06.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.981+0000 7fd23b366740 10 client.25612 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:36:06.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.981+0000 7fd23b366740 10 client.25612 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-25T00:36:06.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.981+0000 7fd23b366740 20 client.25612 populate_metadata read hostname 'smithi032' 2024-04-25T00:36:06.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.981+0000 7fd23b366740 1 --2- 172.21.15.32:0/566062833 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55c1775c3480 0x55c1775c58a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:06.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.981+0000 7fd23b366740 4 client.25612 mds_command: new command op to 24481 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:36:06.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.981+0000 7fd23b366740 1 -- 172.21.15.32:0/566062833 --> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55c17751abb0 con 0x55c1775c3480 2024-04-25T00:36:06.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.981+0000 7fd2363f4640 1 --2- 172.21.15.32:0/566062833 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55c1775c3480 0x55c1775c58a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:06.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.982+0000 7fd2363f4640 1 --2- 172.21.15.32:0/566062833 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55c1775c3480 0x55c1775c58a0 crc :-1 s=READY pgs=37 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=260977b9d701326f server_cookie=5f02c99063f49d53 in_seq=0 out_seq=0 2024-04-25T00:36:06.985 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.984+0000 7fd219ffb640 10 client.25612 ms_handle_connect on v2:172.21.15.130:6838/4267125809 2024-04-25T00:36:06.985 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.984+0000 7fd219ffb640 1 -- 172.21.15.32:0/566062833 <== mds.1 v2:172.21.15.130:6838/4267125809 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x55c17751abb0 con 0x55c1775c3480 2024-04-25T00:36:06.985 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.984+0000 7fd219ffb640 10 client.25612 handle_command_reply: tid=0 2024-04-25T00:36:06.985 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.984+0000 7fd23b366740 1 client.25612 shutdown 2024-04-25T00:36:06.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.987+0000 7fd23b366740 1 -- 172.21.15.32:0/566062833 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c1775a8ea0 msgr2=0x55c1775b01f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:06.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.987+0000 7fd23b366740 1 --2- 172.21.15.32:0/566062833 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c1775a8ea0 0x55c1775b01f0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7fd23011cd30 tx=0x7fd22c00f730 comp rx=0 tx=0).stop 2024-04-25T00:36:06.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.987+0000 7fd23b366740 1 -- 172.21.15.32:0/566062833 shutdown_connections 2024-04-25T00:36:06.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.987+0000 7fd23b366740 1 --2- 172.21.15.32:0/566062833 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c1775b3570 0x55c1775b0770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.987+0000 7fd23b366740 1 --2- 172.21.15.32:0/566062833 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c1775a8ea0 0x55c1775b01f0 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.987+0000 7fd23b366740 1 --2- 172.21.15.32:0/566062833 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55c1775c3480 0x55c1775c58a0 crc :-1 s=READY pgs=37 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.987+0000 7fd23b366740 1 --2- 172.21.15.32:0/566062833 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c1775a8580 0x55c1775b2f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:06.989 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.987+0000 7fd23b366740 1 -- 172.21.15.32:0/566062833 >> 172.21.15.32:0/566062833 conn(0x55c1775a48a0 msgr2=0x55c1775a3b40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:06.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.989+0000 7fd219ffb640 0 client.25612 ms_handle_reset on v2:172.21.15.130:6838/4267125809 2024-04-25T00:36:06.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.989+0000 7fd23b366740 1 -- 172.21.15.32:0/566062833 shutdown_connections 2024-04-25T00:36:06.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.990+0000 7fd23b366740 1 -- 172.21.15.32:0/566062833 wait complete. 2024-04-25T00:36:06.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:06.990+0000 7fd23b366740 20 client.25612 trim_cache size 0 max 16384 2024-04-25T00:36:06.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:06 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1823055582' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:36:07.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.029+0000 7fd23b366740 1 Processor -- start 2024-04-25T00:36:07.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.029+0000 7fd23b366740 1 -- start start 2024-04-25T00:36:07.032 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.029+0000 7fd23b366740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c1775a8580 0x55c1775a42c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.029+0000 7fd23b366740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c1775a8ea0 0x55c1775aa3e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.030+0000 7fd23b366740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c1775b3570 0x55c1775b2a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.030+0000 7fd2363f4640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c1775a8580 0x55c1775a42c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:07.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.030+0000 7fd2363f4640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c1775a8580 0x55c1775a42c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:58146/0 (socket says 172.21.15.32:58146) 2024-04-25T00:36:07.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.030+0000 7fd2363f4640 1 -- 172.21.15.32:0/2099204905 learned_addr learned my addr 172.21.15.32:0/2099204905 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:07.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.030+0000 7fd23b366740 1 -- 172.21.15.32:0/2099204905 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fd214012940 con 0x55c1775a8ea0 2024-04-25T00:36:07.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.030+0000 7fd23b366740 1 -- 172.21.15.32:0/2099204905 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fd214010210 con 0x55c1775b3570 2024-04-25T00:36:07.033 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.030+0000 7fd23b366740 1 -- 172.21.15.32:0/2099204905 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fd21400e060 con 0x55c1775a8580 2024-04-25T00:36:07.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.030+0000 7fd236bf5640 1 --2- 172.21.15.32:0/2099204905 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c1775b3570 0x55c1775b2a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:07.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.030+0000 7fd2363f4640 1 -- 172.21.15.32:0/2099204905 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c1775b3570 msgr2=0x55c1775b2a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:07.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.030+0000 7fd235bf3640 1 --2- 172.21.15.32:0/2099204905 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c1775a8ea0 0x55c1775aa3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:07.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.030+0000 7fd2363f4640 1 --2- 172.21.15.32:0/2099204905 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c1775b3570 0x55c1775b2a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.030+0000 7fd2363f4640 1 -- 172.21.15.32:0/2099204905 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c1775a8ea0 msgr2=0x55c1775aa3e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:07.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.030+0000 7fd2363f4640 1 --2- 172.21.15.32:0/2099204905 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c1775a8ea0 0x55c1775aa3e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.031+0000 7fd2363f4640 1 -- 172.21.15.32:0/2099204905 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c1775a8cd0 con 0x55c1775a8580 2024-04-25T00:36:07.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.031+0000 7fd236bf5640 1 --2- 172.21.15.32:0/2099204905 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c1775b3570 0x55c1775b2a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:36:07.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.031+0000 7fd235bf3640 1 --2- 172.21.15.32:0/2099204905 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c1775a8ea0 0x55c1775aa3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:36:07.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.031+0000 7fd2363f4640 1 --2- 172.21.15.32:0/2099204905 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c1775a8580 0x55c1775a42c0 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7fd220005170 tx=0x7fd22002a360 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6d4145e17ba24fff server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:07.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.031+0000 7fd21bfff640 1 -- 172.21.15.32:0/2099204905 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd2200cbae0 con 0x55c1775a8580 2024-04-25T00:36:07.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.031+0000 7fd21bfff640 1 -- 172.21.15.32:0/2099204905 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd2200fc040 con 0x55c1775a8580 2024-04-25T00:36:07.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.031+0000 7fd21bfff640 1 -- 172.21.15.32:0/2099204905 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd2200bc9b0 con 0x55c1775a8580 2024-04-25T00:36:07.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.035+0000 7fd23b366740 1 -- 172.21.15.32:0/2099204905 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c1775a8580 msgr2=0x55c1775a42c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:07.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.035+0000 7fd23b366740 1 --2- 172.21.15.32:0/2099204905 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c1775a8580 0x55c1775a42c0 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7fd220005170 tx=0x7fd22002a360 comp rx=0 tx=0).stop 2024-04-25T00:36:07.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.037+0000 7fd23b366740 1 -- 172.21.15.32:0/2099204905 shutdown_connections 2024-04-25T00:36:07.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.037+0000 7fd23b366740 1 --2- 172.21.15.32:0/2099204905 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c1775b3570 0x55c1775b2a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.037+0000 7fd23b366740 1 --2- 172.21.15.32:0/2099204905 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c1775a8ea0 0x55c1775aa3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.037+0000 7fd23b366740 1 --2- 172.21.15.32:0/2099204905 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c1775a8580 0x55c1775a42c0 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.038 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.037+0000 7fd23b366740 1 -- 172.21.15.32:0/2099204905 >> 172.21.15.32:0/2099204905 conn(0x55c1775a48a0 msgr2=0x55c1775a9cd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:07.038 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.037+0000 7fd23b366740 1 -- 172.21.15.32:0/2099204905 shutdown_connections 2024-04-25T00:36:07.039 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.038+0000 7fd23b366740 1 -- 172.21.15.32:0/2099204905 wait complete. 2024-04-25T00:36:07.039 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.039+0000 7fd23b366740 1 -- start start 2024-04-25T00:36:07.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.039+0000 7fd23b366740 1 Processor -- start 2024-04-25T00:36:07.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.040+0000 7fd23b366740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c1775a8580 0x55c1775bba40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.040+0000 7fd23b366740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c1775a8ea0 0x55c1775acd00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.040+0000 7fd23b366740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c1775b3570 0x55c1775ad420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.040+0000 7fd2363f4640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c1775a8580 0x55c1775bba40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:07.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.040+0000 7fd2363f4640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c1775a8580 0x55c1775bba40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:54936/0 (socket says 172.21.15.32:54936) 2024-04-25T00:36:07.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.040+0000 7fd2363f4640 1 -- 172.21.15.32:0/1441504335 learned_addr learned my addr 172.21.15.32:0/1441504335 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:07.043 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.040+0000 7fd23b366740 1 -- 172.21.15.32:0/1441504335 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fd21400e6d0 con 0x55c1775b3570 2024-04-25T00:36:07.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.040+0000 7fd23b366740 1 -- 172.21.15.32:0/1441504335 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fd21400fb40 con 0x55c1775a8580 2024-04-25T00:36:07.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.040+0000 7fd23b366740 1 -- 172.21.15.32:0/1441504335 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fd21400eda0 con 0x55c1775a8ea0 2024-04-25T00:36:07.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.040+0000 7fd23b366740 10 client.0 _ll_register_callbacks cb 0x7fd22c005f30 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:36:07.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.041+0000 7fd2363f4640 1 -- 172.21.15.32:0/1441504335 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c1775a8ea0 msgr2=0x55c1775acd00 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-25T00:36:07.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.041+0000 7fd2363f4640 1 --2- 172.21.15.32:0/1441504335 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c1775a8ea0 0x55c1775acd00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.041+0000 7fd2363f4640 1 -- 172.21.15.32:0/1441504335 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c1775b3570 msgr2=0x55c1775ad420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:07.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.041+0000 7fd2363f4640 1 --2- 172.21.15.32:0/1441504335 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c1775b3570 0x55c1775ad420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.041+0000 7fd2363f4640 1 -- 172.21.15.32:0/1441504335 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c1772eb7d0 con 0x55c1775a8580 2024-04-25T00:36:07.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.041+0000 7fd2363f4640 1 --2- 172.21.15.32:0/1441504335 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c1775a8580 0x55c1775bba40 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7fd2200bbfb0 tx=0x7fd2200f7710 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5bc6e3344dc2ef6a server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:07.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.041+0000 7fd219ffb640 10 client.0 ms_handle_connect on v2:172.21.15.121:3300/0 2024-04-25T00:36:07.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.041+0000 7fd219ffb640 1 -- 172.21.15.32:0/1441504335 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd2200bbdf0 con 0x55c1775a8580 2024-04-25T00:36:07.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.041+0000 7fd23b366740 1 -- 172.21.15.32:0/1441504335 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd2200fc040 con 0x55c1775a8580 2024-04-25T00:36:07.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.042+0000 7fd219ffb640 1 -- 172.21.15.32:0/1441504335 <== mon.2 v2:172.21.15.121:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fd2200ba5b0 con 0x55c1775a8580 2024-04-25T00:36:07.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.043+0000 7fd219ffb640 1 -- 172.21.15.32:0/1441504335 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7fd2200bbc10 con 0x55c1775a8580 2024-04-25T00:36:07.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.043+0000 7fd23b366740 10 client.25640 fetch_fsmap learned FSMap version 26 2024-04-25T00:36:07.047 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.043+0000 7fd23b366740 1 -- 172.21.15.32:0/1441504335 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x55c1776298c0 con 0x55c1775a8580 2024-04-25T00:36:07.047 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.043+0000 7fd219ffb640 1 -- 172.21.15.32:0/1441504335 <== mon.2 v2:172.21.15.121:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd2200fc040 con 0x55c1775a8580 2024-04-25T00:36:07.047 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.045+0000 7fd219ffb640 1 -- 172.21.15.32:0/1441504335 <== mon.2 v2:172.21.15.121:3300/0 5 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7fd22003a070 con 0x55c1775a8580 2024-04-25T00:36:07.047 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.045+0000 7fd23b366740 10 client.25640 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:36:07.047 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.045+0000 7fd23b366740 10 client.25640 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-25T00:36:07.047 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.045+0000 7fd23b366740 20 client.25640 populate_metadata read hostname 'smithi032' 2024-04-25T00:36:07.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.045+0000 7fd23b366740 1 --2- 172.21.15.32:0/1441504335 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55c177629cf0 0x55c17762c0e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.045+0000 7fd23b366740 4 client.25640 mds_command: new command op to 24481 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-25T00:36:07.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.045+0000 7fd23b366740 1 -- 172.21.15.32:0/1441504335 --> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55c177514f00 con 0x55c177629cf0 2024-04-25T00:36:07.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.047+0000 7fd235bf3640 1 --2- 172.21.15.32:0/1441504335 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55c177629cf0 0x55c17762c0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:07.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.048+0000 7fd219ffb640 10 client.25640 ms_handle_connect on v2:172.21.15.130:6838/4267125809 2024-04-25T00:36:07.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.048+0000 7fd235bf3640 1 --2- 172.21.15.32:0/1441504335 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55c177629cf0 0x55c17762c0e0 crc :-1 s=READY pgs=38 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=4b38df5858bc2cd7 server_cookie=384f38a450e34a1c in_seq=0 out_seq=0 2024-04-25T00:36:07.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.048+0000 7fd219ffb640 1 -- 172.21.15.32:0/1441504335 <== mds.1 v2:172.21.15.130:6838/4267125809 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55c177514f00 con 0x55c177629cf0 2024-04-25T00:36:07.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.048+0000 7fd219ffb640 10 client.25640 handle_command_reply: tid=0 2024-04-25T00:36:07.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.048+0000 7fd23b366740 1 client.25640 shutdown 2024-04-25T00:36:07.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.054+0000 7fd23b366740 1 -- 172.21.15.32:0/1441504335 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c1775a8580 msgr2=0x55c1775bba40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:07.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.054+0000 7fd23b366740 1 --2- 172.21.15.32:0/1441504335 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c1775a8580 0x55c1775bba40 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7fd2200bbfb0 tx=0x7fd2200f7710 comp rx=0 tx=0).stop 2024-04-25T00:36:07.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.055+0000 7fd23b366740 1 -- 172.21.15.32:0/1441504335 shutdown_connections 2024-04-25T00:36:07.055 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.055+0000 7fd23b366740 1 --2- 172.21.15.32:0/1441504335 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x55c177629cf0 0x55c17762c0e0 crc :-1 s=READY pgs=38 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.055+0000 7fd23b366740 1 --2- 172.21.15.32:0/1441504335 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55c1775b3570 0x55c1775ad420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.055+0000 7fd23b366740 1 --2- 172.21.15.32:0/1441504335 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c1775a8ea0 0x55c1775acd00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.055+0000 7fd23b366740 1 --2- 172.21.15.32:0/1441504335 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55c1775a8580 0x55c1775bba40 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.055+0000 7fd23b366740 1 -- 172.21.15.32:0/1441504335 >> 172.21.15.32:0/1441504335 conn(0x55c1775a48a0 msgr2=0x55c1775aa560 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:07.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.055+0000 7fd219ffb640 0 client.25640 ms_handle_reset on v2:172.21.15.130:6838/4267125809 2024-04-25T00:36:07.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.060+0000 7fd23b366740 1 -- 172.21.15.32:0/1441504335 shutdown_connections 2024-04-25T00:36:07.066 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.063+0000 7fd23b366740 1 -- 172.21.15.32:0/1441504335 wait complete. 2024-04-25T00:36:07.066 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.064+0000 7fd23b366740 20 client.25640 trim_cache size 0 max 16384 2024-04-25T00:36:07.067 INFO:teuthology.orchestra.run.smithi032.stdout:[] 2024-04-25T00:36:07.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.069+0000 7fd238658640 1 -- 172.21.15.32:0/1720125441 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd21c0a2f00 msgr2=0x7fd21c0a53c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:07.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.069+0000 7fd238658640 1 --2- 172.21.15.32:0/1720125441 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd21c0a2f00 0x7fd21c0a53c0 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7fd228005fd0 tx=0x7fd2280096e0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.069+0000 7fd238658640 1 -- 172.21.15.32:0/1720125441 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd23010dfc0 msgr2=0x7fd2301a6e40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:07.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.069+0000 7fd238658640 1 --2- 172.21.15.32:0/1720125441 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd23010dfc0 0x7fd2301a6e40 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7fd22002a8d0 tx=0x7fd220033d10 comp rx=0 tx=0).stop 2024-04-25T00:36:07.072 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.072+0000 7fd238658640 1 -- 172.21.15.32:0/1720125441 shutdown_connections 2024-04-25T00:36:07.073 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.072+0000 7fd238658640 1 --2- 172.21.15.32:0/1720125441 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd21c0a2f00 0x7fd21c0a53c0 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.073 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.072+0000 7fd238658640 1 --2- 172.21.15.32:0/1720125441 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd23011af20 0x7fd2301ab700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.073 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.072+0000 7fd238658640 1 --2- 172.21.15.32:0/1720125441 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd23010ff90 0x7fd2301a7380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.073 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.072+0000 7fd238658640 1 --2- 172.21.15.32:0/1720125441 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd23010dfc0 0x7fd2301a6e40 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.073 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.072+0000 7fd238658640 1 -- 172.21.15.32:0/1720125441 >> 172.21.15.32:0/1720125441 conn(0x7fd23006c850 msgr2=0x7fd2301199f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:07.074 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.074+0000 7fd238658640 1 -- 172.21.15.32:0/1720125441 shutdown_connections 2024-04-25T00:36:07.076 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.076+0000 7fd238658640 1 -- 172.21.15.32:0/1720125441 wait complete. 2024-04-25T00:36:07.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:06 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1823055582' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:36:07.126 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-25T00:36:07.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:06 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1823055582' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:36:07.437 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.436+0000 7ffa362b9640 1 -- 172.21.15.32:0/2047511620 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffa3010fde0 msgr2=0x7ffa301101c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:07.437 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.436+0000 7ffa362b9640 1 --2- 172.21.15.32:0/2047511620 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffa3010fde0 0x7ffa301101c0 secure :-1 s=READY pgs=606 cs=0 l=1 rev1=1 crypto rx=0x7ffa2001a910 tx=0x7ffa2003b4e0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.436+0000 7ffa362b9640 1 -- 172.21.15.32:0/2047511620 shutdown_connections 2024-04-25T00:36:07.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.436+0000 7ffa362b9640 1 --2- 172.21.15.32:0/2047511620 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ffa301091c0 0x7ffa3011cf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.436+0000 7ffa362b9640 1 --2- 172.21.15.32:0/2047511620 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffa30108740 0x7ffa30108ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.436+0000 7ffa362b9640 1 --2- 172.21.15.32:0/2047511620 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffa3010fde0 0x7ffa301101c0 unknown :-1 s=CLOSED pgs=606 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.436+0000 7ffa362b9640 1 -- 172.21.15.32:0/2047511620 >> 172.21.15.32:0/2047511620 conn(0x7ffa3006c6c0 msgr2=0x7ffa3006cad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:07.440 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.440+0000 7ffa362b9640 1 -- 172.21.15.32:0/2047511620 shutdown_connections 2024-04-25T00:36:07.444 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.443+0000 7ffa362b9640 1 -- 172.21.15.32:0/2047511620 wait complete. 2024-04-25T00:36:07.444 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.444+0000 7ffa362b9640 1 Processor -- start 2024-04-25T00:36:07.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.444+0000 7ffa362b9640 1 -- start start 2024-04-25T00:36:07.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.444+0000 7ffa362b9640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffa30108740 0x7ffa3011cc60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.444+0000 7ffa362b9640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffa301091c0 0x7ffa30118d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.444+0000 7ffa362b9640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ffa3010fde0 0x7ffa30119280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.444+0000 7ffa362b9640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7ffa30113120 con 0x7ffa30108740 2024-04-25T00:36:07.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.444+0000 7ffa362b9640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7ffa30112fa0 con 0x7ffa3010fde0 2024-04-25T00:36:07.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.444+0000 7ffa362b9640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7ffa301132a0 con 0x7ffa301091c0 2024-04-25T00:36:07.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.444+0000 7ffa35ab8640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ffa3010fde0 0x7ffa30119280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:07.447 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.444+0000 7ffa35ab8640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ffa3010fde0 0x7ffa30119280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:54944/0 (socket says 172.21.15.32:54944) 2024-04-25T00:36:07.448 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.444+0000 7ffa35ab8640 1 -- 172.21.15.32:0/4075867884 learned_addr learned my addr 172.21.15.32:0/4075867884 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:07.448 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.445+0000 7ffa35ab8640 1 -- 172.21.15.32:0/4075867884 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffa301091c0 msgr2=0x7ffa30118d40 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:36:07.448 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.445+0000 7ffa35ab8640 1 --2- 172.21.15.32:0/4075867884 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffa301091c0 0x7ffa30118d40 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.448 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.445+0000 7ffa35ab8640 1 -- 172.21.15.32:0/4075867884 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffa30108740 msgr2=0x7ffa3011cc60 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:36:07.448 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.445+0000 7ffa35ab8640 1 --2- 172.21.15.32:0/4075867884 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffa30108740 0x7ffa3011cc60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.448 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.445+0000 7ffa35ab8640 1 -- 172.21.15.32:0/4075867884 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ffa30119980 con 0x7ffa3010fde0 2024-04-25T00:36:07.448 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.445+0000 7ffa35ab8640 1 --2- 172.21.15.32:0/4075867884 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ffa3010fde0 0x7ffa30119280 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7ffa2800b990 tx=0x7ffa2800be50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:07.448 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.445+0000 7ffa267fc640 1 -- 172.21.15.32:0/4075867884 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa28019070 con 0x7ffa3010fde0 2024-04-25T00:36:07.448 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.445+0000 7ffa267fc640 1 -- 172.21.15.32:0/4075867884 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ffa2800cba0 con 0x7ffa3010fde0 2024-04-25T00:36:07.448 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.445+0000 7ffa267fc640 1 -- 172.21.15.32:0/4075867884 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa280076a0 con 0x7ffa3010fde0 2024-04-25T00:36:07.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.446+0000 7ffa362b9640 1 -- 172.21.15.32:0/4075867884 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ffa301aaba0 con 0x7ffa3010fde0 2024-04-25T00:36:07.449 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.446+0000 7ffa362b9640 1 -- 172.21.15.32:0/4075867884 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ffa301aafb0 con 0x7ffa3010fde0 2024-04-25T00:36:07.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.450+0000 7ffa3a22b740 1 Processor -- start 2024-04-25T00:36:07.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.451+0000 7ffa3a22b740 1 -- start start 2024-04-25T00:36:07.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.451+0000 7ffa3a22b740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ce6da3d720 0x55ce6da3db00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.451+0000 7ffa3a22b740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ce6da3e040 0x55ce6da481d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.451+0000 7ffa3a22b740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ce6da48710 0x55ce6da4ab00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.451+0000 7ffa3a22b740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55ce6d9a2670 con 0x55ce6da48710 2024-04-25T00:36:07.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.451+0000 7ffa3a22b740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55ce6d94a970 con 0x55ce6da3e040 2024-04-25T00:36:07.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.451+0000 7ffa3a22b740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55ce6d99e630 con 0x55ce6da3d720 2024-04-25T00:36:07.453 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.453+0000 7ffa34ab6640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ce6da3d720 0x55ce6da3db00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:07.453 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.453+0000 7ffa34ab6640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ce6da3d720 0x55ce6da3db00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:58172/0 (socket says 172.21.15.32:58172) 2024-04-25T00:36:07.453 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.453+0000 7ffa34ab6640 1 -- 172.21.15.32:0/3957941900 learned_addr learned my addr 172.21.15.32:0/3957941900 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:07.455 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.454+0000 7ffa267fc640 1 -- 172.21.15.32:0/4075867884 <== mon.2 v2:172.21.15.121:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7ffa2801c070 con 0x7ffa3010fde0 2024-04-25T00:36:07.456 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.455+0000 7ffa267fc640 1 --2- 172.21.15.32:0/4075867884 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7ffa1c0a2f00 0x7ffa1c0a53c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.456 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.455+0000 7ffa34ab6640 1 --2- 172.21.15.32:0/3957941900 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ce6da48710 0x55ce6da4ab00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:07.456 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.456+0000 7ffa34ab6640 1 -- 172.21.15.32:0/3957941900 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ce6da3e040 msgr2=0x55ce6da481d0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-25T00:36:07.456 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.456+0000 7ffa34ab6640 1 --2- 172.21.15.32:0/3957941900 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ce6da3e040 0x55ce6da481d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.456 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.456+0000 7ffa34ab6640 1 -- 172.21.15.32:0/3957941900 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ce6da48710 msgr2=0x55ce6da4ab00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:07.456 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.456+0000 7ffa34ab6640 1 --2- 172.21.15.32:0/3957941900 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ce6da48710 0x55ce6da4ab00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.457 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.456+0000 7ffa34ab6640 1 -- 172.21.15.32:0/3957941900 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55ce6d96e400 con 0x55ce6da3d720 2024-04-25T00:36:07.457 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.456+0000 7ffa267fc640 1 -- 172.21.15.32:0/4075867884 <== mon.2 v2:172.21.15.121:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7ffa280ceb50 con 0x7ffa3010fde0 2024-04-25T00:36:07.457 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.456+0000 7ffa34ab6640 1 --2- 172.21.15.32:0/3957941900 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ce6da3d720 0x55ce6da3db00 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7ffa2c00cf30 tx=0x7ffa2c00c3a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=bfdc37e52650f6 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:07.457 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.457+0000 7ffa35ab8640 1 --2- 172.21.15.32:0/4075867884 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7ffa1c0a2f00 0x7ffa1c0a53c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:07.457 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.457+0000 7ffa1b7fe640 1 -- 172.21.15.32:0/3957941900 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa2c00ee00 con 0x55ce6da3d720 2024-04-25T00:36:07.457 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.457+0000 7ffa1b7fe640 1 -- 172.21.15.32:0/3957941900 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ffa2c017030 con 0x55ce6da3d720 2024-04-25T00:36:07.458 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.457+0000 7ffa1b7fe640 1 -- 172.21.15.32:0/3957941900 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa2c0118a0 con 0x55ce6da3d720 2024-04-25T00:36:07.458 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.458+0000 7ffa3a22b740 1 -- 172.21.15.32:0/3957941900 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ce6da3d720 msgr2=0x55ce6da3db00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:07.458 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.458+0000 7ffa3a22b740 1 --2- 172.21.15.32:0/3957941900 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ce6da3d720 0x55ce6da3db00 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7ffa2c00cf30 tx=0x7ffa2c00c3a0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.458 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.458+0000 7ffa35ab8640 1 --2- 172.21.15.32:0/4075867884 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7ffa1c0a2f00 0x7ffa1c0a53c0 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7ffa28097740 tx=0x7ffa2800e040 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:07.459 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.458+0000 7ffa3a22b740 1 -- 172.21.15.32:0/3957941900 shutdown_connections 2024-04-25T00:36:07.459 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.458+0000 7ffa3a22b740 1 --2- 172.21.15.32:0/3957941900 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ce6da48710 0x55ce6da4ab00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.460 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.458+0000 7ffa3a22b740 1 --2- 172.21.15.32:0/3957941900 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ce6da3e040 0x55ce6da481d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.460 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.458+0000 7ffa3a22b740 1 --2- 172.21.15.32:0/3957941900 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ce6da3d720 0x55ce6da3db00 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.460 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.458+0000 7ffa3a22b740 1 -- 172.21.15.32:0/3957941900 >> 172.21.15.32:0/3957941900 conn(0x55ce6da39a40 msgr2=0x55ce6da39e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:07.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.462+0000 7ffa3a22b740 1 -- 172.21.15.32:0/3957941900 shutdown_connections 2024-04-25T00:36:07.471 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.470+0000 7ffa3a22b740 1 -- 172.21.15.32:0/3957941900 wait complete. 2024-04-25T00:36:07.472 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.471+0000 7ffa3a22b740 1 -- start start 2024-04-25T00:36:07.472 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.471+0000 7ffa3a22b740 1 Processor -- start 2024-04-25T00:36:07.472 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.472+0000 7ffa3a22b740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ce6da3d720 0x55ce6da481c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.472 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.472+0000 7ffa3a22b740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ce6da48710 0x55ce6da454b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.472 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.472+0000 7ffa3a22b740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ce6da45b50 0x55ce6da54a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.472 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.472+0000 7ffa3a22b740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55ce6da4b500 con 0x55ce6da3d720 2024-04-25T00:36:07.473 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.472+0000 7ffa3a22b740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55ce6d95e050 con 0x55ce6da45b50 2024-04-25T00:36:07.473 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.472+0000 7ffa3a22b740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55ce6da4b680 con 0x55ce6da48710 2024-04-25T00:36:07.473 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.472+0000 7ffa3a22b740 10 client.0 _ll_register_callbacks cb 0x55ce6d6b5be0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:36:07.473 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.473+0000 7ffa34ab6640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ce6da45b50 0x55ce6da54a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:07.473 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.473+0000 7ffa34ab6640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ce6da3d720 0x55ce6da481c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:07.473 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.473+0000 7ffa34ab6640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ce6da3d720 0x55ce6da481c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:58262/0 (socket says 172.21.15.32:58262) 2024-04-25T00:36:07.473 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.473+0000 7ffa34ab6640 1 -- 172.21.15.32:0/1146497292 learned_addr learned my addr 172.21.15.32:0/1146497292 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:07.477 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.477+0000 7ffa352b7640 1 --2- 172.21.15.32:0/1146497292 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ce6da48710 0x55ce6da454b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:07.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.477+0000 7ffa34ab6640 1 -- 172.21.15.32:0/1146497292 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ce6da48710 msgr2=0x55ce6da454b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:07.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.477+0000 7ffa34ab6640 1 --2- 172.21.15.32:0/1146497292 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ce6da48710 0x55ce6da454b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.477+0000 7ffa34ab6640 1 -- 172.21.15.32:0/1146497292 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ce6da3d720 msgr2=0x55ce6da481c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:07.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.477+0000 7ffa34ab6640 1 --2- 172.21.15.32:0/1146497292 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ce6da3d720 0x55ce6da481c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.477+0000 7ffa34ab6640 1 -- 172.21.15.32:0/1146497292 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55ce6d92fdc0 con 0x55ce6da45b50 2024-04-25T00:36:07.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.483+0000 7ffa34ab6640 1 --2- 172.21.15.32:0/1146497292 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ce6da45b50 0x55ce6da54a00 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7ffa2c00cf00 tx=0x7ffa2c011480 comp rx=0 tx=0).ready entity=mon.2 client_cookie=54f3f201635e659 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:07.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.483+0000 7ffa18ff9640 10 client.0 ms_handle_connect on v2:172.21.15.121:3300/0 2024-04-25T00:36:07.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.485+0000 7ffa18ff9640 1 -- 172.21.15.32:0/1146497292 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa2c011af0 con 0x55ce6da45b50 2024-04-25T00:36:07.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.486+0000 7ffa3a22b740 1 -- 172.21.15.32:0/1146497292 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ffa2c017030 con 0x55ce6da45b50 2024-04-25T00:36:07.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.486+0000 7ffa18ff9640 1 -- 172.21.15.32:0/1146497292 <== mon.2 v2:172.21.15.121:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7ffa2c009040 con 0x55ce6da45b50 2024-04-25T00:36:07.490 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.489+0000 7ffa18ff9640 1 -- 172.21.15.32:0/1146497292 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7ffa2c011c90 con 0x55ce6da45b50 2024-04-25T00:36:07.490 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.490+0000 7ffa3a22b740 10 client.25646 fetch_fsmap learned FSMap version 26 2024-04-25T00:36:07.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.490+0000 7ffa3a22b740 1 -- 172.21.15.32:0/1146497292 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x55ce6d6b63f0 con 0x55ce6da45b50 2024-04-25T00:36:07.495 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.495+0000 7ffa18ff9640 1 -- 172.21.15.32:0/1146497292 <== mon.2 v2:172.21.15.121:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ffa2c017030 con 0x55ce6da45b50 2024-04-25T00:36:07.495 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.495+0000 7ffa18ff9640 1 -- 172.21.15.32:0/1146497292 <== mon.2 v2:172.21.15.121:3300/0 5 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7ffa2c00f040 con 0x55ce6da45b50 2024-04-25T00:36:07.496 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.495+0000 7ffa3a22b740 10 client.25646 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:36:07.496 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.495+0000 7ffa3a22b740 10 client.25646 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:36:07.496 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.495+0000 7ffa3a22b740 20 client.25646 populate_metadata read hostname 'smithi032' 2024-04-25T00:36:07.496 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.495+0000 7ffa3a22b740 1 --2- 172.21.15.32:0/1146497292 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55ce6da590d0 0x55ce6da5b4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.496 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.495+0000 7ffa3a22b740 4 client.25646 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:36:07.496 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.495+0000 7ffa3a22b740 1 -- 172.21.15.32:0/1146497292 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55ce6d9afbb0 con 0x55ce6da590d0 2024-04-25T00:36:07.496 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.495+0000 7ffa352b7640 1 --2- 172.21.15.32:0/1146497292 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55ce6da590d0 0x55ce6da5b4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:07.500 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.498+0000 7ffa352b7640 1 --2- 172.21.15.32:0/1146497292 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55ce6da590d0 0x55ce6da5b4f0 crc :-1 s=READY pgs=73 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c0f6fd872b24b472 server_cookie=b0efae904b9f29f4 in_seq=0 out_seq=0 2024-04-25T00:36:07.501 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.498+0000 7ffa18ff9640 10 client.25646 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:07.504 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.502+0000 7ffa18ff9640 1 -- 172.21.15.32:0/1146497292 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x55ce6d9afbb0 con 0x55ce6da590d0 2024-04-25T00:36:07.504 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.502+0000 7ffa18ff9640 10 client.25646 handle_command_reply: tid=0 2024-04-25T00:36:07.504 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.502+0000 7ffa3a22b740 1 client.25646 shutdown 2024-04-25T00:36:07.504 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.503+0000 7ffa3a22b740 1 -- 172.21.15.32:0/1146497292 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ce6da45b50 msgr2=0x55ce6da54a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:07.504 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.503+0000 7ffa3a22b740 1 --2- 172.21.15.32:0/1146497292 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ce6da45b50 0x55ce6da54a00 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7ffa2c00cf00 tx=0x7ffa2c011480 comp rx=0 tx=0).stop 2024-04-25T00:36:07.505 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.504+0000 7ffa3a22b740 1 -- 172.21.15.32:0/1146497292 shutdown_connections 2024-04-25T00:36:07.505 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.504+0000 7ffa3a22b740 1 --2- 172.21.15.32:0/1146497292 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55ce6da590d0 0x55ce6da5b4f0 crc :-1 s=READY pgs=73 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.505 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.504+0000 7ffa3a22b740 1 --2- 172.21.15.32:0/1146497292 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ce6da45b50 0x55ce6da54a00 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.505 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.504+0000 7ffa3a22b740 1 --2- 172.21.15.32:0/1146497292 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ce6da48710 0x55ce6da454b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.505 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.504+0000 7ffa3a22b740 1 --2- 172.21.15.32:0/1146497292 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ce6da3d720 0x55ce6da481c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.505 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.504+0000 7ffa3a22b740 1 -- 172.21.15.32:0/1146497292 >> 172.21.15.32:0/1146497292 conn(0x55ce6da39a40 msgr2=0x55ce6da3f740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:07.505 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.504+0000 7ffa18ff9640 0 client.25646 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:07.505 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.504+0000 7ffa3a22b740 1 -- 172.21.15.32:0/1146497292 shutdown_connections 2024-04-25T00:36:07.505 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.504+0000 7ffa3a22b740 1 -- 172.21.15.32:0/1146497292 wait complete. 2024-04-25T00:36:07.506 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.504+0000 7ffa3a22b740 20 client.25646 trim_cache size 0 max 16384 2024-04-25T00:36:07.535 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.534+0000 7ffa3a22b740 1 Processor -- start 2024-04-25T00:36:07.535 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.534+0000 7ffa3a22b740 1 -- start start 2024-04-25T00:36:07.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.535+0000 7ffa3a22b740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ce6da3d720 0x55ce6da47cf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.535+0000 7ffa3a22b740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ce6da45b50 0x55ce6da60dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.535+0000 7ffa3a22b740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ce6da48710 0x55ce6da61300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.535+0000 7ffa3a22b740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7ffa14012950 con 0x55ce6da48710 2024-04-25T00:36:07.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.535+0000 7ffa352b7640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ce6da45b50 0x55ce6da60dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:07.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.535+0000 7ffa352b7640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ce6da45b50 0x55ce6da60dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:54962/0 (socket says 172.21.15.32:54962) 2024-04-25T00:36:07.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.535+0000 7ffa352b7640 1 -- 172.21.15.32:0/2590257095 learned_addr learned my addr 172.21.15.32:0/2590257095 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:07.537 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.535+0000 7ffa3a22b740 1 -- 172.21.15.32:0/2590257095 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7ffa14010220 con 0x55ce6da45b50 2024-04-25T00:36:07.537 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.535+0000 7ffa3a22b740 1 -- 172.21.15.32:0/2590257095 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7ffa1400e070 con 0x55ce6da3d720 2024-04-25T00:36:07.537 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.536+0000 7ffa34ab6640 1 --2- 172.21.15.32:0/2590257095 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ce6da3d720 0x55ce6da47cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:07.538 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.536+0000 7ffa352b7640 1 -- 172.21.15.32:0/2590257095 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ce6da3d720 msgr2=0x55ce6da47cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:07.538 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.536+0000 7ffa352b7640 1 --2- 172.21.15.32:0/2590257095 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ce6da3d720 0x55ce6da47cf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.538 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.536+0000 7ffa352b7640 1 -- 172.21.15.32:0/2590257095 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ce6da48710 msgr2=0x55ce6da61300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:07.538 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.536+0000 7ffa352b7640 1 --2- 172.21.15.32:0/2590257095 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ce6da48710 0x55ce6da61300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.538 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.536+0000 7ffa352b7640 1 -- 172.21.15.32:0/2590257095 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55ce6d7807d0 con 0x55ce6da45b50 2024-04-25T00:36:07.538 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.536+0000 7ffa352b7640 1 --2- 172.21.15.32:0/2590257095 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ce6da45b50 0x55ce6da60dc0 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x55ce6da3ece0 tx=0x7ffa200049c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c067aa36aa146d1a server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:07.538 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.537+0000 7ffa1affd640 1 -- 172.21.15.32:0/2590257095 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa20046040 con 0x55ce6da45b50 2024-04-25T00:36:07.538 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.537+0000 7ffa1affd640 1 -- 172.21.15.32:0/2590257095 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ffa20005430 con 0x55ce6da45b50 2024-04-25T00:36:07.538 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.537+0000 7ffa1affd640 1 -- 172.21.15.32:0/2590257095 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa20040730 con 0x55ce6da45b50 2024-04-25T00:36:07.539 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.538+0000 7ffa3a22b740 1 -- 172.21.15.32:0/2590257095 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ce6da45b50 msgr2=0x55ce6da60dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:07.539 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.538+0000 7ffa3a22b740 1 --2- 172.21.15.32:0/2590257095 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ce6da45b50 0x55ce6da60dc0 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x55ce6da3ece0 tx=0x7ffa200049c0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.539 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.538+0000 7ffa3a22b740 1 -- 172.21.15.32:0/2590257095 shutdown_connections 2024-04-25T00:36:07.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.538+0000 7ffa3a22b740 1 --2- 172.21.15.32:0/2590257095 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ce6da48710 0x55ce6da61300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.538+0000 7ffa3a22b740 1 --2- 172.21.15.32:0/2590257095 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ce6da45b50 0x55ce6da60dc0 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.539+0000 7ffa3a22b740 1 --2- 172.21.15.32:0/2590257095 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ce6da3d720 0x55ce6da47cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.539+0000 7ffa3a22b740 1 -- 172.21.15.32:0/2590257095 >> 172.21.15.32:0/2590257095 conn(0x55ce6da39a40 msgr2=0x55ce6da38bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:07.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.542+0000 7ffa3a22b740 1 -- 172.21.15.32:0/2590257095 shutdown_connections 2024-04-25T00:36:07.543 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.542+0000 7ffa3a22b740 1 -- 172.21.15.32:0/2590257095 wait complete. 2024-04-25T00:36:07.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.546+0000 7ffa3a22b740 1 -- start start 2024-04-25T00:36:07.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.546+0000 7ffa3a22b740 1 Processor -- start 2024-04-25T00:36:07.548 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.547+0000 7ffa3a22b740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ce6da3d720 0x55ce6da4bb10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.548 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.548+0000 7ffa3a22b740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ce6da45b50 0x55ce6da4c050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.549 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.548+0000 7ffa3a22b740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ce6da48710 0x55ce6dabd620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.549 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.548+0000 7ffa34ab6640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ce6da3d720 0x55ce6da4bb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:07.549 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.548+0000 7ffa34ab6640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ce6da3d720 0x55ce6da4bb10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:54976/0 (socket says 172.21.15.32:54976) 2024-04-25T00:36:07.549 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.548+0000 7ffa34ab6640 1 -- 172.21.15.32:0/3748992431 learned_addr learned my addr 172.21.15.32:0/3748992431 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:07.549 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.548+0000 7ffa3a22b740 1 -- 172.21.15.32:0/3748992431 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7ffa1400e6e0 con 0x55ce6da45b50 2024-04-25T00:36:07.549 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.548+0000 7ffa3a22b740 1 -- 172.21.15.32:0/3748992431 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7ffa1400fb50 con 0x55ce6da3d720 2024-04-25T00:36:07.549 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.548+0000 7ffa3a22b740 1 -- 172.21.15.32:0/3748992431 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7ffa1400edb0 con 0x55ce6da48710 2024-04-25T00:36:07.549 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.549+0000 7ffa34ab6640 1 --2- 172.21.15.32:0/3748992431 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ce6da48710 0x55ce6dabd620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:07.551 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.549+0000 7ffa34ab6640 1 -- 172.21.15.32:0/3748992431 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ce6da48710 msgr2=0x55ce6dabd620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:07.551 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.549+0000 7ffa34ab6640 1 --2- 172.21.15.32:0/3748992431 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ce6da48710 0x55ce6dabd620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.551 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.549+0000 7ffa34ab6640 1 -- 172.21.15.32:0/3748992431 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ce6da45b50 msgr2=0x55ce6da4c050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:07.551 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.549+0000 7ffa34ab6640 1 --2- 172.21.15.32:0/3748992431 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ce6da45b50 0x55ce6da4c050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.551 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.549+0000 7ffa34ab6640 1 -- 172.21.15.32:0/3748992431 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55ce6dabdb90 con 0x55ce6da3d720 2024-04-25T00:36:07.551 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.549+0000 7ffa3a22b740 10 client.0 _ll_register_callbacks cb 0x7ffa2c00cd60 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:36:07.551 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.549+0000 7ffa3a22b740 1 -- 172.21.15.32:0/3748992431 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ffa20005430 con 0x55ce6da3d720 2024-04-25T00:36:07.551 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.549+0000 7ffa34ab6640 1 --2- 172.21.15.32:0/3748992431 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ce6da3d720 0x55ce6da4bb10 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7ffa2c00d010 tx=0x7ffa2c00ccb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6b979cfa2f2904df server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:07.551 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.550+0000 7ffa18ff9640 10 client.25652 ms_handle_connect on v2:172.21.15.121:3300/0 2024-04-25T00:36:07.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.550+0000 7ffa18ff9640 1 -- 172.21.15.32:0/3748992431 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa2c011af0 con 0x55ce6da3d720 2024-04-25T00:36:07.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.550+0000 7ffa18ff9640 1 -- 172.21.15.32:0/3748992431 <== mon.2 v2:172.21.15.121:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7ffa2c009070 con 0x55ce6da3d720 2024-04-25T00:36:07.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.550+0000 7ffa18ff9640 1 -- 172.21.15.32:0/3748992431 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7ffa2c005b00 con 0x55ce6da3d720 2024-04-25T00:36:07.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.550+0000 7ffa3a22b740 10 client.25652 fetch_fsmap learned FSMap version 26 2024-04-25T00:36:07.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.551+0000 7ffa3a22b740 1 -- 172.21.15.32:0/3748992431 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({fsmap=26}) -- 0x55ce6dabed50 con 0x55ce6da3d720 2024-04-25T00:36:07.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.551+0000 7ffa18ff9640 1 -- 172.21.15.32:0/3748992431 <== mon.2 v2:172.21.15.121:3300/0 4 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7ffa2c029040 con 0x55ce6da3d720 2024-04-25T00:36:07.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.551+0000 7ffa3a22b740 10 client.25652 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:36:07.553 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.551+0000 7ffa3a22b740 10 client.25652 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:36:07.553 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.551+0000 7ffa3a22b740 20 client.25652 populate_metadata read hostname 'smithi032' 2024-04-25T00:36:07.553 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.551+0000 7ffa3a22b740 1 --2- 172.21.15.32:0/3748992431 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55ce6dabf3a0 0x55ce6dac1860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:07.553 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.551+0000 7ffa3a22b740 4 client.25652 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-25T00:36:07.553 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.551+0000 7ffa3a22b740 1 -- 172.21.15.32:0/3748992431 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55ce6d9a9f00 con 0x55ce6dabf3a0 2024-04-25T00:36:07.555 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.554+0000 7ffa352b7640 1 --2- 172.21.15.32:0/3748992431 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55ce6dabf3a0 0x55ce6dac1860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:07.561 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.560+0000 7ffa352b7640 1 --2- 172.21.15.32:0/3748992431 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55ce6dabf3a0 0x55ce6dac1860 crc :-1 s=READY pgs=74 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=993a48a91e0a2637 server_cookie=c64fea64380e48d0 in_seq=0 out_seq=0 2024-04-25T00:36:07.561 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.561+0000 7ffa18ff9640 10 client.25652 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:07.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.563+0000 7ffa18ff9640 1 -- 172.21.15.32:0/3748992431 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55ce6d9a9f00 con 0x55ce6dabf3a0 2024-04-25T00:36:07.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.563+0000 7ffa18ff9640 10 client.25652 handle_command_reply: tid=0 2024-04-25T00:36:07.564 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.563+0000 7ffa3a22b740 1 client.25652 shutdown 2024-04-25T00:36:07.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.567+0000 7ffa3a22b740 1 -- 172.21.15.32:0/3748992431 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ce6da3d720 msgr2=0x55ce6da4bb10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:07.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.567+0000 7ffa3a22b740 1 --2- 172.21.15.32:0/3748992431 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ce6da3d720 0x55ce6da4bb10 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7ffa2c00d010 tx=0x7ffa2c00ccb0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.567+0000 7ffa3a22b740 1 -- 172.21.15.32:0/3748992431 shutdown_connections 2024-04-25T00:36:07.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.567+0000 7ffa3a22b740 1 --2- 172.21.15.32:0/3748992431 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55ce6dabf3a0 0x55ce6dac1860 crc :-1 s=READY pgs=74 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.567+0000 7ffa3a22b740 1 --2- 172.21.15.32:0/3748992431 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ce6da48710 0x55ce6dabd620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.567+0000 7ffa3a22b740 1 --2- 172.21.15.32:0/3748992431 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55ce6da45b50 0x55ce6da4c050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.567+0000 7ffa3a22b740 1 --2- 172.21.15.32:0/3748992431 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55ce6da3d720 0x55ce6da4bb10 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.567+0000 7ffa18ff9640 0 client.25652 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:07.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.567+0000 7ffa3a22b740 1 -- 172.21.15.32:0/3748992431 >> 172.21.15.32:0/3748992431 conn(0x55ce6da39a40 msgr2=0x55ce6da46ce0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:07.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.575+0000 7ffa3a22b740 1 -- 172.21.15.32:0/3748992431 shutdown_connections 2024-04-25T00:36:07.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.575+0000 7ffa3a22b740 1 -- 172.21.15.32:0/3748992431 wait complete. 2024-04-25T00:36:07.577 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.575+0000 7ffa3a22b740 20 client.25652 trim_cache size 0 max 16384 2024-04-25T00:36:07.582 INFO:teuthology.orchestra.run.smithi032.stdout:[] 2024-04-25T00:36:07.591 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.590+0000 7ffa362b9640 1 -- 172.21.15.32:0/4075867884 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7ffa1c0a2f00 msgr2=0x7ffa1c0a53c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:07.591 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.590+0000 7ffa362b9640 1 --2- 172.21.15.32:0/4075867884 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7ffa1c0a2f00 0x7ffa1c0a53c0 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7ffa28097740 tx=0x7ffa2800e040 comp rx=0 tx=0).stop 2024-04-25T00:36:07.591 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.590+0000 7ffa362b9640 1 -- 172.21.15.32:0/4075867884 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ffa3010fde0 msgr2=0x7ffa30119280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:07.592 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.590+0000 7ffa362b9640 1 --2- 172.21.15.32:0/4075867884 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ffa3010fde0 0x7ffa30119280 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7ffa2800b990 tx=0x7ffa2800be50 comp rx=0 tx=0).stop 2024-04-25T00:36:07.593 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.591+0000 7ffa362b9640 1 -- 172.21.15.32:0/4075867884 shutdown_connections 2024-04-25T00:36:07.593 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.591+0000 7ffa362b9640 1 --2- 172.21.15.32:0/4075867884 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7ffa1c0a2f00 0x7ffa1c0a53c0 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.593 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.591+0000 7ffa362b9640 1 --2- 172.21.15.32:0/4075867884 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ffa3010fde0 0x7ffa30119280 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.593 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.591+0000 7ffa362b9640 1 --2- 172.21.15.32:0/4075867884 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffa301091c0 0x7ffa30118d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.593 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.591+0000 7ffa362b9640 1 --2- 172.21.15.32:0/4075867884 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ffa30108740 0x7ffa3011cc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:07.593 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.591+0000 7ffa362b9640 1 -- 172.21.15.32:0/4075867884 >> 172.21.15.32:0/4075867884 conn(0x7ffa3006c6c0 msgr2=0x7ffa300702c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:07.594 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.593+0000 7ffa362b9640 1 -- 172.21.15.32:0/4075867884 shutdown_connections 2024-04-25T00:36:07.595 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:07.594+0000 7ffa362b9640 1 -- 172.21.15.32:0/4075867884 wait complete. 2024-04-25T00:36:07.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:07 smithi032 ceph-mon[29357]: pgmap v87: 129 pgs: 129 active+clean; 19 GiB data, 40 GiB used, 1.0 TiB / 1.0 TiB avail; 988 KiB/s rd, 425 MiB/s wr, 1.29k op/s 2024-04-25T00:36:08.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:07 smithi121 ceph-mon[32231]: pgmap v87: 129 pgs: 129 active+clean; 19 GiB data, 40 GiB used, 1.0 TiB / 1.0 TiB avail; 988 KiB/s rd, 425 MiB/s wr, 1.29k op/s 2024-04-25T00:36:08.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:07 smithi130 ceph-mon[32525]: pgmap v87: 129 pgs: 129 active+clean; 19 GiB data, 40 GiB used, 1.0 TiB / 1.0 TiB avail; 988 KiB/s rd, 425 MiB/s wr, 1.29k op/s 2024-04-25T00:36:08.646 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-25T00:36:08.646 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:36:08.647 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-25T00:36:09.014 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.010+0000 7f6f12f61640 1 -- 172.21.15.32:0/1077759958 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6f0c113690 msgr2=0x7f6f0c115a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:09.014 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.010+0000 7f6f12f61640 1 --2- 172.21.15.32:0/1077759958 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6f0c113690 0x7f6f0c115a80 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f6f00009930 tx=0x7f6f0002a480 comp rx=0 tx=0).stop 2024-04-25T00:36:09.014 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.014+0000 7f6f12f61640 1 -- 172.21.15.32:0/1077759958 shutdown_connections 2024-04-25T00:36:09.014 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.014+0000 7f6f12f61640 1 --2- 172.21.15.32:0/1077759958 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6f0c113690 0x7f6f0c115a80 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.015 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.014+0000 7f6f12f61640 1 --2- 172.21.15.32:0/1077759958 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6f0c10aa00 0x7f6f0c113150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.015 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.014+0000 7f6f12f61640 1 --2- 172.21.15.32:0/1077759958 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6f0c10a0e0 0x7f6f0c10a4c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.015 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.014+0000 7f6f12f61640 1 -- 172.21.15.32:0/1077759958 >> 172.21.15.32:0/1077759958 conn(0x7f6f0c06c720 msgr2=0x7f6f0c06cb30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:09.018 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.018+0000 7f6f12f61640 1 -- 172.21.15.32:0/1077759958 shutdown_connections 2024-04-25T00:36:09.019 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.018+0000 7f6f12f61640 1 -- 172.21.15.32:0/1077759958 wait complete. 2024-04-25T00:36:09.019 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.019+0000 7f6f12f61640 1 Processor -- start 2024-04-25T00:36:09.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.021+0000 7f6f12f61640 1 -- start start 2024-04-25T00:36:09.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.021+0000 7f6f12f61640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6f0c10a0e0 0x7f6f0c11e650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:09.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.021+0000 7f6f12f61640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6f0c10aa00 0x7f6f0c11eb90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:09.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.021+0000 7f6f12f61640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6f0c113690 0x7f6f0c119760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:09.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.021+0000 7f6f12f61640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f6f0c1180b0 con 0x7f6f0c10a0e0 2024-04-25T00:36:09.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.021+0000 7f6f12f61640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f6f0c117f30 con 0x7f6f0c113690 2024-04-25T00:36:09.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.021+0000 7f6f12f61640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f6f0c118230 con 0x7f6f0c10aa00 2024-04-25T00:36:09.024 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.023+0000 7f6f0bfff640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6f0c10aa00 0x7f6f0c11eb90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:09.024 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.023+0000 7f6f0bfff640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6f0c10aa00 0x7f6f0c11eb90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:58210/0 (socket says 172.21.15.32:58210) 2024-04-25T00:36:09.024 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.023+0000 7f6f0bfff640 1 -- 172.21.15.32:0/2199577796 learned_addr learned my addr 172.21.15.32:0/2199577796 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:09.024 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.024+0000 7f6f10cfd640 1 --2- 172.21.15.32:0/2199577796 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6f0c10a0e0 0x7f6f0c11e650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:09.026 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.025+0000 7f6f0bfff640 1 -- 172.21.15.32:0/2199577796 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6f0c113690 msgr2=0x7f6f0c119760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:09.026 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.025+0000 7f6f0bfff640 1 --2- 172.21.15.32:0/2199577796 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6f0c113690 0x7f6f0c119760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.026 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.025+0000 7f6f0bfff640 1 -- 172.21.15.32:0/2199577796 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6f0c10a0e0 msgr2=0x7f6f0c11e650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:09.026 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.025+0000 7f6f0bfff640 1 --2- 172.21.15.32:0/2199577796 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6f0c10a0e0 0x7f6f0c11e650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.026 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.025+0000 7f6f0bfff640 1 -- 172.21.15.32:0/2199577796 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6f0c11a020 con 0x7f6f0c10aa00 2024-04-25T00:36:09.028 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.027+0000 7f6f0bfff640 1 --2- 172.21.15.32:0/2199577796 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6f0c10aa00 0x7f6f0c11eb90 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f6efc00b450 tx=0x7f6efc00b910 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:09.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.029+0000 7f6f09ffb640 1 -- 172.21.15.32:0/2199577796 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6efc015020 con 0x7f6f0c10aa00 2024-04-25T00:36:09.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.029+0000 7f6f09ffb640 1 -- 172.21.15.32:0/2199577796 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6efc011d70 con 0x7f6f0c10aa00 2024-04-25T00:36:09.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.029+0000 7f6f09ffb640 1 -- 172.21.15.32:0/2199577796 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6efc00f340 con 0x7f6f0c10aa00 2024-04-25T00:36:09.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.030+0000 7f6f12f61640 1 -- 172.21.15.32:0/2199577796 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6f0c11a310 con 0x7f6f0c10aa00 2024-04-25T00:36:09.030 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.030+0000 7f6f12f61640 1 -- 172.21.15.32:0/2199577796 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f6f0c124540 con 0x7f6f0c10aa00 2024-04-25T00:36:09.036 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.036+0000 7f6f15c6f740 1 Processor -- start 2024-04-25T00:36:09.036 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.036+0000 7f6f15c6f740 1 -- start start 2024-04-25T00:36:09.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.036+0000 7f6f15c6f740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555cd6aa3b70 0x555cd6aa1eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:09.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.036+0000 7f6f15c6f740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555cd6aa2480 0x555cd6aa2900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:09.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.036+0000 7f6f15c6f740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555cd6aad200 0x555cd6aaf5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:09.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.036+0000 7f6f15c6f740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x555cd6a062c0 con 0x555cd6aa3b70 2024-04-25T00:36:09.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.036+0000 7f6f15c6f740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x555cd69ae910 con 0x555cd6aad200 2024-04-25T00:36:09.037 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.036+0000 7f6f15c6f740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x555cd6a02280 con 0x555cd6aa2480 2024-04-25T00:36:09.044 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.042+0000 7f6f0bfff640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555cd6aad200 0x555cd6aaf5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:09.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.043+0000 7f6f114fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555cd6aa3b70 0x555cd6aa1eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:09.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.044+0000 7f6f114fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555cd6aa3b70 0x555cd6aa1eb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:58328/0 (socket says 172.21.15.32:58328) 2024-04-25T00:36:09.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.044+0000 7f6f114fe640 1 -- 172.21.15.32:0/2809771833 learned_addr learned my addr 172.21.15.32:0/2809771833 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:09.045 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.044+0000 7f6f09ffb640 1 -- 172.21.15.32:0/2199577796 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f6efc018020 con 0x7f6f0c10aa00 2024-04-25T00:36:09.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.045+0000 7f6f09ffb640 1 --2- 172.21.15.32:0/2199577796 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f6ef40a2f00 0x7f6ef40a53c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:09.046 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.045+0000 7f6f09ffb640 1 -- 172.21.15.32:0/2199577796 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f6efc0d2640 con 0x7f6f0c10aa00 2024-04-25T00:36:09.047 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.047+0000 7f6f10cfd640 1 --2- 172.21.15.32:0/2809771833 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555cd6aa2480 0x555cd6aa2900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:09.047 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.047+0000 7f6f114fe640 1 --2- 172.21.15.32:0/2199577796 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f6ef40a2f00 0x7f6ef40a53c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:09.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.049+0000 7f6f10cfd640 1 -- 172.21.15.32:0/2809771833 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555cd6aad200 msgr2=0x555cd6aaf5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:09.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.049+0000 7f6f10cfd640 1 --2- 172.21.15.32:0/2809771833 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555cd6aad200 0x555cd6aaf5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.049+0000 7f6f10cfd640 1 -- 172.21.15.32:0/2809771833 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555cd6aa3b70 msgr2=0x555cd6aa1eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:09.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.049+0000 7f6f10cfd640 1 --2- 172.21.15.32:0/2809771833 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555cd6aa3b70 0x555cd6aa1eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.049+0000 7f6f10cfd640 1 -- 172.21.15.32:0/2809771833 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x555cd69bffb0 con 0x555cd6aa2480 2024-04-25T00:36:09.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.051+0000 7f6f10cfd640 1 --2- 172.21.15.32:0/2809771833 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555cd6aa2480 0x555cd6aa2900 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f6f04009ee0 tx=0x7f6f0400c7f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=bea8b34f7088543d server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:09.053 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.052+0000 7f6f114fe640 1 --2- 172.21.15.32:0/2199577796 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f6ef40a2f00 0x7f6ef40a53c0 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f6f000057c0 tx=0x7f6f000095e0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:09.054 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.053+0000 7f6ef2ffd640 1 -- 172.21.15.32:0/2809771833 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6f0401a070 con 0x555cd6aa2480 2024-04-25T00:36:09.054 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.053+0000 7f6ef2ffd640 1 -- 172.21.15.32:0/2809771833 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6f040158f0 con 0x555cd6aa2480 2024-04-25T00:36:09.054 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.053+0000 7f6ef2ffd640 1 -- 172.21.15.32:0/2809771833 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6f0400a9c0 con 0x555cd6aa2480 2024-04-25T00:36:09.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.055+0000 7f6f15c6f740 1 -- 172.21.15.32:0/2809771833 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555cd6aa2480 msgr2=0x555cd6aa2900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:09.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.055+0000 7f6f15c6f740 1 --2- 172.21.15.32:0/2809771833 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555cd6aa2480 0x555cd6aa2900 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f6f04009ee0 tx=0x7f6f0400c7f0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.055+0000 7f6f15c6f740 1 -- 172.21.15.32:0/2809771833 shutdown_connections 2024-04-25T00:36:09.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.055+0000 7f6f15c6f740 1 --2- 172.21.15.32:0/2809771833 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555cd6aad200 0x555cd6aaf5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.055+0000 7f6f15c6f740 1 --2- 172.21.15.32:0/2809771833 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555cd6aa2480 0x555cd6aa2900 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.055+0000 7f6f15c6f740 1 --2- 172.21.15.32:0/2809771833 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555cd6aa3b70 0x555cd6aa1eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.055+0000 7f6f15c6f740 1 -- 172.21.15.32:0/2809771833 >> 172.21.15.32:0/2809771833 conn(0x555cd6a6a950 msgr2=0x555cd6a9e9e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:09.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.058+0000 7f6f15c6f740 1 -- 172.21.15.32:0/2809771833 shutdown_connections 2024-04-25T00:36:09.065 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.064+0000 7f6f15c6f740 1 -- 172.21.15.32:0/2809771833 wait complete. 2024-04-25T00:36:09.065 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.064+0000 7f6f15c6f740 1 -- start start 2024-04-25T00:36:09.065 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.064+0000 7f6f15c6f740 1 Processor -- start 2024-04-25T00:36:09.067 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.065+0000 7f6f15c6f740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555cd6aa2480 0x555cd6aaa180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:09.067 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.066+0000 7f6f15c6f740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555cd6aa3b70 0x555cd6aaa6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:09.067 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.066+0000 7f6f15c6f740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555cd6aad200 0x555cd6aaac00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:09.067 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.066+0000 7f6f15c6f740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x555cd6ab0090 con 0x555cd6aa3b70 2024-04-25T00:36:09.067 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.066+0000 7f6f15c6f740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x555cd69c1e90 con 0x555cd6aad200 2024-04-25T00:36:09.067 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.066+0000 7f6f15c6f740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x555cd6ab0210 con 0x555cd6aa2480 2024-04-25T00:36:09.067 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.066+0000 7f6f15c6f740 10 client.0 _ll_register_callbacks cb 0x555cd69fd970 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:36:09.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.067+0000 7f6f10cfd640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555cd6aa2480 0x555cd6aaa180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:09.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.067+0000 7f6f10cfd640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555cd6aa2480 0x555cd6aaa180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:58228/0 (socket says 172.21.15.32:58228) 2024-04-25T00:36:09.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.067+0000 7f6f10cfd640 1 -- 172.21.15.32:0/3558989110 learned_addr learned my addr 172.21.15.32:0/3558989110 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:09.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.068+0000 7f6f10cfd640 1 -- 172.21.15.32:0/3558989110 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555cd6aad200 msgr2=0x555cd6aaac00 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-25T00:36:09.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.068+0000 7f6f10cfd640 1 --2- 172.21.15.32:0/3558989110 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555cd6aad200 0x555cd6aaac00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.068+0000 7f6f10cfd640 1 -- 172.21.15.32:0/3558989110 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555cd6aa3b70 msgr2=0x555cd6aaa6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:09.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.068+0000 7f6f10cfd640 1 --2- 172.21.15.32:0/3558989110 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555cd6aa3b70 0x555cd6aaa6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.068+0000 7f6f10cfd640 1 -- 172.21.15.32:0/3558989110 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x555cd67e27e0 con 0x555cd6aa2480 2024-04-25T00:36:09.073 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.070+0000 7f6eebfff640 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:36:09.073 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.072+0000 7f6f10cfd640 1 --2- 172.21.15.32:0/3558989110 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555cd6aa2480 0x555cd6aaa180 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f6f04004b90 tx=0x7f6f04015cd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5c35513d6ffd5975 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:09.074 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.073+0000 7f6eebfff640 1 -- 172.21.15.32:0/3558989110 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6f0401a040 con 0x555cd6aa2480 2024-04-25T00:36:09.074 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.074+0000 7f6eebfff640 1 -- 172.21.15.32:0/3558989110 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f6f0400a740 con 0x555cd6aa2480 2024-04-25T00:36:09.077 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.077+0000 7f6f15c6f740 1 -- 172.21.15.32:0/3558989110 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f6f040158f0 con 0x555cd6aa2480 2024-04-25T00:36:09.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.081+0000 7f6eebfff640 1 -- 172.21.15.32:0/3558989110 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7f6f04015e40 con 0x555cd6aa2480 2024-04-25T00:36:09.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.088+0000 7f6f15c6f740 10 client.25636 fetch_fsmap learned FSMap version 26 2024-04-25T00:36:09.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.088+0000 7f6f15c6f740 1 -- 172.21.15.32:0/3558989110 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x555cd6842f70 con 0x555cd6aa2480 2024-04-25T00:36:09.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.094+0000 7f6eebfff640 1 -- 172.21.15.32:0/3558989110 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6f040158f0 con 0x555cd6aa2480 2024-04-25T00:36:09.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.094+0000 7f6eebfff640 1 -- 172.21.15.32:0/3558989110 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7f6f0401d070 con 0x555cd6aa2480 2024-04-25T00:36:09.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.095+0000 7f6f15c6f740 10 client.25636 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:36:09.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.095+0000 7f6f15c6f740 10 client.25636 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:36:09.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.095+0000 7f6f15c6f740 20 client.25636 populate_metadata read hostname 'smithi032' 2024-04-25T00:36:09.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.095+0000 7f6f15c6f740 1 --2- 172.21.15.32:0/3558989110 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x555cd6abd1f0 0x555cd6abf5e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:09.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.095+0000 7f6f15c6f740 4 client.25636 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:36:09.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.095+0000 7f6f15c6f740 1 -- 172.21.15.32:0/3558989110 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x555cd69da5a0 con 0x555cd6abd1f0 2024-04-25T00:36:09.104 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.103+0000 7f6f0bfff640 1 --2- 172.21.15.32:0/3558989110 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x555cd6abd1f0 0x555cd6abf5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:09.130 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.129+0000 7f6eebfff640 10 client.25636 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:09.130 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.129+0000 7f6f0bfff640 1 --2- 172.21.15.32:0/3558989110 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x555cd6abd1f0 0x555cd6abf5e0 crc :-1 s=READY pgs=75 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9905f06645be59a3 server_cookie=d223c0d70a072b30 in_seq=0 out_seq=0 2024-04-25T00:36:09.138 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.137+0000 7f6eebfff640 1 -- 172.21.15.32:0/3558989110 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x555cd69da5a0 con 0x555cd6abd1f0 2024-04-25T00:36:09.138 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.138+0000 7f6eebfff640 10 client.25636 handle_command_reply: tid=0 2024-04-25T00:36:09.143 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.142+0000 7f6f15c6f740 1 client.25636 shutdown 2024-04-25T00:36:09.145 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.145+0000 7f6f15c6f740 1 -- 172.21.15.32:0/3558989110 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555cd6aa2480 msgr2=0x555cd6aaa180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:09.145 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.145+0000 7f6f15c6f740 1 --2- 172.21.15.32:0/3558989110 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555cd6aa2480 0x555cd6aaa180 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f6f04004b90 tx=0x7f6f04015cd0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.151+0000 7f6f15c6f740 1 -- 172.21.15.32:0/3558989110 shutdown_connections 2024-04-25T00:36:09.151 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.151+0000 7f6f15c6f740 1 --2- 172.21.15.32:0/3558989110 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x555cd6abd1f0 0x555cd6abf5e0 crc :-1 s=READY pgs=75 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.151+0000 7f6f15c6f740 1 --2- 172.21.15.32:0/3558989110 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555cd6aad200 0x555cd6aaac00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.151+0000 7f6f15c6f740 1 --2- 172.21.15.32:0/3558989110 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555cd6aa3b70 0x555cd6aaa6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.151+0000 7f6f15c6f740 1 --2- 172.21.15.32:0/3558989110 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555cd6aa2480 0x555cd6aaa180 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.151+0000 7f6f15c6f740 1 -- 172.21.15.32:0/3558989110 >> 172.21.15.32:0/3558989110 conn(0x555cd6a6a950 msgr2=0x555cd6aa4850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:09.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.151+0000 7f6eebfff640 0 client.25636 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:09.155 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.155+0000 7f6f15c6f740 1 -- 172.21.15.32:0/3558989110 shutdown_connections 2024-04-25T00:36:09.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.163+0000 7f6f15c6f740 1 -- 172.21.15.32:0/3558989110 wait complete. 2024-04-25T00:36:09.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.164+0000 7f6f15c6f740 20 client.25636 trim_cache size 0 max 16384 2024-04-25T00:36:09.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.224+0000 7f6f15c6f740 1 Processor -- start 2024-04-25T00:36:09.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.224+0000 7f6f15c6f740 1 -- start start 2024-04-25T00:36:09.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.224+0000 7f6f15c6f740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555cd6aa2480 0x555cd6ab8f90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:09.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.225+0000 7f6f15c6f740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555cd6aa3b70 0x555cd6aa9a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:09.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.225+0000 7f6f15c6f740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555cd6aad200 0x555cd6aaa850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:09.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.225+0000 7f6f15c6f740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f6f0c1c8a60 con 0x555cd6aa2480 2024-04-25T00:36:09.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.225+0000 7f6f15c6f740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f6f0c1c6330 con 0x555cd6aad200 2024-04-25T00:36:09.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.225+0000 7f6f15c6f740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f6f0c1c4180 con 0x555cd6aa3b70 2024-04-25T00:36:09.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.225+0000 7f6f0bfff640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555cd6aa3b70 0x555cd6aa9a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:09.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.225+0000 7f6f0bfff640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555cd6aa3b70 0x555cd6aa9a60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:58240/0 (socket says 172.21.15.32:58240) 2024-04-25T00:36:09.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.225+0000 7f6f0bfff640 1 -- 172.21.15.32:0/1896976643 learned_addr learned my addr 172.21.15.32:0/1896976643 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:09.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.226+0000 7f6f114fe640 1 --2- 172.21.15.32:0/1896976643 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555cd6aad200 0x555cd6aaa850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:09.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.226+0000 7f6f10cfd640 1 --2- 172.21.15.32:0/1896976643 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555cd6aa2480 0x555cd6ab8f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:09.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.226+0000 7f6f0bfff640 1 -- 172.21.15.32:0/1896976643 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555cd6aad200 msgr2=0x555cd6aaa850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:09.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.226+0000 7f6f0bfff640 1 --2- 172.21.15.32:0/1896976643 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555cd6aad200 0x555cd6aaa850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.226+0000 7f6f0bfff640 1 -- 172.21.15.32:0/1896976643 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555cd6aa2480 msgr2=0x555cd6ab8f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:09.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.226+0000 7f6f0bfff640 1 --2- 172.21.15.32:0/1896976643 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555cd6aa2480 0x555cd6ab8f90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.226+0000 7f6f0bfff640 1 -- 172.21.15.32:0/1896976643 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x555cd66da170 con 0x555cd6aa3b70 2024-04-25T00:36:09.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.226+0000 7f6f0bfff640 1 --2- 172.21.15.32:0/1896976643 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555cd6aa3b70 0x555cd6aa9a60 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f6efc09e540 tx=0x7f6efc021000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f75b95345dd6ce8a server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:09.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.226+0000 7f6f10cfd640 1 --2- 172.21.15.32:0/1896976643 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555cd6aa2480 0x555cd6ab8f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:36:09.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.226+0000 7f6ef27fc640 1 -- 172.21.15.32:0/1896976643 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6efc0b7070 con 0x555cd6aa3b70 2024-04-25T00:36:09.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.226+0000 7f6ef27fc640 1 -- 172.21.15.32:0/1896976643 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6efc09e840 con 0x555cd6aa3b70 2024-04-25T00:36:09.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.227+0000 7f6f15c6f740 1 -- 172.21.15.32:0/1896976643 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555cd6aa3b70 msgr2=0x555cd6aa9a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:09.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.227+0000 7f6f15c6f740 1 --2- 172.21.15.32:0/1896976643 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555cd6aa3b70 0x555cd6aa9a60 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f6efc09e540 tx=0x7f6efc021000 comp rx=0 tx=0).stop 2024-04-25T00:36:09.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.232+0000 7f6f15c6f740 1 -- 172.21.15.32:0/1896976643 shutdown_connections 2024-04-25T00:36:09.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.232+0000 7f6f15c6f740 1 --2- 172.21.15.32:0/1896976643 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555cd6aad200 0x555cd6aaa850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.232+0000 7f6f15c6f740 1 --2- 172.21.15.32:0/1896976643 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555cd6aa3b70 0x555cd6aa9a60 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.232+0000 7f6f15c6f740 1 --2- 172.21.15.32:0/1896976643 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555cd6aa2480 0x555cd6ab8f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.232+0000 7f6f15c6f740 1 -- 172.21.15.32:0/1896976643 >> 172.21.15.32:0/1896976643 conn(0x555cd6a6a950 msgr2=0x555cd6aaa140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:09.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.233+0000 7f6f15c6f740 1 -- 172.21.15.32:0/1896976643 shutdown_connections 2024-04-25T00:36:09.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.233+0000 7f6f15c6f740 1 -- 172.21.15.32:0/1896976643 wait complete. 2024-04-25T00:36:09.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.233+0000 7f6f15c6f740 1 -- start start 2024-04-25T00:36:09.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.233+0000 7f6f15c6f740 1 Processor -- start 2024-04-25T00:36:09.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.234+0000 7f6f15c6f740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555cd6aa2480 0x555cd6ab5930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:09.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.234+0000 7f6f15c6f740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555cd6aa3b70 0x555cd6aa6c60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:09.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.234+0000 7f6f15c6f740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555cd6aad200 0x555cd6aa7270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:09.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.234+0000 7f6f15c6f740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f6f0c1c47f0 con 0x555cd6aa2480 2024-04-25T00:36:09.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.234+0000 7f6f15c6f740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f6f0c1c5c60 con 0x555cd6aad200 2024-04-25T00:36:09.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.234+0000 7f6f15c6f740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f6f0c1c4ec0 con 0x555cd6aa3b70 2024-04-25T00:36:09.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.234+0000 7f6f15c6f740 10 client.0 _ll_register_callbacks cb 0x7f6f04018070 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:36:09.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.235+0000 7f6f10cfd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555cd6aa2480 0x555cd6ab5930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:09.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.235+0000 7f6f10cfd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555cd6aa2480 0x555cd6ab5930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:58370/0 (socket says 172.21.15.32:58370) 2024-04-25T00:36:09.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.235+0000 7f6f10cfd640 1 -- 172.21.15.32:0/2241307268 learned_addr learned my addr 172.21.15.32:0/2241307268 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:09.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.236+0000 7f6f10cfd640 1 -- 172.21.15.32:0/2241307268 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555cd6aa3b70 msgr2=0x555cd6aa6c60 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-25T00:36:09.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.236+0000 7f6f10cfd640 1 --2- 172.21.15.32:0/2241307268 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555cd6aa3b70 0x555cd6aa6c60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.236+0000 7f6f10cfd640 1 -- 172.21.15.32:0/2241307268 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555cd6aad200 msgr2=0x555cd6aa7270 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-25T00:36:09.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.236+0000 7f6f10cfd640 1 --2- 172.21.15.32:0/2241307268 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555cd6aad200 0x555cd6aa7270 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:09.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.236+0000 7f6f10cfd640 1 -- 172.21.15.32:0/2241307268 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x555cd6aa7aa0 con 0x555cd6aa2480 2024-04-25T00:36:09.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.236+0000 7f6f10cfd640 1 --2- 172.21.15.32:0/2241307268 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555cd6aa2480 0x555cd6ab5930 secure :-1 s=READY pgs=607 cs=0 l=1 rev1=1 crypto rx=0x7f6f0400cf00 tx=0x7f6f0401ed50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1e85267bf52ef4cd server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:09.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.236+0000 7f6eebfff640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:36:09.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.237+0000 7f6eebfff640 1 -- 172.21.15.32:0/2241307268 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6f0401a040 con 0x555cd6aa2480 2024-04-25T00:36:09.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.237+0000 7f6f15c6f740 1 -- 172.21.15.32:0/2241307268 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f6efc09e840 con 0x555cd6aa2480 2024-04-25T00:36:09.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.237+0000 7f6eebfff640 1 -- 172.21.15.32:0/2241307268 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f6f0401e640 con 0x555cd6aa2480 2024-04-25T00:36:09.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.238+0000 7f6eebfff640 1 -- 172.21.15.32:0/2241307268 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7f6f0400a400 con 0x555cd6aa2480 2024-04-25T00:36:09.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.238+0000 7f6f15c6f740 10 client.16101 fetch_fsmap learned FSMap version 26 2024-04-25T00:36:09.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.238+0000 7f6f15c6f740 1 -- 172.21.15.32:0/2241307268 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x555cd6b455f0 con 0x555cd6aa2480 2024-04-25T00:36:09.240 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.239+0000 7f6eebfff640 1 -- 172.21.15.32:0/2241307268 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6efc09e840 con 0x555cd6aa2480 2024-04-25T00:36:09.241 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.241+0000 7f6eebfff640 1 -- 172.21.15.32:0/2241307268 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7f6f0401d040 con 0x555cd6aa2480 2024-04-25T00:36:09.241 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.241+0000 7f6f15c6f740 10 client.16101 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:36:09.241 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.241+0000 7f6f15c6f740 10 client.16101 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:36:09.241 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.241+0000 7f6f15c6f740 20 client.16101 populate_metadata read hostname 'smithi032' 2024-04-25T00:36:09.242 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.241+0000 7f6f15c6f740 1 --2- 172.21.15.32:0/2241307268 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x555cd6b45dd0 0x555cd6b481f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:09.242 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.241+0000 7f6f15c6f740 4 client.16101 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-25T00:36:09.242 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.241+0000 7f6f15c6f740 1 -- 172.21.15.32:0/2241307268 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) -- 0x555cd6a13800 con 0x555cd6b45dd0 2024-04-25T00:36:09.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.246+0000 7f6f0bfff640 1 --2- 172.21.15.32:0/2241307268 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x555cd6b45dd0 0x555cd6b481f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:09.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.251+0000 7f6f0bfff640 1 --2- 172.21.15.32:0/2241307268 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x555cd6b45dd0 0x555cd6b481f0 crc :-1 s=READY pgs=76 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9c421907fdecc023 server_cookie=21eea6036df02bf8 in_seq=0 out_seq=0 2024-04-25T00:36:09.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:09.251+0000 7f6eebfff640 10 client.16101 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:09.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:09 smithi032 ceph-mon[29357]: pgmap v88: 129 pgs: 129 active+clean; 19 GiB data, 40 GiB used, 1.0 TiB / 1.0 TiB avail; 988 KiB/s rd, 369 MiB/s wr, 1.22k op/s 2024-04-25T00:36:10.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:09 smithi121 ceph-mon[32231]: pgmap v88: 129 pgs: 129 active+clean; 19 GiB data, 40 GiB used, 1.0 TiB / 1.0 TiB avail; 988 KiB/s rd, 369 MiB/s wr, 1.22k op/s 2024-04-25T00:36:10.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:09 smithi130 ceph-mon[32525]: pgmap v88: 129 pgs: 129 active+clean; 19 GiB data, 40 GiB used, 1.0 TiB / 1.0 TiB avail; 988 KiB/s rd, 369 MiB/s wr, 1.22k op/s 2024-04-25T00:36:10.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.360+0000 7f6eebfff640 1 -- 172.21.15.32:0/2241307268 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+110 (crc 0 0 0) 0x555cd6a13800 con 0x555cd6b45dd0 2024-04-25T00:36:10.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.360+0000 7f6eebfff640 10 client.16101 handle_command_reply: tid=0 2024-04-25T00:36:10.362 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.360+0000 7f6f15c6f740 1 client.16101 shutdown 2024-04-25T00:36:10.362 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.361+0000 7f6f15c6f740 1 -- 172.21.15.32:0/2241307268 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555cd6aa2480 msgr2=0x555cd6ab5930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:10.362 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.361+0000 7f6f15c6f740 1 --2- 172.21.15.32:0/2241307268 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555cd6aa2480 0x555cd6ab5930 secure :-1 s=READY pgs=607 cs=0 l=1 rev1=1 crypto rx=0x7f6f0400cf00 tx=0x7f6f0401ed50 comp rx=0 tx=0).stop 2024-04-25T00:36:10.362 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.361+0000 7f6f15c6f740 1 -- 172.21.15.32:0/2241307268 shutdown_connections 2024-04-25T00:36:10.362 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.361+0000 7f6f15c6f740 1 --2- 172.21.15.32:0/2241307268 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x555cd6b45dd0 0x555cd6b481f0 crc :-1 s=READY pgs=76 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.362 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.361+0000 7f6f15c6f740 1 --2- 172.21.15.32:0/2241307268 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x555cd6aad200 0x555cd6aa7270 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.362 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.361+0000 7f6f15c6f740 1 --2- 172.21.15.32:0/2241307268 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555cd6aa3b70 0x555cd6aa6c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.362 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.361+0000 7f6f15c6f740 1 --2- 172.21.15.32:0/2241307268 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x555cd6aa2480 0x555cd6ab5930 unknown :-1 s=CLOSED pgs=607 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.362 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.361+0000 7f6f15c6f740 1 -- 172.21.15.32:0/2241307268 >> 172.21.15.32:0/2241307268 conn(0x555cd6a6a950 msgr2=0x555cd6aa47e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:10.362 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.361+0000 7f6eebfff640 0 client.16101 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:10.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.361+0000 7f6f15c6f740 1 -- 172.21.15.32:0/2241307268 shutdown_connections 2024-04-25T00:36:10.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.361+0000 7f6f15c6f740 1 -- 172.21.15.32:0/2241307268 wait complete. 2024-04-25T00:36:10.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.362+0000 7f6f15c6f740 20 client.16101 trim_cache size 0 max 16384 2024-04-25T00:36:10.367 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:36:10.368 INFO:teuthology.orchestra.run.smithi032.stdout: "return_code": 0, 2024-04-25T00:36:10.368 INFO:teuthology.orchestra.run.smithi032.stdout: "scrub_tag": "f9ea4c9a-574a-4be2-8054-d0cda1e643a1", 2024-04-25T00:36:10.368 INFO:teuthology.orchestra.run.smithi032.stdout: "mode": "asynchronous" 2024-04-25T00:36:10.368 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:36:10.382 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.381+0000 7f6f12f61640 1 -- 172.21.15.32:0/2199577796 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f6ef40a2f00 msgr2=0x7f6ef40a53c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:10.382 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.381+0000 7f6f12f61640 1 --2- 172.21.15.32:0/2199577796 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f6ef40a2f00 0x7f6ef40a53c0 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f6f000057c0 tx=0x7f6f000095e0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.382 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.381+0000 7f6f12f61640 1 -- 172.21.15.32:0/2199577796 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6f0c10aa00 msgr2=0x7f6f0c11eb90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:10.382 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.381+0000 7f6f12f61640 1 --2- 172.21.15.32:0/2199577796 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6f0c10aa00 0x7f6f0c11eb90 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f6efc00b450 tx=0x7f6efc00b910 comp rx=0 tx=0).stop 2024-04-25T00:36:10.383 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.382+0000 7f6f12f61640 1 -- 172.21.15.32:0/2199577796 shutdown_connections 2024-04-25T00:36:10.383 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.382+0000 7f6f12f61640 1 --2- 172.21.15.32:0/2199577796 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f6ef40a2f00 0x7f6ef40a53c0 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.383 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.382+0000 7f6f12f61640 1 --2- 172.21.15.32:0/2199577796 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6f0c113690 0x7f6f0c119760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.383 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.382+0000 7f6f12f61640 1 --2- 172.21.15.32:0/2199577796 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6f0c10aa00 0x7f6f0c11eb90 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.383 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.382+0000 7f6f12f61640 1 --2- 172.21.15.32:0/2199577796 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6f0c10a0e0 0x7f6f0c11e650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.383 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.382+0000 7f6f12f61640 1 -- 172.21.15.32:0/2199577796 >> 172.21.15.32:0/2199577796 conn(0x7f6f0c06c720 msgr2=0x7f6f0c0703a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:10.388 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.387+0000 7f6f12f61640 1 -- 172.21.15.32:0/2199577796 shutdown_connections 2024-04-25T00:36:10.388 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.388+0000 7f6f12f61640 1 -- 172.21.15.32:0/2199577796 wait complete. 2024-04-25T00:36:10.442 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-25T00:36:10.442 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-25T00:36:10.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.731+0000 7f5c41197640 1 -- 172.21.15.32:0/1646197887 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5c3c1091c0 msgr2=0x7f5c3c11cf30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:10.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.731+0000 7f5c41197640 1 --2- 172.21.15.32:0/1646197887 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5c3c1091c0 0x7f5c3c11cf30 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f5c30009900 tx=0x7f5c3002a4d0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.737 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.734+0000 7f5c41197640 1 -- 172.21.15.32:0/1646197887 shutdown_connections 2024-04-25T00:36:10.737 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.734+0000 7f5c41197640 1 --2- 172.21.15.32:0/1646197887 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5c3c1091c0 0x7f5c3c11cf30 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.737 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.734+0000 7f5c41197640 1 --2- 172.21.15.32:0/1646197887 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5c3c108740 0x7f5c3c108ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.737 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.734+0000 7f5c41197640 1 --2- 172.21.15.32:0/1646197887 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5c3c10fde0 0x7f5c3c1101c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.737 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.734+0000 7f5c41197640 1 -- 172.21.15.32:0/1646197887 >> 172.21.15.32:0/1646197887 conn(0x7f5c3c06c6c0 msgr2=0x7f5c3c06cad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:10.739 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.738+0000 7f5c41197640 1 -- 172.21.15.32:0/1646197887 shutdown_connections 2024-04-25T00:36:10.740 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.739+0000 7f5c41197640 1 -- 172.21.15.32:0/1646197887 wait complete. 2024-04-25T00:36:10.744 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.741+0000 7f5c41197640 1 Processor -- start 2024-04-25T00:36:10.744 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.744+0000 7f5c41197640 1 -- start start 2024-04-25T00:36:10.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.744+0000 7f5c41197640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5c3c108740 0x7f5c3c1a6e10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:10.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.744+0000 7f5c41197640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5c3c1091c0 0x7f5c3c1a7350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:10.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.744+0000 7f5c41197640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5c3c10fde0 0x7f5c3c1ab6d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:10.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.744+0000 7f5c41197640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f5c3c1208b0 con 0x7f5c3c10fde0 2024-04-25T00:36:10.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.744+0000 7f5c41197640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f5c3c120730 con 0x7f5c3c108740 2024-04-25T00:36:10.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.744+0000 7f5c41197640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f5c3c120a30 con 0x7f5c3c1091c0 2024-04-25T00:36:10.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.745+0000 7f5c3bfff640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5c3c108740 0x7f5c3c1a6e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:10.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.745+0000 7f5c3bfff640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5c3c108740 0x7f5c3c1a6e10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:55030/0 (socket says 172.21.15.32:55030) 2024-04-25T00:36:10.748 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.745+0000 7f5c3bfff640 1 -- 172.21.15.32:0/3060606247 learned_addr learned my addr 172.21.15.32:0/3060606247 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:10.748 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.745+0000 7f5c3b7fe640 1 --2- 172.21.15.32:0/3060606247 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5c3c1091c0 0x7f5c3c1a7350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:10.748 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.746+0000 7f5c3bfff640 1 -- 172.21.15.32:0/3060606247 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5c3c1091c0 msgr2=0x7f5c3c1a7350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:10.748 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.746+0000 7f5c3bfff640 1 --2- 172.21.15.32:0/3060606247 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5c3c1091c0 0x7f5c3c1a7350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.748 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.746+0000 7f5c3bfff640 1 -- 172.21.15.32:0/3060606247 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5c3c10fde0 msgr2=0x7f5c3c1ab6d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:10.748 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.746+0000 7f5c3bfff640 1 --2- 172.21.15.32:0/3060606247 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5c3c10fde0 0x7f5c3c1ab6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.748 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.746+0000 7f5c3bfff640 1 -- 172.21.15.32:0/3060606247 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5c3c1abe00 con 0x7f5c3c108740 2024-04-25T00:36:10.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.755+0000 7f5c3bfff640 1 --2- 172.21.15.32:0/3060606247 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5c3c108740 0x7f5c3c1a6e10 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f5c3400ba50 tx=0x7f5c3400bf10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:10.763 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.757+0000 7f5c397fa640 1 -- 172.21.15.32:0/3060606247 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5c34019070 con 0x7f5c3c108740 2024-04-25T00:36:10.763 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.757+0000 7f5c41197640 1 -- 172.21.15.32:0/3060606247 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5c3c1ac0f0 con 0x7f5c3c108740 2024-04-25T00:36:10.763 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.757+0000 7f5c41197640 1 -- 172.21.15.32:0/3060606247 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5c3c110700 con 0x7f5c3c108740 2024-04-25T00:36:10.763 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.757+0000 7f5c397fa640 1 -- 172.21.15.32:0/3060606247 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5c3400cbe0 con 0x7f5c3c108740 2024-04-25T00:36:10.763 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.757+0000 7f5c397fa640 1 -- 172.21.15.32:0/3060606247 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5c340076a0 con 0x7f5c3c108740 2024-04-25T00:36:10.763 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.762+0000 7f5c451e4740 1 Processor -- start 2024-04-25T00:36:10.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.763+0000 7f5c451e4740 1 -- start start 2024-04-25T00:36:10.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.764+0000 7f5c451e4740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe696c9800 0x55fe696c9be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:10.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.764+0000 7f5c451e4740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe696ca120 0x55fe696d42b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:10.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.764+0000 7f5c451e4740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe696d47f0 0x55fe696d6be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:10.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.764+0000 7f5c451e4740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55fe6962e6b0 con 0x55fe696ca120 2024-04-25T00:36:10.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.764+0000 7f5c451e4740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55fe695d6880 con 0x55fe696d47f0 2024-04-25T00:36:10.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.764+0000 7f5c451e4740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55fe6962a670 con 0x55fe696c9800 2024-04-25T00:36:10.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.764+0000 7f5c40996640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe696c9800 0x55fe696c9be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:10.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.764+0000 7f5c40996640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe696c9800 0x55fe696c9be0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:58268/0 (socket says 172.21.15.32:58268) 2024-04-25T00:36:10.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.764+0000 7f5c40996640 1 -- 172.21.15.32:0/3056657410 learned_addr learned my addr 172.21.15.32:0/3056657410 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:10.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.765+0000 7f5c40996640 1 -- 172.21.15.32:0/3056657410 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe696d47f0 msgr2=0x55fe696d6be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:10.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.765+0000 7f5c3b7fe640 1 --2- 172.21.15.32:0/3056657410 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe696d47f0 0x55fe696d6be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:10.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.765+0000 7f5c40996640 1 --2- 172.21.15.32:0/3056657410 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe696d47f0 0x55fe696d6be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.765+0000 7f5c40996640 1 -- 172.21.15.32:0/3056657410 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe696ca120 msgr2=0x55fe696d42b0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-25T00:36:10.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.765+0000 7f5c40996640 1 --2- 172.21.15.32:0/3056657410 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe696ca120 0x55fe696d42b0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.765+0000 7f5c40996640 1 -- 172.21.15.32:0/3056657410 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55fe6940c7d0 con 0x55fe696c9800 2024-04-25T00:36:10.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.765+0000 7f5c3b7fe640 1 --2- 172.21.15.32:0/3056657410 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe696d47f0 0x55fe696d6be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:36:10.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.765+0000 7f5c40996640 1 --2- 172.21.15.32:0/3056657410 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe696c9800 0x55fe696c9be0 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f5c30005c20 tx=0x7f5c300040f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5661a516efb5dcda server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:10.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.767+0000 7f5c397fa640 1 -- 172.21.15.32:0/3060606247 <== mon.2 v2:172.21.15.121:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f5c3401c070 con 0x7f5c3c108740 2024-04-25T00:36:10.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.768+0000 7f5c397fa640 1 --2- 172.21.15.32:0/3060606247 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f5c240a2e80 0x7f5c240a5340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:10.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.769+0000 7f5c397fa640 1 -- 172.21.15.32:0/3060606247 <== mon.2 v2:172.21.15.121:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f5c340cebc0 con 0x7f5c3c108740 2024-04-25T00:36:10.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.769+0000 7f5c227fc640 1 -- 172.21.15.32:0/3056657410 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5c3003e030 con 0x55fe696c9800 2024-04-25T00:36:10.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.769+0000 7f5c227fc640 1 -- 172.21.15.32:0/3056657410 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5c30004260 con 0x55fe696c9800 2024-04-25T00:36:10.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.770+0000 7f5c451e4740 1 -- 172.21.15.32:0/3056657410 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe696c9800 msgr2=0x55fe696c9be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:10.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.770+0000 7f5c451e4740 1 --2- 172.21.15.32:0/3056657410 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe696c9800 0x55fe696c9be0 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f5c30005c20 tx=0x7f5c300040f0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.771+0000 7f5c3bfff640 1 --2- 172.21.15.32:0/3060606247 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f5c240a2e80 0x7f5c240a5340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:10.772 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.772+0000 7f5c451e4740 1 -- 172.21.15.32:0/3056657410 shutdown_connections 2024-04-25T00:36:10.772 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.772+0000 7f5c451e4740 1 --2- 172.21.15.32:0/3056657410 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe696d47f0 0x55fe696d6be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.772 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.772+0000 7f5c451e4740 1 --2- 172.21.15.32:0/3056657410 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe696ca120 0x55fe696d42b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.772 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.772+0000 7f5c451e4740 1 --2- 172.21.15.32:0/3056657410 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe696c9800 0x55fe696c9be0 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.772 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.772+0000 7f5c451e4740 1 -- 172.21.15.32:0/3056657410 >> 172.21.15.32:0/3056657410 conn(0x55fe696c5b20 msgr2=0x55fe696c5f00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:10.774 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.772+0000 7f5c451e4740 1 -- 172.21.15.32:0/3056657410 shutdown_connections 2024-04-25T00:36:10.774 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.773+0000 7f5c451e4740 1 -- 172.21.15.32:0/3056657410 wait complete. 2024-04-25T00:36:10.776 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.774+0000 7f5c451e4740 1 -- start start 2024-04-25T00:36:10.776 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.774+0000 7f5c451e4740 1 Processor -- start 2024-04-25T00:36:10.776 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.775+0000 7f5c451e4740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe696c9800 0x55fe696d42b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:10.776 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.775+0000 7f5c451e4740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe696ca120 0x55fe696d1640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:10.776 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.775+0000 7f5c451e4740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe696d47f0 0x55fe696d1d80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:10.776 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.775+0000 7f5c451e4740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55fe696d75d0 con 0x55fe696d47f0 2024-04-25T00:36:10.776 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.775+0000 7f5c451e4740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55fe695e9f50 con 0x55fe696ca120 2024-04-25T00:36:10.776 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.775+0000 7f5c451e4740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55fe696d7750 con 0x55fe696c9800 2024-04-25T00:36:10.776 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.775+0000 7f5c451e4740 10 client.0 _ll_register_callbacks cb 0x55fe69625000 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:36:10.777 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.775+0000 7f5c40996640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe696c9800 0x55fe696d42b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:10.777 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.775+0000 7f5c40996640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe696c9800 0x55fe696d42b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:58276/0 (socket says 172.21.15.32:58276) 2024-04-25T00:36:10.777 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.775+0000 7f5c40996640 1 -- 172.21.15.32:0/1790246583 learned_addr learned my addr 172.21.15.32:0/1790246583 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:10.777 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.776+0000 7f5c3b7fe640 1 --2- 172.21.15.32:0/1790246583 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe696ca120 0x55fe696d1640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:10.778 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.776+0000 7f5c40996640 1 --2- 172.21.15.32:0/1790246583 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe696d47f0 0x55fe696d1d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:10.778 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.776+0000 7f5c40996640 1 -- 172.21.15.32:0/1790246583 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe696ca120 msgr2=0x55fe696d1640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:10.778 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.776+0000 7f5c40996640 1 --2- 172.21.15.32:0/1790246583 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe696ca120 0x55fe696d1640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.776+0000 7f5c40996640 1 -- 172.21.15.32:0/1790246583 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe696d47f0 msgr2=0x55fe696d1d80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:10.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.776+0000 7f5c40996640 1 --2- 172.21.15.32:0/1790246583 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe696d47f0 0x55fe696d1d80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.776+0000 7f5c40996640 1 -- 172.21.15.32:0/1790246583 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55fe693423f0 con 0x55fe696c9800 2024-04-25T00:36:10.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.776+0000 7f5c40996640 1 --2- 172.21.15.32:0/1790246583 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe696c9800 0x55fe696d42b0 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f5c30009850 tx=0x7f5c30031650 comp rx=0 tx=0).ready entity=mon.1 client_cookie=506041dbfd87f6a3 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:10.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.778+0000 7f5c3bfff640 1 --2- 172.21.15.32:0/3060606247 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f5c240a2e80 0x7f5c240a5340 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x55fe696cadc0 tx=0x7f5c3400e040 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:10.780 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.778+0000 7f5c1b7fe640 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:36:10.780 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.778+0000 7f5c1b7fe640 1 -- 172.21.15.32:0/1790246583 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5c3004f060 con 0x55fe696c9800 2024-04-25T00:36:10.780 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.779+0000 7f5c1b7fe640 1 -- 172.21.15.32:0/1790246583 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f5c30003a00 con 0x55fe696c9800 2024-04-25T00:36:10.780 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.779+0000 7f5c451e4740 1 -- 172.21.15.32:0/1790246583 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5c30004260 con 0x55fe696c9800 2024-04-25T00:36:10.780 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.779+0000 7f5c1b7fe640 1 -- 172.21.15.32:0/1790246583 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7f5c3003e050 con 0x55fe696c9800 2024-04-25T00:36:10.781 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.781+0000 7f5c451e4740 10 client.25651 fetch_fsmap learned FSMap version 26 2024-04-25T00:36:10.781 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.781+0000 7f5c451e4740 1 -- 172.21.15.32:0/1790246583 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x55fe693020f0 con 0x55fe696c9800 2024-04-25T00:36:10.782 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.781+0000 7f5c1b7fe640 1 -- 172.21.15.32:0/1790246583 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5c30004260 con 0x55fe696c9800 2024-04-25T00:36:10.782 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.782+0000 7f5c1b7fe640 1 -- 172.21.15.32:0/1790246583 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7f5c3003a3f0 con 0x55fe696c9800 2024-04-25T00:36:10.782 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.782+0000 7f5c451e4740 10 client.25651 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:36:10.782 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.782+0000 7f5c451e4740 10 client.25651 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:36:10.782 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.782+0000 7f5c451e4740 20 client.25651 populate_metadata read hostname 'smithi032' 2024-04-25T00:36:10.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.782+0000 7f5c451e4740 1 --2- 172.21.15.32:0/1790246583 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55fe696e4a60 0x55fe696e6ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:10.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.782+0000 7f5c451e4740 4 client.25651 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:36:10.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.782+0000 7f5c451e4740 1 -- 172.21.15.32:0/1790246583 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55fe6963bbf0 con 0x55fe696e4a60 2024-04-25T00:36:10.784 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.783+0000 7f5c3b7fe640 1 --2- 172.21.15.32:0/1790246583 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55fe696e4a60 0x55fe696e6ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:10.788 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.788+0000 7f5c3b7fe640 1 --2- 172.21.15.32:0/1790246583 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55fe696e4a60 0x55fe696e6ea0 crc :-1 s=READY pgs=77 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b93106ed9eba95f3 server_cookie=615516e7ad52c180 in_seq=0 out_seq=0 2024-04-25T00:36:10.788 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.788+0000 7f5c1b7fe640 10 client.25651 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:10.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.793+0000 7f5c1b7fe640 1 -- 172.21.15.32:0/1790246583 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x55fe6963bbf0 con 0x55fe696e4a60 2024-04-25T00:36:10.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.793+0000 7f5c1b7fe640 10 client.25651 handle_command_reply: tid=0 2024-04-25T00:36:10.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.793+0000 7f5c451e4740 1 client.25651 shutdown 2024-04-25T00:36:10.794 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.793+0000 7f5c451e4740 1 -- 172.21.15.32:0/1790246583 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe696c9800 msgr2=0x55fe696d42b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:10.794 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.793+0000 7f5c451e4740 1 --2- 172.21.15.32:0/1790246583 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe696c9800 0x55fe696d42b0 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f5c30009850 tx=0x7f5c30031650 comp rx=0 tx=0).stop 2024-04-25T00:36:10.794 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.793+0000 7f5c451e4740 1 -- 172.21.15.32:0/1790246583 shutdown_connections 2024-04-25T00:36:10.794 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.793+0000 7f5c451e4740 1 --2- 172.21.15.32:0/1790246583 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55fe696e4a60 0x55fe696e6ea0 crc :-1 s=READY pgs=77 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.794 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.793+0000 7f5c451e4740 1 --2- 172.21.15.32:0/1790246583 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe696d47f0 0x55fe696d1d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.794 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.793+0000 7f5c451e4740 1 --2- 172.21.15.32:0/1790246583 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe696ca120 0x55fe696d1640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.794 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.793+0000 7f5c451e4740 1 --2- 172.21.15.32:0/1790246583 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe696c9800 0x55fe696d42b0 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.794 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.793+0000 7f5c451e4740 1 -- 172.21.15.32:0/1790246583 >> 172.21.15.32:0/1790246583 conn(0x55fe696c5b20 msgr2=0x55fe696cb820 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:10.794 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.793+0000 7f5c1b7fe640 0 client.25651 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:10.794 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.793+0000 7f5c451e4740 1 -- 172.21.15.32:0/1790246583 shutdown_connections 2024-04-25T00:36:10.794 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.793+0000 7f5c451e4740 1 -- 172.21.15.32:0/1790246583 wait complete. 2024-04-25T00:36:10.796 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.794+0000 7f5c451e4740 20 client.25651 trim_cache size 0 max 16384 2024-04-25T00:36:10.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.859+0000 7f5c451e4740 1 Processor -- start 2024-04-25T00:36:10.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.859+0000 7f5c451e4740 1 -- start start 2024-04-25T00:36:10.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.860+0000 7f5c451e4740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe696c9800 0x55fe696e0570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:10.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.860+0000 7f5c451e4740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe696ca120 0x55fe696d3d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:10.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.860+0000 7f5c451e4740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe696d47f0 0x55fe696d42b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:10.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.865+0000 7f5c3b7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe696ca120 0x55fe696d3d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:10.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.865+0000 7f5c3b7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe696ca120 0x55fe696d3d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:58382/0 (socket says 172.21.15.32:58382) 2024-04-25T00:36:10.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.865+0000 7f5c40996640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe696c9800 0x55fe696e0570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:10.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.865+0000 7f5c40996640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe696c9800 0x55fe696e0570 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:55074/0 (socket says 172.21.15.32:55074) 2024-04-25T00:36:10.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.865+0000 7f5c3b7fe640 1 -- 172.21.15.32:0/3961277872 learned_addr learned my addr 172.21.15.32:0/3961277872 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:10.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.865+0000 7f5c451e4740 1 -- 172.21.15.32:0/3961277872 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f5c28012940 con 0x55fe696ca120 2024-04-25T00:36:10.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.865+0000 7f5c451e4740 1 -- 172.21.15.32:0/3961277872 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f5c28010210 con 0x55fe696c9800 2024-04-25T00:36:10.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.865+0000 7f5c451e4740 1 -- 172.21.15.32:0/3961277872 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f5c2800e060 con 0x55fe696d47f0 2024-04-25T00:36:10.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.869+0000 7f5c40996640 1 --2- 172.21.15.32:0/3961277872 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe696d47f0 0x55fe696d42b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:10.872 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.871+0000 7f5c40996640 1 -- 172.21.15.32:0/3961277872 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe696d47f0 msgr2=0x55fe696d42b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:10.872 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.871+0000 7f5c40996640 1 --2- 172.21.15.32:0/3961277872 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe696d47f0 0x55fe696d42b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.872 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.871+0000 7f5c40996640 1 -- 172.21.15.32:0/3961277872 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe696ca120 msgr2=0x55fe696d3d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:10.872 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.871+0000 7f5c40996640 1 --2- 172.21.15.32:0/3961277872 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe696ca120 0x55fe696d3d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.872 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.871+0000 7f5c40996640 1 -- 172.21.15.32:0/3961277872 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55fe696ccfb0 con 0x55fe696c9800 2024-04-25T00:36:10.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.877+0000 7f5c40996640 1 --2- 172.21.15.32:0/3961277872 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe696c9800 0x55fe696e0570 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f5c3003a960 tx=0x7f5c300040f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1d53c309d704dea0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:10.879 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.878+0000 7f5c21ffb640 1 -- 172.21.15.32:0/3961277872 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5c30061070 con 0x55fe696c9800 2024-04-25T00:36:10.879 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.878+0000 7f5c21ffb640 1 -- 172.21.15.32:0/3961277872 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5c3003f040 con 0x55fe696c9800 2024-04-25T00:36:10.879 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.878+0000 7f5c21ffb640 1 -- 172.21.15.32:0/3961277872 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5c3004f070 con 0x55fe696c9800 2024-04-25T00:36:10.880 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.879+0000 7f5c451e4740 1 -- 172.21.15.32:0/3961277872 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe696c9800 msgr2=0x55fe696e0570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:10.881 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.880+0000 7f5c451e4740 1 --2- 172.21.15.32:0/3961277872 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe696c9800 0x55fe696e0570 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f5c3003a960 tx=0x7f5c300040f0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.881 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.880+0000 7f5c451e4740 1 -- 172.21.15.32:0/3961277872 shutdown_connections 2024-04-25T00:36:10.881 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.880+0000 7f5c451e4740 1 --2- 172.21.15.32:0/3961277872 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe696d47f0 0x55fe696d42b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.881 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.880+0000 7f5c451e4740 1 --2- 172.21.15.32:0/3961277872 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe696ca120 0x55fe696d3d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.881 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.880+0000 7f5c451e4740 1 --2- 172.21.15.32:0/3961277872 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe696c9800 0x55fe696e0570 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.881 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.880+0000 7f5c451e4740 1 -- 172.21.15.32:0/3961277872 >> 172.21.15.32:0/3961277872 conn(0x55fe696c5b20 msgr2=0x55fe696c5f00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:10.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.888+0000 7f5c451e4740 1 -- 172.21.15.32:0/3961277872 shutdown_connections 2024-04-25T00:36:10.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.888+0000 7f5c451e4740 1 -- 172.21.15.32:0/3961277872 wait complete. 2024-04-25T00:36:10.890 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.889+0000 7f5c451e4740 1 -- start start 2024-04-25T00:36:10.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.890+0000 7f5c451e4740 1 Processor -- start 2024-04-25T00:36:10.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.892+0000 7f5c451e4740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe696c9800 0x55fe696dccf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:10.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.892+0000 7f5c451e4740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe696ca120 0x55fe696cede0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:10.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.893+0000 7f5c451e4740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe696d47f0 0x55fe696cf320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:10.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.893+0000 7f5c451e4740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f5c2800e6d0 con 0x55fe696ca120 2024-04-25T00:36:10.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.893+0000 7f5c451e4740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f5c2800fb40 con 0x55fe696d47f0 2024-04-25T00:36:10.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.893+0000 7f5c451e4740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f5c2800eda0 con 0x55fe696c9800 2024-04-25T00:36:10.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.894+0000 7f5c451e4740 10 client.0 _ll_register_callbacks cb 0x7f5c30004040 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:36:10.896 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.896+0000 7f5c3b7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe696ca120 0x55fe696cede0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:10.896 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.896+0000 7f5c3b7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe696ca120 0x55fe696cede0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:58398/0 (socket says 172.21.15.32:58398) 2024-04-25T00:36:10.896 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.896+0000 7f5c3b7fe640 1 -- 172.21.15.32:0/1571470582 learned_addr learned my addr 172.21.15.32:0/1571470582 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:10.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.897+0000 7f5c40996640 1 --2- 172.21.15.32:0/1571470582 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe696d47f0 0x55fe696cf320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:10.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.898+0000 7f5c40996640 1 --2- 172.21.15.32:0/1571470582 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe696c9800 0x55fe696dccf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:10.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.901+0000 7f5c40996640 1 -- 172.21.15.32:0/1571470582 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe696c9800 msgr2=0x55fe696dccf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:10.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.901+0000 7f5c40996640 1 --2- 172.21.15.32:0/1571470582 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe696c9800 0x55fe696dccf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.901+0000 7f5c40996640 1 -- 172.21.15.32:0/1571470582 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe696ca120 msgr2=0x55fe696cede0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:10.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.901+0000 7f5c40996640 1 --2- 172.21.15.32:0/1571470582 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe696ca120 0x55fe696cede0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.901+0000 7f5c40996640 1 -- 172.21.15.32:0/1571470582 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55fe696cd990 con 0x55fe696d47f0 2024-04-25T00:36:10.908 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.907+0000 7f5c40996640 1 --2- 172.21.15.32:0/1571470582 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe696d47f0 0x55fe696cf320 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f5c3003a960 tx=0x7f5c3003aca0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7630483e5eee9f0b server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:10.909 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.908+0000 7f5c1b7fe640 10 client.0 ms_handle_connect on v2:172.21.15.121:3300/0 2024-04-25T00:36:10.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.909+0000 7f5c1b7fe640 1 -- 172.21.15.32:0/1571470582 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5c30061070 con 0x55fe696d47f0 2024-04-25T00:36:10.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.910+0000 7f5c1b7fe640 1 -- 172.21.15.32:0/1571470582 <== mon.2 v2:172.21.15.121:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f5c30031a10 con 0x55fe696d47f0 2024-04-25T00:36:10.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.910+0000 7f5c451e4740 1 -- 172.21.15.32:0/1571470582 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5c3003f040 con 0x55fe696d47f0 2024-04-25T00:36:10.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.913+0000 7f5c1b7fe640 1 -- 172.21.15.32:0/1571470582 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7f5c3004f050 con 0x55fe696d47f0 2024-04-25T00:36:10.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.914+0000 7f5c451e4740 10 client.25667 fetch_fsmap learned FSMap version 26 2024-04-25T00:36:10.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.914+0000 7f5c451e4740 1 -- 172.21.15.32:0/1571470582 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x55fe696dbe60 con 0x55fe696d47f0 2024-04-25T00:36:10.917 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.917+0000 7f5c1b7fe640 1 -- 172.21.15.32:0/1571470582 <== mon.2 v2:172.21.15.121:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5c30062040 con 0x55fe696d47f0 2024-04-25T00:36:10.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.918+0000 7f5c1b7fe640 1 -- 172.21.15.32:0/1571470582 <== mon.2 v2:172.21.15.121:3300/0 5 ==== fsmap(e 26) ==== 8350+0+0 (secure 0 0 0) 0x7f5c30054420 con 0x55fe696d47f0 2024-04-25T00:36:10.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.919+0000 7f5c451e4740 10 client.25667 fetch_fsmap finished waiting for FSMap version 26 2024-04-25T00:36:10.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.919+0000 7f5c451e4740 10 client.25667 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:36:10.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.919+0000 7f5c451e4740 20 client.25667 populate_metadata read hostname 'smithi032' 2024-04-25T00:36:10.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.919+0000 7f5c451e4740 1 --2- 172.21.15.32:0/1571470582 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55fe696dc590 0x55fe6974d030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:10.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.919+0000 7f5c451e4740 4 client.25667 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-25T00:36:10.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.919+0000 7f5c451e4740 1 -- 172.21.15.32:0/1571470582 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "scrub status"}) -- 0x7f5c3003f040 con 0x55fe696dc590 2024-04-25T00:36:10.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.921+0000 7f5c3b7fe640 1 --2- 172.21.15.32:0/1571470582 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55fe696dc590 0x55fe6974d030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:10.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.925+0000 7f5c3b7fe640 1 --2- 172.21.15.32:0/1571470582 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55fe696dc590 0x55fe6974d030 crc :-1 s=READY pgs=78 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3815b6f152e53323 server_cookie=7f3beb1a70a24fea in_seq=0 out_seq=0 2024-04-25T00:36:10.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.926+0000 7f5c1b7fe640 10 client.25667 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:10.931 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.931+0000 7f5c1b7fe640 1 -- 172.21.15.32:0/1571470582 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+265 (crc 0 0 0) 0x7f5c3003f040 con 0x55fe696dc590 2024-04-25T00:36:10.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.931+0000 7f5c1b7fe640 10 client.25667 handle_command_reply: tid=0 2024-04-25T00:36:10.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.931+0000 7f5c451e4740 1 client.25667 shutdown 2024-04-25T00:36:10.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.932+0000 7f5c451e4740 1 -- 172.21.15.32:0/1571470582 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe696d47f0 msgr2=0x55fe696cf320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:10.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.932+0000 7f5c451e4740 1 --2- 172.21.15.32:0/1571470582 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe696d47f0 0x55fe696cf320 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f5c3003a960 tx=0x7f5c3003aca0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.935 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.934+0000 7f5c451e4740 1 -- 172.21.15.32:0/1571470582 shutdown_connections 2024-04-25T00:36:10.935 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.934+0000 7f5c451e4740 1 --2- 172.21.15.32:0/1571470582 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55fe696dc590 0x55fe6974d030 crc :-1 s=READY pgs=78 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.935 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.935+0000 7f5c451e4740 1 --2- 172.21.15.32:0/1571470582 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55fe696d47f0 0x55fe696cf320 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.935 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.935+0000 7f5c1b7fe640 0 client.25667 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:10.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.935+0000 7f5c451e4740 1 --2- 172.21.15.32:0/1571470582 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55fe696ca120 0x55fe696cede0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.935+0000 7f5c451e4740 1 --2- 172.21.15.32:0/1571470582 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fe696c9800 0x55fe696dccf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.935+0000 7f5c451e4740 1 -- 172.21.15.32:0/1571470582 >> 172.21.15.32:0/1571470582 conn(0x55fe696c5b20 msgr2=0x55fe696cfed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:10.942 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.942+0000 7f5c451e4740 1 -- 172.21.15.32:0/1571470582 shutdown_connections 2024-04-25T00:36:10.942 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.942+0000 7f5c451e4740 1 -- 172.21.15.32:0/1571470582 wait complete. 2024-04-25T00:36:10.943 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.943+0000 7f5c451e4740 20 client.25667 trim_cache size 0 max 16384 2024-04-25T00:36:10.950 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:36:10.950 INFO:teuthology.orchestra.run.smithi032.stdout: "status": "scrub active (3 inodes in the stack)", 2024-04-25T00:36:10.950 INFO:teuthology.orchestra.run.smithi032.stdout: "scrubs": { 2024-04-25T00:36:10.950 INFO:teuthology.orchestra.run.smithi032.stdout: "f9ea4c9a-574a-4be2-8054-d0cda1e643a1": { 2024-04-25T00:36:10.950 INFO:teuthology.orchestra.run.smithi032.stdout: "path": "/", 2024-04-25T00:36:10.950 INFO:teuthology.orchestra.run.smithi032.stdout: "tag": "f9ea4c9a-574a-4be2-8054-d0cda1e643a1", 2024-04-25T00:36:10.950 INFO:teuthology.orchestra.run.smithi032.stdout: "options": "recursive,force" 2024-04-25T00:36:10.950 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:36:10.950 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:36:10.950 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:36:10.957 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.954+0000 7f5c41197640 1 -- 172.21.15.32:0/3060606247 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f5c240a2e80 msgr2=0x7f5c240a5340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:10.958 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.954+0000 7f5c41197640 1 --2- 172.21.15.32:0/3060606247 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f5c240a2e80 0x7f5c240a5340 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x55fe696cadc0 tx=0x7f5c3400e040 comp rx=0 tx=0).stop 2024-04-25T00:36:10.958 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.954+0000 7f5c41197640 1 -- 172.21.15.32:0/3060606247 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5c3c108740 msgr2=0x7f5c3c1a6e10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:10.958 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.954+0000 7f5c41197640 1 --2- 172.21.15.32:0/3060606247 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5c3c108740 0x7f5c3c1a6e10 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f5c3400ba50 tx=0x7f5c3400bf10 comp rx=0 tx=0).stop 2024-04-25T00:36:10.958 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.955+0000 7f5c41197640 1 -- 172.21.15.32:0/3060606247 shutdown_connections 2024-04-25T00:36:10.958 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.955+0000 7f5c41197640 1 --2- 172.21.15.32:0/3060606247 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f5c240a2e80 0x7f5c240a5340 secure :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0x55fe696cadc0 tx=0x7f5c3400e040 comp rx=0 tx=0).stop 2024-04-25T00:36:10.958 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.955+0000 7f5c41197640 1 --2- 172.21.15.32:0/3060606247 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5c3c10fde0 0x7f5c3c1ab6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.958 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.955+0000 7f5c41197640 1 --2- 172.21.15.32:0/3060606247 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5c3c1091c0 0x7f5c3c1a7350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:10.958 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.955+0000 7f5c41197640 1 --2- 172.21.15.32:0/3060606247 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5c3c108740 0x7f5c3c1a6e10 secure :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f5c3400ba50 tx=0x7f5c3400bf10 comp rx=0 tx=0).stop 2024-04-25T00:36:10.958 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.955+0000 7f5c41197640 1 -- 172.21.15.32:0/3060606247 >> 172.21.15.32:0/3060606247 conn(0x7f5c3c06c6c0 msgr2=0x7f5c3c0703c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:10.959 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.958+0000 7f5c41197640 1 -- 172.21.15.32:0/3060606247 shutdown_connections 2024-04-25T00:36:10.962 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:10.961+0000 7f5c41197640 1 -- 172.21.15.32:0/3060606247 wait complete. 2024-04-25T00:36:10.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:10 smithi032 ceph-mon[29357]: scrub queued for path: / 2024-04-25T00:36:10.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:10 smithi032 ceph-mon[29357]: scrub summary: idle+waiting paths [/] 2024-04-25T00:36:10.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:10 smithi032 ceph-mon[29357]: scrub summary: active paths [/] 2024-04-25T00:36:11.012 INFO:tasks.cephfs.filesystem:scrub status for tag:f9ea4c9a-574a-4be2-8054-d0cda1e643a1 - {'path': '/', 'tag': 'f9ea4c9a-574a-4be2-8054-d0cda1e643a1', 'options': 'recursive,force'} 2024-04-25T00:36:11.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:10 smithi121 ceph-mon[32231]: scrub queued for path: / 2024-04-25T00:36:11.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:10 smithi121 ceph-mon[32231]: scrub summary: idle+waiting paths [/] 2024-04-25T00:36:11.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:10 smithi121 ceph-mon[32231]: scrub summary: active paths [/] 2024-04-25T00:36:11.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:10 smithi130 ceph-mon[32525]: scrub queued for path: / 2024-04-25T00:36:11.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:10 smithi130 ceph-mon[32525]: scrub summary: idle+waiting paths [/] 2024-04-25T00:36:11.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:10 smithi130 ceph-mon[32525]: scrub summary: active paths [/] 2024-04-25T00:36:11.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:11 smithi032 ceph-mon[29357]: pgmap v89: 129 pgs: 129 active+clean; 21 GiB data, 44 GiB used, 1.0 TiB / 1.0 TiB avail; 1.6 MiB/s rd, 523 MiB/s wr, 1.77k op/s 2024-04-25T00:36:12.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:11 smithi121 ceph-mon[32231]: pgmap v89: 129 pgs: 129 active+clean; 21 GiB data, 44 GiB used, 1.0 TiB / 1.0 TiB avail; 1.6 MiB/s rd, 523 MiB/s wr, 1.77k op/s 2024-04-25T00:36:12.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:11 smithi130 ceph-mon[32525]: pgmap v89: 129 pgs: 129 active+clean; 21 GiB data, 44 GiB used, 1.0 TiB / 1.0 TiB avail; 1.6 MiB/s rd, 523 MiB/s wr, 1.77k op/s 2024-04-25T00:36:13.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:12 smithi121 ceph-mon[32231]: pgmap v90: 129 pgs: 129 active+clean; 21 GiB data, 45 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 360 MiB/s wr, 1.06k op/s 2024-04-25T00:36:13.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:12 smithi121 ceph-mon[32231]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:36:13.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:12 smithi130 ceph-mon[32525]: pgmap v90: 129 pgs: 129 active+clean; 21 GiB data, 45 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 360 MiB/s wr, 1.06k op/s 2024-04-25T00:36:13.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:12 smithi130 ceph-mon[32525]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:36:13.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:12 smithi032 ceph-mon[29357]: pgmap v90: 129 pgs: 129 active+clean; 21 GiB data, 45 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 360 MiB/s wr, 1.06k op/s 2024-04-25T00:36:13.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:12 smithi032 ceph-mon[29357]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:36:15.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:15 smithi130 ceph-mon[32525]: pgmap v91: 129 pgs: 129 active+clean; 23 GiB data, 47 GiB used, 1.0 TiB / 1.0 TiB avail; 1.6 MiB/s rd, 445 MiB/s wr, 1.50k op/s 2024-04-25T00:36:15.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:15 smithi032 ceph-mon[29357]: pgmap v91: 129 pgs: 129 active+clean; 23 GiB data, 47 GiB used, 1.0 TiB / 1.0 TiB avail; 1.6 MiB/s rd, 445 MiB/s wr, 1.50k op/s 2024-04-25T00:36:15.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:15 smithi121 ceph-mon[32231]: pgmap v91: 129 pgs: 129 active+clean; 23 GiB data, 47 GiB used, 1.0 TiB / 1.0 TiB avail; 1.6 MiB/s rd, 445 MiB/s wr, 1.50k op/s 2024-04-25T00:36:17.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:17 smithi130 ceph-mon[32525]: pgmap v92: 129 pgs: 129 active+clean; 24 GiB data, 49 GiB used, 1023 GiB / 1.0 TiB avail; 1.8 MiB/s rd, 486 MiB/s wr, 1.97k op/s 2024-04-25T00:36:17.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:17 smithi032 ceph-mon[29357]: pgmap v92: 129 pgs: 129 active+clean; 24 GiB data, 49 GiB used, 1023 GiB / 1.0 TiB avail; 1.8 MiB/s rd, 486 MiB/s wr, 1.97k op/s 2024-04-25T00:36:17.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:17 smithi121 ceph-mon[32231]: pgmap v92: 129 pgs: 129 active+clean; 24 GiB data, 49 GiB used, 1023 GiB / 1.0 TiB avail; 1.8 MiB/s rd, 486 MiB/s wr, 1.97k op/s 2024-04-25T00:36:19.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:19 smithi130 ceph-mon[32525]: pgmap v93: 129 pgs: 129 active+clean; 24 GiB data, 50 GiB used, 1023 GiB / 1.0 TiB avail; 2.2 MiB/s rd, 493 MiB/s wr, 2.06k op/s 2024-04-25T00:36:19.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:19 smithi032 ceph-mon[29357]: pgmap v93: 129 pgs: 129 active+clean; 24 GiB data, 50 GiB used, 1023 GiB / 1.0 TiB avail; 2.2 MiB/s rd, 493 MiB/s wr, 2.06k op/s 2024-04-25T00:36:19.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:19 smithi121 ceph-mon[32231]: pgmap v93: 129 pgs: 129 active+clean; 24 GiB data, 50 GiB used, 1023 GiB / 1.0 TiB avail; 2.2 MiB/s rd, 493 MiB/s wr, 2.06k op/s 2024-04-25T00:36:21.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:21 smithi130 ceph-mon[32525]: pgmap v94: 129 pgs: 129 active+clean; 26 GiB data, 55 GiB used, 1018 GiB / 1.0 TiB avail; 2.3 MiB/s rd, 712 MiB/s wr, 2.64k op/s 2024-04-25T00:36:21.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:21 smithi032 ceph-mon[29357]: pgmap v94: 129 pgs: 129 active+clean; 26 GiB data, 55 GiB used, 1018 GiB / 1.0 TiB avail; 2.3 MiB/s rd, 712 MiB/s wr, 2.64k op/s 2024-04-25T00:36:21.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:21 smithi121 ceph-mon[32231]: pgmap v94: 129 pgs: 129 active+clean; 26 GiB data, 55 GiB used, 1018 GiB / 1.0 TiB avail; 2.3 MiB/s rd, 712 MiB/s wr, 2.64k op/s 2024-04-25T00:36:23.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:23 smithi130 ceph-mon[32525]: pgmap v95: 129 pgs: 129 active+clean; 27 GiB data, 57 GiB used, 1016 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 576 MiB/s wr, 2.17k op/s 2024-04-25T00:36:23.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:23 smithi032 ceph-mon[29357]: pgmap v95: 129 pgs: 129 active+clean; 27 GiB data, 57 GiB used, 1016 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 576 MiB/s wr, 2.17k op/s 2024-04-25T00:36:23.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:23 smithi121 ceph-mon[32231]: pgmap v95: 129 pgs: 129 active+clean; 27 GiB data, 57 GiB used, 1016 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 576 MiB/s wr, 2.17k op/s 2024-04-25T00:36:25.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:25 smithi032 ceph-mon[29357]: pgmap v96: 129 pgs: 129 active+clean; 27 GiB data, 58 GiB used, 1015 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 584 MiB/s wr, 2.12k op/s 2024-04-25T00:36:25.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:25 smithi121 ceph-mon[32231]: pgmap v96: 129 pgs: 129 active+clean; 27 GiB data, 58 GiB used, 1015 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 584 MiB/s wr, 2.12k op/s 2024-04-25T00:36:25.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:25 smithi130 ceph-mon[32525]: pgmap v96: 129 pgs: 129 active+clean; 27 GiB data, 58 GiB used, 1015 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 584 MiB/s wr, 2.12k op/s 2024-04-25T00:36:27.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:27 smithi130 ceph-mon[32525]: pgmap v97: 129 pgs: 129 active+clean; 29 GiB data, 60 GiB used, 1013 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 733 MiB/s wr, 2.15k op/s 2024-04-25T00:36:27.444 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:27 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:36:27.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:27 smithi032 ceph-mon[29357]: pgmap v97: 129 pgs: 129 active+clean; 29 GiB data, 60 GiB used, 1013 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 733 MiB/s wr, 2.15k op/s 2024-04-25T00:36:27.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:27 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:36:27.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:27 smithi121 ceph-mon[32231]: pgmap v97: 129 pgs: 129 active+clean; 29 GiB data, 60 GiB used, 1013 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 733 MiB/s wr, 2.15k op/s 2024-04-25T00:36:27.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:27 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:36:28.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:28 smithi032 ceph-mon[29357]: scrub summary: active paths [/] 2024-04-25T00:36:28.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:28 smithi032 ceph-mon[29357]: scrub summary: idle+waiting paths [/] 2024-04-25T00:36:28.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:28 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:36:28.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:28 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:36:28.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:28 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:36:28.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:28 smithi121 ceph-mon[32231]: scrub summary: active paths [/] 2024-04-25T00:36:28.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:28 smithi121 ceph-mon[32231]: scrub summary: idle+waiting paths [/] 2024-04-25T00:36:28.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:28 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:36:28.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:28 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:36:28.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:28 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:36:28.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:28 smithi130 ceph-mon[32525]: scrub summary: active paths [/] 2024-04-25T00:36:28.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:28 smithi130 ceph-mon[32525]: scrub summary: idle+waiting paths [/] 2024-04-25T00:36:28.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:28 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:36:28.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:28 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:36:28.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:28 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:36:29.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:29 smithi032 ceph-mon[29357]: scrub summary: idle+waiting paths [/] 2024-04-25T00:36:29.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:29 smithi032 ceph-mon[29357]: pgmap v98: 129 pgs: 129 active+clean; 29 GiB data, 61 GiB used, 1012 GiB / 1.0 TiB avail; 861 KiB/s rd, 661 MiB/s wr, 1.71k op/s 2024-04-25T00:36:29.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:29 smithi121 ceph-mon[32231]: scrub summary: idle+waiting paths [/] 2024-04-25T00:36:29.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:29 smithi121 ceph-mon[32231]: pgmap v98: 129 pgs: 129 active+clean; 29 GiB data, 61 GiB used, 1012 GiB / 1.0 TiB avail; 861 KiB/s rd, 661 MiB/s wr, 1.71k op/s 2024-04-25T00:36:29.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:29 smithi130 ceph-mon[32525]: scrub summary: idle+waiting paths [/] 2024-04-25T00:36:29.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:29 smithi130 ceph-mon[32525]: pgmap v98: 129 pgs: 129 active+clean; 29 GiB data, 61 GiB used, 1012 GiB / 1.0 TiB avail; 861 KiB/s rd, 661 MiB/s wr, 1.71k op/s 2024-04-25T00:36:31.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:31 smithi032 ceph-mon[29357]: pgmap v99: 129 pgs: 129 active+clean; 29 GiB data, 61 GiB used, 1011 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 834 MiB/s wr, 2.32k op/s 2024-04-25T00:36:31.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:31 smithi121 ceph-mon[32231]: pgmap v99: 129 pgs: 129 active+clean; 29 GiB data, 61 GiB used, 1011 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 834 MiB/s wr, 2.32k op/s 2024-04-25T00:36:31.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:31 smithi130 ceph-mon[32525]: pgmap v99: 129 pgs: 129 active+clean; 29 GiB data, 61 GiB used, 1011 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 834 MiB/s wr, 2.32k op/s 2024-04-25T00:36:33.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:33 smithi032 ceph-mon[29357]: pgmap v100: 129 pgs: 129 active+clean; 29 GiB data, 62 GiB used, 1011 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 644 MiB/s wr, 1.78k op/s 2024-04-25T00:36:33.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:33 smithi032 ceph-mon[29357]: scrub summary: idle 2024-04-25T00:36:33.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:33 smithi032 ceph-mon[29357]: scrub summary: idle 2024-04-25T00:36:33.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:33 smithi121 ceph-mon[32231]: pgmap v100: 129 pgs: 129 active+clean; 29 GiB data, 62 GiB used, 1011 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 644 MiB/s wr, 1.78k op/s 2024-04-25T00:36:33.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:33 smithi121 ceph-mon[32231]: scrub summary: idle 2024-04-25T00:36:33.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:33 smithi121 ceph-mon[32231]: scrub summary: idle 2024-04-25T00:36:33.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:33 smithi130 ceph-mon[32525]: pgmap v100: 129 pgs: 129 active+clean; 29 GiB data, 62 GiB used, 1011 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 644 MiB/s wr, 1.78k op/s 2024-04-25T00:36:33.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:33 smithi130 ceph-mon[32525]: scrub summary: idle 2024-04-25T00:36:33.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:33 smithi130 ceph-mon[32525]: scrub summary: idle 2024-04-25T00:36:35.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:35 smithi032 ceph-mon[29357]: pgmap v101: 129 pgs: 129 active+clean; 29 GiB data, 62 GiB used, 1011 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 603 MiB/s wr, 1.65k op/s 2024-04-25T00:36:35.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:35 smithi121 ceph-mon[32231]: pgmap v101: 129 pgs: 129 active+clean; 29 GiB data, 62 GiB used, 1011 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 603 MiB/s wr, 1.65k op/s 2024-04-25T00:36:36.000 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:35 smithi130 ceph-mon[32525]: pgmap v101: 129 pgs: 129 active+clean; 29 GiB data, 62 GiB used, 1011 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 603 MiB/s wr, 1.65k op/s 2024-04-25T00:36:37.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:37 smithi032 ceph-mon[29357]: pgmap v102: 129 pgs: 129 active+clean; 30 GiB data, 63 GiB used, 1010 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 736 MiB/s wr, 2.47k op/s 2024-04-25T00:36:37.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:37 smithi121 ceph-mon[32231]: pgmap v102: 129 pgs: 129 active+clean; 30 GiB data, 63 GiB used, 1010 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 736 MiB/s wr, 2.47k op/s 2024-04-25T00:36:37.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:37 smithi130 ceph-mon[32525]: pgmap v102: 129 pgs: 129 active+clean; 30 GiB data, 63 GiB used, 1010 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 736 MiB/s wr, 2.47k op/s 2024-04-25T00:36:39.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:39 smithi032 ceph-mon[29357]: pgmap v103: 129 pgs: 129 active+clean; 30 GiB data, 63 GiB used, 1010 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 558 MiB/s wr, 2.13k op/s 2024-04-25T00:36:39.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:39 smithi121 ceph-mon[32231]: pgmap v103: 129 pgs: 129 active+clean; 30 GiB data, 63 GiB used, 1010 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 558 MiB/s wr, 2.13k op/s 2024-04-25T00:36:39.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:39 smithi130 ceph-mon[32525]: pgmap v103: 129 pgs: 129 active+clean; 30 GiB data, 63 GiB used, 1010 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 558 MiB/s wr, 2.13k op/s 2024-04-25T00:36:41.014 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-25T00:36:41.014 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-25T00:36:41.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:41 smithi121 ceph-mon[32231]: pgmap v104: 129 pgs: 129 active+clean; 30 GiB data, 63 GiB used, 1009 GiB / 1.0 TiB avail; 2.6 MiB/s rd, 746 MiB/s wr, 2.28k op/s 2024-04-25T00:36:41.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:41 smithi130 ceph-mon[32525]: pgmap v104: 129 pgs: 129 active+clean; 30 GiB data, 63 GiB used, 1009 GiB / 1.0 TiB avail; 2.6 MiB/s rd, 746 MiB/s wr, 2.28k op/s 2024-04-25T00:36:41.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.722+0000 7f7a5d163640 1 -- 172.21.15.32:0/4049660089 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7a58101720 msgr2=0x7f7a58101ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:41.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.722+0000 7f7a5d163640 1 --2- 172.21.15.32:0/4049660089 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7a58101720 0x7f7a58101ba0 secure :-1 s=READY pgs=608 cs=0 l=1 rev1=1 crypto rx=0x7f7a4001a940 tx=0x7f7a4003b470 comp rx=0 tx=0).stop 2024-04-25T00:36:41.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.722+0000 7f7a5d163640 1 -- 172.21.15.32:0/4049660089 shutdown_connections 2024-04-25T00:36:41.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.722+0000 7f7a5d163640 1 --2- 172.21.15.32:0/4049660089 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7a581020e0 0x7f7a5810e5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:41.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.722+0000 7f7a5d163640 1 --2- 172.21.15.32:0/4049660089 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7a58101720 0x7f7a58101ba0 unknown :-1 s=CLOSED pgs=608 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:41.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.722+0000 7f7a5d163640 1 --2- 172.21.15.32:0/4049660089 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7a58107730 0x7f7a58107b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:41.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.722+0000 7f7a5d163640 1 -- 172.21.15.32:0/4049660089 >> 172.21.15.32:0/4049660089 conn(0x7f7a58078b40 msgr2=0x7f7a58078f50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:41.726 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.722+0000 7f7a5d163640 1 -- 172.21.15.32:0/4049660089 shutdown_connections 2024-04-25T00:36:41.726 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.722+0000 7f7a5d163640 1 -- 172.21.15.32:0/4049660089 wait complete. 2024-04-25T00:36:41.726 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.723+0000 7f7a5d163640 1 Processor -- start 2024-04-25T00:36:41.728 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.724+0000 7f7a5d163640 1 -- start start 2024-04-25T00:36:41.729 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.725+0000 7f7a5d163640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7a58101720 0x7f7a58075290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:41.729 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.725+0000 7f7a5d163640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7a581020e0 0x7f7a5806f300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:41.729 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.725+0000 7f7a5d163640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7a58107730 0x7f7a5806f840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:41.729 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.725+0000 7f7a5d163640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f7a5806db60 con 0x7f7a58107730 2024-04-25T00:36:41.729 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.725+0000 7f7a5d163640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f7a5806d9e0 con 0x7f7a58101720 2024-04-25T00:36:41.729 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.725+0000 7f7a5d163640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f7a5806dce0 con 0x7f7a581020e0 2024-04-25T00:36:41.730 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.727+0000 7f7a5659c640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7a581020e0 0x7f7a5806f300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:41.730 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.727+0000 7f7a5659c640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7a581020e0 0x7f7a5806f300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:56292/0 (socket says 172.21.15.32:56292) 2024-04-25T00:36:41.730 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.727+0000 7f7a5659c640 1 -- 172.21.15.32:0/85844978 learned_addr learned my addr 172.21.15.32:0/85844978 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:41.732 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.728+0000 7f7a5759e640 1 --2- 172.21.15.32:0/85844978 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7a58107730 0x7f7a5806f840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:41.732 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.729+0000 7f7a5659c640 1 -- 172.21.15.32:0/85844978 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7a58101720 msgr2=0x7f7a58075290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:41.732 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.729+0000 7f7a5659c640 1 --2- 172.21.15.32:0/85844978 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7a58101720 0x7f7a58075290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:41.732 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.729+0000 7f7a5659c640 1 -- 172.21.15.32:0/85844978 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7a58107730 msgr2=0x7f7a5806f840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:41.732 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.729+0000 7f7a5659c640 1 --2- 172.21.15.32:0/85844978 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7a58107730 0x7f7a5806f840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:41.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.729+0000 7f7a5659c640 1 -- 172.21.15.32:0/85844978 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7a5806ff40 con 0x7f7a581020e0 2024-04-25T00:36:41.736 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.731+0000 7f7a5659c640 1 --2- 172.21.15.32:0/85844978 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7a581020e0 0x7f7a5806f300 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f7a4001aa60 tx=0x7f7a40040a10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:41.736 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.733+0000 7f7a4ffff640 1 -- 172.21.15.32:0/85844978 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7a4004f020 con 0x7f7a581020e0 2024-04-25T00:36:41.736 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.733+0000 7f7a4ffff640 1 -- 172.21.15.32:0/85844978 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7a40040eb0 con 0x7f7a581020e0 2024-04-25T00:36:41.736 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.733+0000 7f7a4ffff640 1 -- 172.21.15.32:0/85844978 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7a40046070 con 0x7f7a581020e0 2024-04-25T00:36:41.736 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.734+0000 7f7a5d163640 1 -- 172.21.15.32:0/85844978 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7a58193ef0 con 0x7f7a581020e0 2024-04-25T00:36:41.736 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.734+0000 7f7a5d163640 1 -- 172.21.15.32:0/85844978 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7a58194390 con 0x7f7a581020e0 2024-04-25T00:36:41.745 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.741+0000 7f7a4ffff640 1 -- 172.21.15.32:0/85844978 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f7a40052020 con 0x7f7a581020e0 2024-04-25T00:36:41.748 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.743+0000 7f7a4ffff640 1 --2- 172.21.15.32:0/85844978 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7a440a2ed0 0x7f7a440a5390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:41.748 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.744+0000 7f7a4ffff640 1 -- 172.21.15.32:0/85844978 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f7a40101aa0 con 0x7f7a581020e0 2024-04-25T00:36:41.753 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:41 smithi032 ceph-mon[29357]: pgmap v104: 129 pgs: 129 active+clean; 30 GiB data, 63 GiB used, 1009 GiB / 1.0 TiB avail; 2.6 MiB/s rd, 746 MiB/s wr, 2.28k op/s 2024-04-25T00:36:42.544 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:41.744+0000 7f7a56d9d640 1 --2- 172.21.15.32:0/85844978 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7a440a2ed0 0x7f7a440a5390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:42.544 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.543+0000 7f7a5fe7b740 1 Processor -- start 2024-04-25T00:36:42.545 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.544+0000 7f7a5fe7b740 1 -- start start 2024-04-25T00:36:42.546 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.545+0000 7f7a5fe7b740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56416432b630 0x56416432ba10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:42.546 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.545+0000 7f7a5fe7b740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56416432bf50 0x5641643360e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:42.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.545+0000 7f7a5fe7b740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564164336620 0x564164338a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:42.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.545+0000 7f7a5fe7b740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5641642906b0 con 0x564164336620 2024-04-25T00:36:42.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.545+0000 7f7a5fe7b740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x564164238880 con 0x56416432b630 2024-04-25T00:36:42.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.545+0000 7f7a5fe7b740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x56416428c670 con 0x56416432bf50 2024-04-25T00:36:42.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.545+0000 7f7a5659c640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56416432bf50 0x5641643360e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:42.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.545+0000 7f7a5659c640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56416432bf50 0x5641643360e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:56294/0 (socket says 172.21.15.32:56294) 2024-04-25T00:36:42.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.545+0000 7f7a5759e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564164336620 0x564164338a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:42.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.545+0000 7f7a5659c640 1 -- 172.21.15.32:0/362152809 learned_addr learned my addr 172.21.15.32:0/362152809 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:42.548 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.545+0000 7f7a5659c640 1 -- 172.21.15.32:0/362152809 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56416432b630 msgr2=0x56416432ba10 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-25T00:36:42.548 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.545+0000 7f7a5659c640 1 --2- 172.21.15.32:0/362152809 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56416432b630 0x56416432ba10 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.548 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.545+0000 7f7a5659c640 1 -- 172.21.15.32:0/362152809 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564164336620 msgr2=0x564164338a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:42.549 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.545+0000 7f7a5659c640 1 --2- 172.21.15.32:0/362152809 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564164336620 0x564164338a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.549 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.545+0000 7f7a5659c640 1 -- 172.21.15.32:0/362152809 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56416406e7d0 con 0x56416432bf50 2024-04-25T00:36:42.549 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.546+0000 7f7a5659c640 1 --2- 172.21.15.32:0/362152809 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56416432bf50 0x5641643360e0 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f7a400424a0 tx=0x7f7a400ce370 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5a9cc5e8ea98d3e5 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:42.549 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.546+0000 7f7a4d7fa640 1 -- 172.21.15.32:0/362152809 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7a400e4020 con 0x56416432bf50 2024-04-25T00:36:42.549 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.546+0000 7f7a4d7fa640 1 -- 172.21.15.32:0/362152809 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7a400d6820 con 0x56416432bf50 2024-04-25T00:36:42.549 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.546+0000 7f7a4d7fa640 1 -- 172.21.15.32:0/362152809 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7a400d65f0 con 0x56416432bf50 2024-04-25T00:36:42.550 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.548+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/362152809 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56416432bf50 msgr2=0x5641643360e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:42.550 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.548+0000 7f7a5fe7b740 1 --2- 172.21.15.32:0/362152809 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56416432bf50 0x5641643360e0 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f7a400424a0 tx=0x7f7a400ce370 comp rx=0 tx=0).stop 2024-04-25T00:36:42.550 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.549+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/362152809 shutdown_connections 2024-04-25T00:36:42.551 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.549+0000 7f7a5fe7b740 1 --2- 172.21.15.32:0/362152809 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564164336620 0x564164338a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.551 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.549+0000 7f7a5fe7b740 1 --2- 172.21.15.32:0/362152809 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56416432bf50 0x5641643360e0 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.551 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.549+0000 7f7a5fe7b740 1 --2- 172.21.15.32:0/362152809 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56416432b630 0x56416432ba10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.551 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.549+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/362152809 >> 172.21.15.32:0/362152809 conn(0x564164327950 msgr2=0x564164327d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:42.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.550+0000 7f7a56d9d640 1 --2- 172.21.15.32:0/85844978 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7a440a2ed0 0x7f7a440a5390 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f7a500096f0 tx=0x7f7a50006c40 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:42.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.550+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/362152809 shutdown_connections 2024-04-25T00:36:42.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.550+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/362152809 wait complete. 2024-04-25T00:36:42.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.551+0000 7f7a5fe7b740 1 -- start start 2024-04-25T00:36:42.553 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.551+0000 7f7a5fe7b740 1 Processor -- start 2024-04-25T00:36:42.553 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.552+0000 7f7a5fe7b740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56416432bf50 0x564164333600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:42.554 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.552+0000 7f7a5fe7b740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564164336620 0x564164333b40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:42.554 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.552+0000 7f7a5fe7b740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564164334080 0x564164342600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:42.554 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.552+0000 7f7a5fe7b740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5641643391d0 con 0x564164334080 2024-04-25T00:36:42.554 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.552+0000 7f7a5fe7b740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x56416424bf50 con 0x564164336620 2024-04-25T00:36:42.554 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.552+0000 7f7a5fe7b740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x564164339350 con 0x56416432bf50 2024-04-25T00:36:42.555 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.552+0000 7f7a5fe7b740 10 client.0 _ll_register_callbacks cb 0x564164287000 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:36:42.555 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.553+0000 7f7a5759e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564164334080 0x564164342600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:42.555 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.553+0000 7f7a5759e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564164334080 0x564164342600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44064/0 (socket says 172.21.15.32:44064) 2024-04-25T00:36:42.555 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.553+0000 7f7a5759e640 1 -- 172.21.15.32:0/1087598465 learned_addr learned my addr 172.21.15.32:0/1087598465 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:42.555 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.553+0000 7f7a56d9d640 1 --2- 172.21.15.32:0/1087598465 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56416432bf50 0x564164333600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:42.555 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.553+0000 7f7a5659c640 1 --2- 172.21.15.32:0/1087598465 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564164336620 0x564164333b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:42.555 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.553+0000 7f7a5759e640 1 -- 172.21.15.32:0/1087598465 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56416432bf50 msgr2=0x564164333600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:42.555 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.553+0000 7f7a5759e640 1 --2- 172.21.15.32:0/1087598465 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56416432bf50 0x564164333600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.555 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.553+0000 7f7a5759e640 1 -- 172.21.15.32:0/1087598465 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564164336620 msgr2=0x564164333b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:42.555 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.553+0000 7f7a5759e640 1 --2- 172.21.15.32:0/1087598465 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564164336620 0x564164333b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.556 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.553+0000 7f7a5759e640 1 -- 172.21.15.32:0/1087598465 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564163fa43f0 con 0x564164334080 2024-04-25T00:36:42.556 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.554+0000 7f7a5759e640 1 --2- 172.21.15.32:0/1087598465 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564164334080 0x564164342600 secure :-1 s=READY pgs=609 cs=0 l=1 rev1=1 crypto rx=0x7f7a4800ae40 tx=0x7f7a48007680 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c1375ab72d7141d server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:42.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.555+0000 7f7a3e7fc640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:36:42.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.555+0000 7f7a3e7fc640 1 -- 172.21.15.32:0/1087598465 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7a48016410 con 0x564164334080 2024-04-25T00:36:42.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.555+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/1087598465 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f7a400d6820 con 0x564164334080 2024-04-25T00:36:42.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.555+0000 7f7a3e7fc640 1 -- 172.21.15.32:0/1087598465 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f7a480165b0 con 0x564164334080 2024-04-25T00:36:42.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.555+0000 7f7a3e7fc640 1 -- 172.21.15.32:0/1087598465 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f7a480167b0 con 0x564164334080 2024-04-25T00:36:42.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.555+0000 7f7a5fe7b740 10 client.16122 fetch_fsmap learned FSMap version 27 2024-04-25T00:36:42.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.555+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/1087598465 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x564163f640f0 con 0x564164334080 2024-04-25T00:36:42.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.556+0000 7f7a3e7fc640 1 -- 172.21.15.32:0/1087598465 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7a400d6820 con 0x564164334080 2024-04-25T00:36:42.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.557+0000 7f7a3e7fc640 1 -- 172.21.15.32:0/1087598465 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 27) ==== 8354+0+0 (secure 0 0 0) 0x7f7a4801fb40 con 0x564164334080 2024-04-25T00:36:42.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.557+0000 7f7a5fe7b740 10 client.16122 fetch_fsmap finished waiting for FSMap version 27 2024-04-25T00:36:42.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.557+0000 7f7a5fe7b740 10 client.16122 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:36:42.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.557+0000 7f7a5fe7b740 20 client.16122 populate_metadata read hostname 'smithi032' 2024-04-25T00:36:42.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.557+0000 7f7a5fe7b740 1 --2- 172.21.15.32:0/1087598465 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x564164346e30 0x564164349250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:42.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.557+0000 7f7a5fe7b740 4 client.16122 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:36:42.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.557+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/1087598465 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56416429dbf0 con 0x564164346e30 2024-04-25T00:36:42.559 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.557+0000 7f7a56d9d640 1 --2- 172.21.15.32:0/1087598465 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x564164346e30 0x564164349250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:42.561 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.559+0000 7f7a56d9d640 1 --2- 172.21.15.32:0/1087598465 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x564164346e30 0x564164349250 crc :-1 s=READY pgs=79 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=6a4c76b4dadaa49a server_cookie=2da04b64b6eaeaa6 in_seq=0 out_seq=0 2024-04-25T00:36:42.561 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.559+0000 7f7a3e7fc640 10 client.16122 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:42.562 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.560+0000 7f7a3e7fc640 1 -- 172.21.15.32:0/1087598465 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x56416429dbf0 con 0x564164346e30 2024-04-25T00:36:42.562 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.560+0000 7f7a3e7fc640 10 client.16122 handle_command_reply: tid=0 2024-04-25T00:36:42.562 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.561+0000 7f7a5fe7b740 1 client.16122 shutdown 2024-04-25T00:36:42.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.563+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/1087598465 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564164334080 msgr2=0x564164342600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:42.565 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.563+0000 7f7a5fe7b740 1 --2- 172.21.15.32:0/1087598465 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564164334080 0x564164342600 secure :-1 s=READY pgs=609 cs=0 l=1 rev1=1 crypto rx=0x7f7a4800ae40 tx=0x7f7a48007680 comp rx=0 tx=0).stop 2024-04-25T00:36:42.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.563+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/1087598465 shutdown_connections 2024-04-25T00:36:42.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.563+0000 7f7a5fe7b740 1 --2- 172.21.15.32:0/1087598465 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x564164346e30 0x564164349250 crc :-1 s=READY pgs=79 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.564+0000 7f7a5fe7b740 1 --2- 172.21.15.32:0/1087598465 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564164334080 0x564164342600 unknown :-1 s=CLOSED pgs=609 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.564+0000 7f7a5fe7b740 1 --2- 172.21.15.32:0/1087598465 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564164336620 0x564164333b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.564+0000 7f7a3e7fc640 0 client.16122 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:42.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.564+0000 7f7a5fe7b740 1 --2- 172.21.15.32:0/1087598465 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56416432bf50 0x564164333600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.564+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/1087598465 >> 172.21.15.32:0/1087598465 conn(0x564164327950 msgr2=0x56416432d630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:42.571 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.569+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/1087598465 shutdown_connections 2024-04-25T00:36:42.572 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.569+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/1087598465 wait complete. 2024-04-25T00:36:42.572 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.569+0000 7f7a5fe7b740 20 client.16122 trim_cache size 0 max 16384 2024-04-25T00:36:42.631 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.627+0000 7f7a5fe7b740 1 Processor -- start 2024-04-25T00:36:42.631 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.627+0000 7f7a5fe7b740 1 -- start start 2024-04-25T00:36:42.631 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.627+0000 7f7a5fe7b740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56416432bf50 0x564164342090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:42.631 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.627+0000 7f7a5fe7b740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564164334080 0x5641643425d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:42.631 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.628+0000 7f7a5fe7b740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564164336620 0x56416434eb40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:42.632 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.628+0000 7f7a5fe7b740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f7a38012950 con 0x564164336620 2024-04-25T00:36:42.632 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.628+0000 7f7a5fe7b740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f7a38010220 con 0x564164334080 2024-04-25T00:36:42.632 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.628+0000 7f7a5fe7b740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f7a3800e070 con 0x56416432bf50 2024-04-25T00:36:42.635 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.630+0000 7f7a56d9d640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56416432bf50 0x564164342090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:42.635 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.630+0000 7f7a5759e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564164336620 0x56416434eb40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:42.635 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.630+0000 7f7a5659c640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564164334080 0x5641643425d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:42.636 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.630+0000 7f7a5759e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564164336620 0x56416434eb40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44078/0 (socket says 172.21.15.32:44078) 2024-04-25T00:36:42.636 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.630+0000 7f7a5659c640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564164334080 0x5641643425d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:47592/0 (socket says 172.21.15.32:47592) 2024-04-25T00:36:42.636 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.630+0000 7f7a5759e640 1 -- 172.21.15.32:0/4183147041 learned_addr learned my addr 172.21.15.32:0/4183147041 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:42.636 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.633+0000 7f7a5659c640 1 -- 172.21.15.32:0/4183147041 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56416432bf50 msgr2=0x564164342090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:42.636 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.633+0000 7f7a5659c640 1 --2- 172.21.15.32:0/4183147041 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56416432bf50 0x564164342090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.636 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.633+0000 7f7a5659c640 1 -- 172.21.15.32:0/4183147041 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564164336620 msgr2=0x56416434eb40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:42.636 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.633+0000 7f7a5659c640 1 --2- 172.21.15.32:0/4183147041 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564164336620 0x56416434eb40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.636 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.633+0000 7f7a5659c640 1 -- 172.21.15.32:0/4183147041 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564164319090 con 0x564164334080 2024-04-25T00:36:42.642 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.640+0000 7f7a5659c640 1 --2- 172.21.15.32:0/4183147041 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564164334080 0x5641643425d0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x56416432cbf0 tx=0x7f7a400058d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8cfc6d8b6d0868fc server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:42.644 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.642+0000 7f7a4cff9640 1 -- 172.21.15.32:0/4183147041 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7a400e4040 con 0x564164334080 2024-04-25T00:36:42.644 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.642+0000 7f7a4cff9640 1 -- 172.21.15.32:0/4183147041 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7a40005160 con 0x564164334080 2024-04-25T00:36:42.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.642+0000 7f7a4cff9640 1 -- 172.21.15.32:0/4183147041 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7a400cd000 con 0x564164334080 2024-04-25T00:36:42.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.644+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/4183147041 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564164334080 msgr2=0x5641643425d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:42.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.644+0000 7f7a5fe7b740 1 --2- 172.21.15.32:0/4183147041 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564164334080 0x5641643425d0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x56416432cbf0 tx=0x7f7a400058d0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.644+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/4183147041 shutdown_connections 2024-04-25T00:36:42.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.644+0000 7f7a5fe7b740 1 --2- 172.21.15.32:0/4183147041 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564164336620 0x56416434eb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.644+0000 7f7a5fe7b740 1 --2- 172.21.15.32:0/4183147041 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564164334080 0x5641643425d0 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.644+0000 7f7a5fe7b740 1 --2- 172.21.15.32:0/4183147041 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56416432bf50 0x564164342090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.644+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/4183147041 >> 172.21.15.32:0/4183147041 conn(0x564164327950 msgr2=0x5641643335c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:42.650 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.649+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/4183147041 shutdown_connections 2024-04-25T00:36:42.651 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.649+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/4183147041 wait complete. 2024-04-25T00:36:42.651 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.650+0000 7f7a5fe7b740 1 -- start start 2024-04-25T00:36:42.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.650+0000 7f7a5fe7b740 1 Processor -- start 2024-04-25T00:36:42.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.650+0000 7f7a5fe7b740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56416432bf50 0x56416433e5a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:42.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.650+0000 7f7a5fe7b740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564164336620 0x56416433eae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:42.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.650+0000 7f7a5fe7b740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56416432fa50 0x56416432fed0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:42.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.651+0000 7f7a5fe7b740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f7a3800e6e0 con 0x56416432fa50 2024-04-25T00:36:42.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.651+0000 7f7a5fe7b740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f7a3800fb50 con 0x56416432bf50 2024-04-25T00:36:42.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.651+0000 7f7a5fe7b740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f7a3800edb0 con 0x564164336620 2024-04-25T00:36:42.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.651+0000 7f7a5fe7b740 10 client.0 _ll_register_callbacks cb 0x7f7a48009ad0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:36:42.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.651+0000 7f7a5659c640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564164336620 0x56416433eae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:42.654 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.651+0000 7f7a5659c640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564164336620 0x56416433eae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:56326/0 (socket says 172.21.15.32:56326) 2024-04-25T00:36:42.654 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.651+0000 7f7a5659c640 1 -- 172.21.15.32:0/3466725202 learned_addr learned my addr 172.21.15.32:0/3466725202 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:42.654 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.652+0000 7f7a56d9d640 1 --2- 172.21.15.32:0/3466725202 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56416432bf50 0x56416433e5a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:42.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.654+0000 7f7a5659c640 1 -- 172.21.15.32:0/3466725202 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56416432bf50 msgr2=0x56416433e5a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:42.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.654+0000 7f7a5659c640 1 --2- 172.21.15.32:0/3466725202 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56416432bf50 0x56416433e5a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.654+0000 7f7a5659c640 1 -- 172.21.15.32:0/3466725202 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56416432fa50 msgr2=0x56416432fed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:42.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.654+0000 7f7a5759e640 1 --2- 172.21.15.32:0/3466725202 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56416432fa50 0x56416432fed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:42.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.655+0000 7f7a5659c640 1 --2- 172.21.15.32:0/3466725202 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56416432fa50 0x56416432fed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.655+0000 7f7a5659c640 1 -- 172.21.15.32:0/3466725202 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5641643306d0 con 0x564164336620 2024-04-25T00:36:42.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.655+0000 7f7a5759e640 1 --2- 172.21.15.32:0/3466725202 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56416432fa50 0x56416432fed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:36:42.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.659+0000 7f7a5659c640 1 --2- 172.21.15.32:0/3466725202 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564164336620 0x56416433eae0 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f7a400ca010 tx=0x7f7a400ff290 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2eba227dc174e401 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:42.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.659+0000 7f7a3e7fc640 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:36:42.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.661+0000 7f7a3e7fc640 1 -- 172.21.15.32:0/3466725202 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7a400ec070 con 0x564164336620 2024-04-25T00:36:42.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.661+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/3466725202 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f7a40005160 con 0x564164336620 2024-04-25T00:36:42.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.661+0000 7f7a3e7fc640 1 -- 172.21.15.32:0/3466725202 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f7a400d6590 con 0x564164336620 2024-04-25T00:36:42.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.665+0000 7f7a3e7fc640 1 -- 172.21.15.32:0/3466725202 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f7a400e4040 con 0x564164336620 2024-04-25T00:36:42.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.665+0000 7f7a5fe7b740 10 client.25666 fetch_fsmap learned FSMap version 27 2024-04-25T00:36:42.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.665+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/3466725202 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x5641643ad250 con 0x564164336620 2024-04-25T00:36:42.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.669+0000 7f7a3e7fc640 1 -- 172.21.15.32:0/3466725202 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7a40005160 con 0x564164336620 2024-04-25T00:36:42.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.669+0000 7f7a3e7fc640 1 -- 172.21.15.32:0/3466725202 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 27) ==== 8354+0+0 (secure 0 0 0) 0x7f7a400ec210 con 0x564164336620 2024-04-25T00:36:42.672 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.670+0000 7f7a5fe7b740 10 client.25666 fetch_fsmap finished waiting for FSMap version 27 2024-04-25T00:36:42.672 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.670+0000 7f7a5fe7b740 10 client.25666 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:36:42.672 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.670+0000 7f7a5fe7b740 20 client.25666 populate_metadata read hostname 'smithi032' 2024-04-25T00:36:42.672 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.670+0000 7f7a5fe7b740 1 --2- 172.21.15.32:0/3466725202 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5641643ad530 0x5641643af950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:42.672 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.670+0000 7f7a5fe7b740 4 client.25666 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-25T00:36:42.672 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.670+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/3466725202 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "scrub status"}) -- 0x564164297f40 con 0x5641643ad530 2024-04-25T00:36:42.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.674+0000 7f7a56d9d640 1 --2- 172.21.15.32:0/3466725202 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5641643ad530 0x5641643af950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:42.683 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.681+0000 7f7a56d9d640 1 --2- 172.21.15.32:0/3466725202 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5641643ad530 0x5641643af950 crc :-1 s=READY pgs=80 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f8124d6b12ebf286 server_cookie=f7904c56e6ca6639 in_seq=0 out_seq=0 2024-04-25T00:36:42.684 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.682+0000 7f7a3e7fc640 10 client.25666 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:42.684 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.682+0000 7f7a3e7fc640 1 -- 172.21.15.32:0/3466725202 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+63 (crc 0 0 0) 0x564164297f40 con 0x5641643ad530 2024-04-25T00:36:42.684 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.682+0000 7f7a3e7fc640 10 client.25666 handle_command_reply: tid=0 2024-04-25T00:36:42.685 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.683+0000 7f7a5fe7b740 1 client.25666 shutdown 2024-04-25T00:36:42.685 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.683+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/3466725202 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564164336620 msgr2=0x56416433eae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:42.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.683+0000 7f7a5fe7b740 1 --2- 172.21.15.32:0/3466725202 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564164336620 0x56416433eae0 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f7a400ca010 tx=0x7f7a400ff290 comp rx=0 tx=0).stop 2024-04-25T00:36:42.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.683+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/3466725202 shutdown_connections 2024-04-25T00:36:42.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.683+0000 7f7a5fe7b740 1 --2- 172.21.15.32:0/3466725202 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5641643ad530 0x5641643af950 crc :-1 s=READY pgs=80 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.683+0000 7f7a5fe7b740 1 --2- 172.21.15.32:0/3466725202 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x56416432fa50 0x56416432fed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.683+0000 7f7a5fe7b740 1 --2- 172.21.15.32:0/3466725202 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564164336620 0x56416433eae0 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.683+0000 7f7a5fe7b740 1 --2- 172.21.15.32:0/3466725202 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x56416432bf50 0x56416433e5a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.683+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/3466725202 >> 172.21.15.32:0/3466725202 conn(0x564164327950 msgr2=0x5641643335a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:42.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.683+0000 7f7a3e7fc640 0 client.25666 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:42.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.686+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/3466725202 shutdown_connections 2024-04-25T00:36:42.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.686+0000 7f7a5fe7b740 1 -- 172.21.15.32:0/3466725202 wait complete. 2024-04-25T00:36:42.688 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.687+0000 7f7a5fe7b740 20 client.25666 trim_cache size 0 max 16384 2024-04-25T00:36:42.692 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:36:42.692 INFO:teuthology.orchestra.run.smithi032.stdout: "status": "no active scrubs running", 2024-04-25T00:36:42.692 INFO:teuthology.orchestra.run.smithi032.stdout: "scrubs": {} 2024-04-25T00:36:42.692 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:36:42.702 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.701+0000 7f7a5d163640 1 -- 172.21.15.32:0/85844978 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7a440a2ed0 msgr2=0x7f7a440a5390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:42.702 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.701+0000 7f7a5d163640 1 --2- 172.21.15.32:0/85844978 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7a440a2ed0 0x7f7a440a5390 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f7a500096f0 tx=0x7f7a50006c40 comp rx=0 tx=0).stop 2024-04-25T00:36:42.703 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.701+0000 7f7a5d163640 1 -- 172.21.15.32:0/85844978 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7a581020e0 msgr2=0x7f7a5806f300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:42.703 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.701+0000 7f7a5d163640 1 --2- 172.21.15.32:0/85844978 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7a581020e0 0x7f7a5806f300 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f7a4001aa60 tx=0x7f7a40040a10 comp rx=0 tx=0).stop 2024-04-25T00:36:42.703 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.702+0000 7f7a5d163640 1 -- 172.21.15.32:0/85844978 shutdown_connections 2024-04-25T00:36:42.703 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.702+0000 7f7a5d163640 1 --2- 172.21.15.32:0/85844978 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7a440a2ed0 0x7f7a440a5390 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.703 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.702+0000 7f7a5d163640 1 --2- 172.21.15.32:0/85844978 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7a58107730 0x7f7a5806f840 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.704 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.702+0000 7f7a5d163640 1 --2- 172.21.15.32:0/85844978 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7a581020e0 0x7f7a5806f300 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.704 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.702+0000 7f7a5d163640 1 --2- 172.21.15.32:0/85844978 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7a58101720 0x7f7a58075290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:42.704 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.702+0000 7f7a5d163640 1 -- 172.21.15.32:0/85844978 >> 172.21.15.32:0/85844978 conn(0x7f7a58078b40 msgr2=0x7f7a58104180 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:42.708 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.706+0000 7f7a5d163640 1 -- 172.21.15.32:0/85844978 shutdown_connections 2024-04-25T00:36:42.708 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:42.706+0000 7f7a5d163640 1 -- 172.21.15.32:0/85844978 wait complete. 2024-04-25T00:36:42.748 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-25T00:36:42.748 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:36:43.568 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:43 smithi032 ceph-mon[29357]: pgmap v105: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 1.9 MiB/s rd, 657 MiB/s wr, 1.66k op/s 2024-04-25T00:36:43.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:43 smithi121 ceph-mon[32231]: pgmap v105: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 1.9 MiB/s rd, 657 MiB/s wr, 1.66k op/s 2024-04-25T00:36:43.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:43 smithi130 ceph-mon[32525]: pgmap v105: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 1.9 MiB/s rd, 657 MiB/s wr, 1.66k op/s 2024-04-25T00:36:43.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.875+0000 7ff6c6134640 1 -- 172.21.15.32:0/3070375620 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff6c010e6a0 msgr2=0x7ff6c011cf90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:43.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.875+0000 7ff6c6134640 1 --2- 172.21.15.32:0/3070375620 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff6c010e6a0 0x7ff6c011cf90 secure :-1 s=READY pgs=610 cs=0 l=1 rev1=1 crypto rx=0x7ff6b8009ca0 tx=0x7ff6b802a8f0 comp rx=0 tx=0).stop 2024-04-25T00:36:43.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.875+0000 7ff6c6134640 1 -- 172.21.15.32:0/3070375620 shutdown_connections 2024-04-25T00:36:43.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.875+0000 7ff6c6134640 1 --2- 172.21.15.32:0/3070375620 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff6c010e6a0 0x7ff6c011cf90 unknown :-1 s=CLOSED pgs=610 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:43.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.875+0000 7ff6c6134640 1 --2- 172.21.15.32:0/3070375620 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff6c0110310 0x7ff6c010dfc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:43.878 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.875+0000 7ff6c6134640 1 --2- 172.21.15.32:0/3070375620 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff6c010fea0 0x7ff6c010da80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:43.878 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.875+0000 7ff6c6134640 1 -- 172.21.15.32:0/3070375620 >> 172.21.15.32:0/3070375620 conn(0x7ff6c006c860 msgr2=0x7ff6c006cc70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:43.884 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.882+0000 7ff6c6134640 1 -- 172.21.15.32:0/3070375620 shutdown_connections 2024-04-25T00:36:43.885 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.883+0000 7ff6c6134640 1 -- 172.21.15.32:0/3070375620 wait complete. 2024-04-25T00:36:43.885 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.884+0000 7ff6c6134640 1 Processor -- start 2024-04-25T00:36:43.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.884+0000 7ff6c6134640 1 -- start start 2024-04-25T00:36:43.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.885+0000 7ff6c6134640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff6c010fea0 0x7ff6c01ac8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:43.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.885+0000 7ff6c6134640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff6c01acdf0 0x7ff6c01a6a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:43.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.885+0000 7ff6c6134640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff6c01a6fa0 0x7ff6c01a7420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:43.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.885+0000 7ff6c6134640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7ff6c0120a20 con 0x7ff6c010fea0 2024-04-25T00:36:43.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.885+0000 7ff6c6134640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7ff6c01208a0 con 0x7ff6c01a6fa0 2024-04-25T00:36:43.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.885+0000 7ff6c6134640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7ff6c0120ba0 con 0x7ff6c01acdf0 2024-04-25T00:36:43.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.886+0000 7ff6bf7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff6c010fea0 0x7ff6c01ac8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:43.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.886+0000 7ff6bf7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff6c010fea0 0x7ff6c01ac8b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44094/0 (socket says 172.21.15.32:44094) 2024-04-25T00:36:43.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.886+0000 7ff6bf7fe640 1 -- 172.21.15.32:0/1165248343 learned_addr learned my addr 172.21.15.32:0/1165248343 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:43.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.887+0000 7ff6bffff640 1 --2- 172.21.15.32:0/1165248343 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff6c01a6fa0 0x7ff6c01a7420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:43.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.887+0000 7ff6bf7fe640 1 -- 172.21.15.32:0/1165248343 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff6c01acdf0 msgr2=0x7ff6c01a6a60 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:36:43.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.887+0000 7ff6bf7fe640 1 --2- 172.21.15.32:0/1165248343 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff6c01acdf0 0x7ff6c01a6a60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:43.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.887+0000 7ff6bf7fe640 1 -- 172.21.15.32:0/1165248343 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff6c01a6fa0 msgr2=0x7ff6c01a7420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:43.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.887+0000 7ff6bf7fe640 1 --2- 172.21.15.32:0/1165248343 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff6c01a6fa0 0x7ff6c01a7420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:43.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.887+0000 7ff6bf7fe640 1 -- 172.21.15.32:0/1165248343 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff6c0110700 con 0x7ff6c010fea0 2024-04-25T00:36:43.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.888+0000 7ff6bf7fe640 1 --2- 172.21.15.32:0/1165248343 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff6c010fea0 0x7ff6c01ac8b0 secure :-1 s=READY pgs=611 cs=0 l=1 rev1=1 crypto rx=0x7ff6b400b4a0 tx=0x7ff6b400b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:43.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.891+0000 7ff6bcff9640 1 -- 172.21.15.32:0/1165248343 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff6b4014070 con 0x7ff6c010fea0 2024-04-25T00:36:43.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.891+0000 7ff6bcff9640 1 -- 172.21.15.32:0/1165248343 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff6b4010af0 con 0x7ff6c010fea0 2024-04-25T00:36:43.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.891+0000 7ff6bcff9640 1 -- 172.21.15.32:0/1165248343 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff6b4017cc0 con 0x7ff6c010fea0 2024-04-25T00:36:43.897 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.893+0000 7ff6c6134640 1 -- 172.21.15.32:0/1165248343 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff6c01109f0 con 0x7ff6c010fea0 2024-04-25T00:36:43.897 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.893+0000 7ff6c6134640 1 -- 172.21.15.32:0/1165248343 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff6c0110ec0 con 0x7ff6c010fea0 2024-04-25T00:36:43.899 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.898+0000 7ff6c6134640 1 -- 172.21.15.32:0/1165248343 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff6c006daf0 con 0x7ff6c010fea0 2024-04-25T00:36:43.904 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.902+0000 7ff6bcff9640 1 -- 172.21.15.32:0/1165248343 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7ff6b4017070 con 0x7ff6c010fea0 2024-04-25T00:36:43.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.905+0000 7ff6bcff9640 1 --2- 172.21.15.32:0/1165248343 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7ff6a80a2f00 0x7ff6a80a53c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:43.908 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.906+0000 7ff6bcff9640 1 -- 172.21.15.32:0/1165248343 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7ff6b40cf040 con 0x7ff6c010fea0 2024-04-25T00:36:43.909 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.907+0000 7ff6beffd640 1 --2- 172.21.15.32:0/1165248343 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7ff6a80a2f00 0x7ff6a80a53c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:43.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.908+0000 7ff6beffd640 1 --2- 172.21.15.32:0/1165248343 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7ff6a80a2f00 0x7ff6a80a53c0 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7ff6b0009670 tx=0x7ff6b0006bd0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:43.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:43.919+0000 7ff6bcff9640 1 -- 172.21.15.32:0/1165248343 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7ff6b401e030 con 0x7ff6c010fea0 2024-04-25T00:36:44.497 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:44.497+0000 7ff6c6134640 1 -- 172.21.15.32:0/1165248343 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7ff6c0068560 con 0x7ff6c010fea0 2024-04-25T00:36:44.501 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:44.501+0000 7ff6bcff9640 1 -- 172.21.15.32:0/1165248343 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 27 v27) ==== 94+0+11879 (secure 0 0 0) 0x7ff6b409b9e0 con 0x7ff6c010fea0 2024-04-25T00:36:44.506 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:36:44.506 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":27,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14643,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6839/187492674","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6838","nonce":187492674},{"type":"v1","addr":"172.21.15.32:6839","nonce":187492674}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14655,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6841/1948785545","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6840","nonce":1948785545},{"type":"v1","addr":"172.21.15.32:6841","nonce":1948785545}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":27,"flags":306,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":true},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-25T00:30:47.866875+0000","modified":"2024-04-25T00:36:12.357299+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24506,"mds_1":24481,"mds_2":24476},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24443":{"gid":24443,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24445":{"gid":24445,"name":"f","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24457":{"gid":24457,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24476":{"gid":24476,"name":"h","rank":2,"incarnation":22,"state":"up:active","state_seq":12,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24481":{"gid":24481,"name":"i","rank":1,"incarnation":20,"state":"up:active","state_seq":11,"addr":"172.21.15.130:6839/4267125809","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":4267125809},{"type":"v1","addr":"172.21.15.130:6839","nonce":4267125809}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24506":{"gid":24506,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.121:6839/258444853","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6838","nonce":258444853},{"type":"v1","addr":"172.21.15.121:6839","nonce":258444853}]},"join_fscid":-1,"export_targets":[1,2],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24506,"qdb_cluster":[24476,24481,24506]},"id":1}]} 2024-04-25T00:36:44.506 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 27 2024-04-25T00:36:44.508 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:44.507+0000 7ff6a67fc640 1 -- 172.21.15.32:0/1165248343 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7ff6a80a2f00 msgr2=0x7ff6a80a53c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:44.508 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:44.507+0000 7ff6a67fc640 1 --2- 172.21.15.32:0/1165248343 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7ff6a80a2f00 0x7ff6a80a53c0 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7ff6b0009670 tx=0x7ff6b0006bd0 comp rx=0 tx=0).stop 2024-04-25T00:36:44.508 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:44.507+0000 7ff6a67fc640 1 -- 172.21.15.32:0/1165248343 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff6c010fea0 msgr2=0x7ff6c01ac8b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:44.509 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:44.507+0000 7ff6a67fc640 1 --2- 172.21.15.32:0/1165248343 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff6c010fea0 0x7ff6c01ac8b0 secure :-1 s=READY pgs=611 cs=0 l=1 rev1=1 crypto rx=0x7ff6b400b4a0 tx=0x7ff6b400b960 comp rx=0 tx=0).stop 2024-04-25T00:36:44.509 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:44.508+0000 7ff6a67fc640 1 -- 172.21.15.32:0/1165248343 shutdown_connections 2024-04-25T00:36:44.509 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:44.508+0000 7ff6a67fc640 1 --2- 172.21.15.32:0/1165248343 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7ff6a80a2f00 0x7ff6a80a53c0 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:44.509 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:44.508+0000 7ff6a67fc640 1 --2- 172.21.15.32:0/1165248343 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff6c01a6fa0 0x7ff6c01a7420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:44.509 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:44.508+0000 7ff6a67fc640 1 --2- 172.21.15.32:0/1165248343 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff6c01acdf0 0x7ff6c01a6a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:44.509 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:44.508+0000 7ff6a67fc640 1 --2- 172.21.15.32:0/1165248343 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff6c010fea0 0x7ff6c01ac8b0 unknown :-1 s=CLOSED pgs=611 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:44.509 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:44.508+0000 7ff6a67fc640 1 -- 172.21.15.32:0/1165248343 >> 172.21.15.32:0/1165248343 conn(0x7ff6c006c860 msgr2=0x7ff6c0072bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:44.509 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:44.508+0000 7ff6a67fc640 1 -- 172.21.15.32:0/1165248343 shutdown_connections 2024-04-25T00:36:44.509 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:44.508+0000 7ff6a67fc640 1 -- 172.21.15.32:0/1165248343 wait complete. 2024-04-25T00:36:44.553 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-25T00:36:45.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:45 smithi121 ceph-mon[32231]: pgmap v106: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 2.1 MiB/s rd, 673 MiB/s wr, 1.69k op/s 2024-04-25T00:36:45.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:45 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1165248343' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:36:45.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:45 smithi130 ceph-mon[32525]: pgmap v106: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 2.1 MiB/s rd, 673 MiB/s wr, 1.69k op/s 2024-04-25T00:36:45.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:45 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1165248343' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:36:45.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:45 smithi032 ceph-mon[29357]: pgmap v106: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 2.1 MiB/s rd, 673 MiB/s wr, 1.69k op/s 2024-04-25T00:36:45.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:45 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1165248343' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:36:47.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:47 smithi121 ceph-mon[32231]: pgmap v107: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 2.6 MiB/s rd, 803 MiB/s wr, 2.05k op/s 2024-04-25T00:36:47.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:47 smithi130 ceph-mon[32525]: pgmap v107: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 2.6 MiB/s rd, 803 MiB/s wr, 2.05k op/s 2024-04-25T00:36:47.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:47 smithi032 ceph-mon[29357]: pgmap v107: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 2.6 MiB/s rd, 803 MiB/s wr, 2.05k op/s 2024-04-25T00:36:48.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.581+0000 7f77c6251640 1 -- 172.21.15.32:0/2813190527 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f77c0107c80 msgr2=0x7f77c01080c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:48.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.581+0000 7f77c6251640 1 --2- 172.21.15.32:0/2813190527 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f77c0107c80 0x7f77c01080c0 secure :-1 s=READY pgs=612 cs=0 l=1 rev1=1 crypto rx=0x7f77b8009cd0 tx=0x7f77b802a930 comp rx=0 tx=0).stop 2024-04-25T00:36:48.590 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.589+0000 7f77c6251640 1 -- 172.21.15.32:0/2813190527 shutdown_connections 2024-04-25T00:36:48.591 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.589+0000 7f77c6251640 1 --2- 172.21.15.32:0/2813190527 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f77c0108600 0x7f77c0114b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.591 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.589+0000 7f77c6251640 1 --2- 172.21.15.32:0/2813190527 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f77c0107c80 0x7f77c01080c0 unknown :-1 s=CLOSED pgs=612 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.591 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.589+0000 7f77c6251640 1 --2- 172.21.15.32:0/2813190527 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f77c010dcd0 0x7f77c010e0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.591 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.589+0000 7f77c6251640 1 -- 172.21.15.32:0/2813190527 >> 172.21.15.32:0/2813190527 conn(0x7f77c0077560 msgr2=0x7f77c0106560 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:48.597 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.596+0000 7f77c6251640 1 -- 172.21.15.32:0/2813190527 shutdown_connections 2024-04-25T00:36:48.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.599+0000 7f77c6251640 1 -- 172.21.15.32:0/2813190527 wait complete. 2024-04-25T00:36:48.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.600+0000 7f77c6251640 1 Processor -- start 2024-04-25T00:36:48.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.605+0000 7f77c6251640 1 -- start start 2024-04-25T00:36:48.606 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.605+0000 7f77c6251640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f77c0107c80 0x7f77c019e620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:48.607 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.606+0000 7f77c6251640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f77c0108600 0x7f77c019eb60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:48.607 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.606+0000 7f77c6251640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f77c010dcd0 0x7f77c01a2ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:48.607 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.606+0000 7f77c6251640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f77c01180c0 con 0x7f77c0108600 2024-04-25T00:36:48.607 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.606+0000 7f77c6251640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f77c0117f40 con 0x7f77c0107c80 2024-04-25T00:36:48.607 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.606+0000 7f77c6251640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f77c0118240 con 0x7f77c010dcd0 2024-04-25T00:36:48.608 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.608+0000 7f77c524f640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f77c0107c80 0x7f77c019e620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:48.612 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.608+0000 7f77c4a4e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f77c0108600 0x7f77c019eb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:48.612 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.608+0000 7f77c4a4e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f77c0108600 0x7f77c019eb60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44120/0 (socket says 172.21.15.32:44120) 2024-04-25T00:36:48.612 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.608+0000 7f77c4a4e640 1 -- 172.21.15.32:0/402161466 learned_addr learned my addr 172.21.15.32:0/402161466 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:48.613 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.608+0000 7f77c5a50640 1 --2- 172.21.15.32:0/402161466 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f77c010dcd0 0x7f77c01a2ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:48.613 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.610+0000 7f77c4a4e640 1 -- 172.21.15.32:0/402161466 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f77c010dcd0 msgr2=0x7f77c01a2ee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:48.613 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.610+0000 7f77c4a4e640 1 --2- 172.21.15.32:0/402161466 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f77c010dcd0 0x7f77c01a2ee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.613 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.610+0000 7f77c4a4e640 1 -- 172.21.15.32:0/402161466 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f77c0107c80 msgr2=0x7f77c019e620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:48.613 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.610+0000 7f77c4a4e640 1 --2- 172.21.15.32:0/402161466 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f77c0107c80 0x7f77c019e620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.613 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.610+0000 7f77c4a4e640 1 -- 172.21.15.32:0/402161466 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f77c01a3610 con 0x7f77c0108600 2024-04-25T00:36:48.613 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.610+0000 7f77c5a50640 1 --2- 172.21.15.32:0/402161466 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f77c010dcd0 0x7f77c01a2ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:36:48.613 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.611+0000 7f77c4a4e640 1 --2- 172.21.15.32:0/402161466 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f77c0108600 0x7f77c019eb60 secure :-1 s=READY pgs=613 cs=0 l=1 rev1=1 crypto rx=0x7f77b8030870 tx=0x7f77b80308a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:48.623 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.617+0000 7f77b67fc640 1 -- 172.21.15.32:0/402161466 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f77b8030c40 con 0x7f77c0108600 2024-04-25T00:36:48.623 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.617+0000 7f77b67fc640 1 -- 172.21.15.32:0/402161466 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f77b802d070 con 0x7f77c0108600 2024-04-25T00:36:48.623 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.617+0000 7f77b67fc640 1 -- 172.21.15.32:0/402161466 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f77b803a420 con 0x7f77c0108600 2024-04-25T00:36:48.625 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.624+0000 7f77c6251640 1 -- 172.21.15.32:0/402161466 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f77c01a38a0 con 0x7f77c0108600 2024-04-25T00:36:48.625 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.625+0000 7f77c6251640 1 -- 172.21.15.32:0/402161466 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f77c01ab190 con 0x7f77c0108600 2024-04-25T00:36:48.630 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.629+0000 7f77ca1c4740 1 Processor -- start 2024-04-25T00:36:48.635 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.630+0000 7f77ca1c4740 1 -- start start 2024-04-25T00:36:48.636 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.636+0000 7f77ca1c4740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5622134ca710 0x5622134caaf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:48.643 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.638+0000 7f77ca1c4740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5622134cb030 0x5622134d51c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:48.643 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.638+0000 7f77ca1c4740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134d5700 0x5622134d7af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:48.643 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.638+0000 7f77ca1c4740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x56221342f670 con 0x5622134cb030 2024-04-25T00:36:48.643 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.638+0000 7f77ca1c4740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5622133d7970 con 0x5622134d5700 2024-04-25T00:36:48.643 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.638+0000 7f77ca1c4740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x56221342b630 con 0x5622134ca710 2024-04-25T00:36:48.643 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.638+0000 7f77b67fc640 1 -- 172.21.15.32:0/402161466 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f77b8004050 con 0x7f77c0108600 2024-04-25T00:36:48.643 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.639+0000 7f77c5a50640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5622134ca710 0x5622134caaf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:48.643 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.640+0000 7f77c524f640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5622134cb030 0x5622134d51c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:48.644 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.640+0000 7f77c4a4e640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134d5700 0x5622134d7af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:48.644 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.640+0000 7f77c5a50640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5622134ca710 0x5622134caaf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:56374/0 (socket says 172.21.15.32:56374) 2024-04-25T00:36:48.644 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.640+0000 7f77c5a50640 1 -- 172.21.15.32:0/1086467562 learned_addr learned my addr 172.21.15.32:0/1086467562 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:48.644 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.641+0000 7f77c5a50640 1 -- 172.21.15.32:0/1086467562 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134d5700 msgr2=0x5622134d7af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:48.644 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.642+0000 7f77c5a50640 1 --2- 172.21.15.32:0/1086467562 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134d5700 0x5622134d7af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.644 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.642+0000 7f77c5a50640 1 -- 172.21.15.32:0/1086467562 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5622134cb030 msgr2=0x5622134d51c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:48.644 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.642+0000 7f77c5a50640 1 --2- 172.21.15.32:0/1086467562 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5622134cb030 0x5622134d51c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.644 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.642+0000 7f77c5a50640 1 -- 172.21.15.32:0/1086467562 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5622133fb400 con 0x5622134ca710 2024-04-25T00:36:48.644 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.642+0000 7f77c5a50640 1 --2- 172.21.15.32:0/1086467562 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5622134ca710 0x5622134caaf0 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f77bc00bae0 tx=0x7f77bc00d340 comp rx=0 tx=0).ready entity=mon.1 client_cookie=18c93906a2596c1a server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:48.645 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.643+0000 7f77b67fc640 1 --2- 172.21.15.32:0/402161466 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f77ac0a2f00 0x7f77ac0a53c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:48.645 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.643+0000 7f77b67fc640 1 -- 172.21.15.32:0/402161466 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f77b80f0cf0 con 0x7f77c0108600 2024-04-25T00:36:48.645 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.644+0000 7f77ab7fe640 1 -- 172.21.15.32:0/1086467562 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f77bc017420 con 0x5622134ca710 2024-04-25T00:36:48.645 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.644+0000 7f77ab7fe640 1 -- 172.21.15.32:0/1086467562 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f77bc017aa0 con 0x5622134ca710 2024-04-25T00:36:48.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.645+0000 7f77ca1c4740 1 -- 172.21.15.32:0/1086467562 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5622134ca710 msgr2=0x5622134caaf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:48.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.645+0000 7f77ca1c4740 1 --2- 172.21.15.32:0/1086467562 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5622134ca710 0x5622134caaf0 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f77bc00bae0 tx=0x7f77bc00d340 comp rx=0 tx=0).stop 2024-04-25T00:36:48.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.645+0000 7f77ca1c4740 1 -- 172.21.15.32:0/1086467562 shutdown_connections 2024-04-25T00:36:48.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.645+0000 7f77ca1c4740 1 --2- 172.21.15.32:0/1086467562 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134d5700 0x5622134d7af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.645+0000 7f77ca1c4740 1 --2- 172.21.15.32:0/1086467562 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5622134cb030 0x5622134d51c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.646+0000 7f77ca1c4740 1 --2- 172.21.15.32:0/1086467562 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5622134ca710 0x5622134caaf0 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.646+0000 7f77ca1c4740 1 -- 172.21.15.32:0/1086467562 >> 172.21.15.32:0/1086467562 conn(0x5622134c6a30 msgr2=0x5622134c6e10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:48.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.652+0000 7f77c524f640 1 --2- 172.21.15.32:0/402161466 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f77ac0a2f00 0x7f77ac0a53c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:48.653 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.653+0000 7f77ca1c4740 1 -- 172.21.15.32:0/1086467562 shutdown_connections 2024-04-25T00:36:48.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.659+0000 7f77ca1c4740 1 -- 172.21.15.32:0/1086467562 wait complete. 2024-04-25T00:36:48.661 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.660+0000 7f77ca1c4740 1 -- start start 2024-04-25T00:36:48.661 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.660+0000 7f77ca1c4740 1 Processor -- start 2024-04-25T00:36:48.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.661+0000 7f77ca1c4740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5622134cb030 0x5622134d4df0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:48.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.661+0000 7f77ca1c4740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134d5700 0x5622134d20e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:48.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.662+0000 7f77ca1c4740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5622134d2620 0x5622134d2aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:48.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.662+0000 7f77ca1c4740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5622134d8540 con 0x5622134cb030 2024-04-25T00:36:48.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.662+0000 7f77ca1c4740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5622133eb050 con 0x5622134d5700 2024-04-25T00:36:48.663 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.662+0000 7f77ca1c4740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5622134d86c0 con 0x5622134d2620 2024-04-25T00:36:48.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.662+0000 7f77c4a4e640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134d5700 0x5622134d20e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:48.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.663+0000 7f77c4a4e640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134d5700 0x5622134d20e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:47642/0 (socket says 172.21.15.32:47642) 2024-04-25T00:36:48.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.663+0000 7f77c4a4e640 1 -- 172.21.15.32:0/3615093342 learned_addr learned my addr 172.21.15.32:0/3615093342 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:48.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.663+0000 7f77ca1c4740 10 client.0 _ll_register_callbacks cb 0x562213142be0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:36:48.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.664+0000 7f77c5a50640 1 --2- 172.21.15.32:0/3615093342 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5622134d2620 0x5622134d2aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:48.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.664+0000 7f77c4a4e640 1 -- 172.21.15.32:0/3615093342 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5622134d2620 msgr2=0x5622134d2aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:48.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.664+0000 7f77c4a4e640 1 --2- 172.21.15.32:0/3615093342 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5622134d2620 0x5622134d2aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.664+0000 7f77c4a4e640 1 -- 172.21.15.32:0/3615093342 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5622134cb030 msgr2=0x5622134d4df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:48.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.664+0000 7f77c4a4e640 1 --2- 172.21.15.32:0/3615093342 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5622134cb030 0x5622134d4df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.664+0000 7f77c4a4e640 1 -- 172.21.15.32:0/3615093342 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5622133bcdc0 con 0x5622134d5700 2024-04-25T00:36:48.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.665+0000 7f77c4a4e640 1 --2- 172.21.15.32:0/3615093342 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134d5700 0x5622134d20e0 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f77b80b9370 tx=0x7f77b8005f60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9a54c166fb7a069f server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:48.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.667+0000 7f77c524f640 1 --2- 172.21.15.32:0/402161466 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f77ac0a2f00 0x7f77ac0a53c0 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f77b0006fd0 tx=0x7f77b0008040 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:48.672 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.668+0000 7f77a8ff9640 10 client.0 ms_handle_connect on v2:172.21.15.121:3300/0 2024-04-25T00:36:48.672 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.668+0000 7f77a8ff9640 1 -- 172.21.15.32:0/3615093342 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f77b80bca50 con 0x5622134d5700 2024-04-25T00:36:48.672 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.668+0000 7f77ca1c4740 1 -- 172.21.15.32:0/3615093342 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x56221343cbb0 con 0x5622134d5700 2024-04-25T00:36:48.672 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.668+0000 7f77a8ff9640 1 -- 172.21.15.32:0/3615093342 <== mon.2 v2:172.21.15.121:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f77b80b9580 con 0x5622134d5700 2024-04-25T00:36:48.672 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.670+0000 7f77a8ff9640 1 -- 172.21.15.32:0/3615093342 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f77b80bc870 con 0x5622134d5700 2024-04-25T00:36:48.672 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.670+0000 7f77ca1c4740 10 client.25685 fetch_fsmap learned FSMap version 27 2024-04-25T00:36:48.672 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.670+0000 7f77ca1c4740 1 -- 172.21.15.32:0/3615093342 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x5622131433f0 con 0x5622134d5700 2024-04-25T00:36:48.673 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.671+0000 7f77a8ff9640 1 -- 172.21.15.32:0/3615093342 <== mon.2 v2:172.21.15.121:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x56221343cbb0 con 0x5622134d5700 2024-04-25T00:36:48.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.672+0000 7f77a8ff9640 1 -- 172.21.15.32:0/3615093342 <== mon.2 v2:172.21.15.121:3300/0 5 ==== fsmap(e 27) ==== 8354+0+0 (secure 0 0 0) 0x7f77b80db020 con 0x5622134d5700 2024-04-25T00:36:48.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.672+0000 7f77ca1c4740 10 client.25685 fetch_fsmap finished waiting for FSMap version 27 2024-04-25T00:36:48.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.672+0000 7f77ca1c4740 10 client.25685 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-25T00:36:48.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.672+0000 7f77ca1c4740 20 client.25685 populate_metadata read hostname 'smithi032' 2024-04-25T00:36:48.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.673+0000 7f77ca1c4740 1 --2- 172.21.15.32:0/3615093342 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x5622134e5720 0x5622134e7b30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:48.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.673+0000 7f77ca1c4740 4 client.25685 mds_command: new command op to 24476 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:36:48.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.673+0000 7f77ca1c4740 1 -- 172.21.15.32:0/3615093342 --> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x562213436f00 con 0x5622134e5720 2024-04-25T00:36:48.678 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.677+0000 7f77c524f640 1 --2- 172.21.15.32:0/3615093342 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x5622134e5720 0x5622134e7b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:48.713 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.710+0000 7f77c524f640 1 --2- 172.21.15.32:0/3615093342 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x5622134e5720 0x5622134e7b30 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=3749975e23bbd8d4 server_cookie=88f0d557d2cbc9cd in_seq=0 out_seq=0 2024-04-25T00:36:48.713 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.710+0000 7f77a8ff9640 10 client.25685 ms_handle_connect on v2:172.21.15.121:6836/3456022860 2024-04-25T00:36:48.713 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.712+0000 7f77a8ff9640 1 -- 172.21.15.32:0/3615093342 <== mds.2 v2:172.21.15.121:6836/3456022860 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x562213436f00 con 0x5622134e5720 2024-04-25T00:36:48.713 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.712+0000 7f77a8ff9640 10 client.25685 handle_command_reply: tid=0 2024-04-25T00:36:48.713 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.712+0000 7f77ca1c4740 1 client.25685 shutdown 2024-04-25T00:36:48.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.722+0000 7f77ca1c4740 1 -- 172.21.15.32:0/3615093342 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134d5700 msgr2=0x5622134d20e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:48.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.722+0000 7f77ca1c4740 1 --2- 172.21.15.32:0/3615093342 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134d5700 0x5622134d20e0 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f77b80b9370 tx=0x7f77b8005f60 comp rx=0 tx=0).stop 2024-04-25T00:36:48.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.724+0000 7f77ca1c4740 1 -- 172.21.15.32:0/3615093342 shutdown_connections 2024-04-25T00:36:48.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.724+0000 7f77ca1c4740 1 --2- 172.21.15.32:0/3615093342 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x5622134e5720 0x5622134e7b30 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.724 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.724+0000 7f77ca1c4740 1 --2- 172.21.15.32:0/3615093342 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5622134d2620 0x5622134d2aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.724+0000 7f77ca1c4740 1 --2- 172.21.15.32:0/3615093342 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134d5700 0x5622134d20e0 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.724+0000 7f77ca1c4740 1 --2- 172.21.15.32:0/3615093342 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5622134cb030 0x5622134d4df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.724+0000 7f77ca1c4740 1 -- 172.21.15.32:0/3615093342 >> 172.21.15.32:0/3615093342 conn(0x5622134c6a30 msgr2=0x5622134cbfa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:48.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.724+0000 7f77a8ff9640 0 client.25685 ms_handle_reset on v2:172.21.15.121:6836/3456022860 2024-04-25T00:36:48.728 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.727+0000 7f77ca1c4740 1 -- 172.21.15.32:0/3615093342 shutdown_connections 2024-04-25T00:36:48.728 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.727+0000 7f77ca1c4740 1 -- 172.21.15.32:0/3615093342 wait complete. 2024-04-25T00:36:48.728 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.727+0000 7f77ca1c4740 20 client.25685 trim_cache size 0 max 16384 2024-04-25T00:36:48.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.763+0000 7f77ca1c4740 1 Processor -- start 2024-04-25T00:36:48.764 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.763+0000 7f77ca1c4740 1 -- start start 2024-04-25T00:36:48.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.763+0000 7f77ca1c4740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134cb030 0x5622134e1450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:48.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.763+0000 7f77ca1c4740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5622134d2620 0x5622134cd750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:48.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.764+0000 7f77c524f640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134cb030 0x5622134e1450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:48.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.764+0000 7f77c524f640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134cb030 0x5622134e1450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:47644/0 (socket says 172.21.15.32:47644) 2024-04-25T00:36:48.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.764+0000 7f77c4a4e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5622134d2620 0x5622134cd750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:48.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.764+0000 7f77c524f640 1 -- 172.21.15.32:0/338856156 learned_addr learned my addr 172.21.15.32:0/338856156 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:48.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.764+0000 7f77ca1c4740 1 --2- 172.21.15.32:0/338856156 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5622134d5700 0x5622134d4920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:48.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.764+0000 7f77ca1c4740 1 -- 172.21.15.32:0/338856156 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f77c01b7920 con 0x5622134d2620 2024-04-25T00:36:48.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.764+0000 7f77ca1c4740 1 -- 172.21.15.32:0/338856156 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f77c01b51f0 con 0x5622134cb030 2024-04-25T00:36:48.765 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.764+0000 7f77ca1c4740 1 -- 172.21.15.32:0/338856156 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f77c01b30f0 con 0x5622134d5700 2024-04-25T00:36:48.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.765+0000 7f77c5a50640 1 --2- 172.21.15.32:0/338856156 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5622134d5700 0x5622134d4920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:48.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.765+0000 7f77c524f640 1 -- 172.21.15.32:0/338856156 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5622134d5700 msgr2=0x5622134d4920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:48.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.765+0000 7f77c524f640 1 --2- 172.21.15.32:0/338856156 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5622134d5700 0x5622134d4920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.765+0000 7f77c524f640 1 -- 172.21.15.32:0/338856156 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5622134d2620 msgr2=0x5622134cd750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:48.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.765+0000 7f77c524f640 1 --2- 172.21.15.32:0/338856156 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5622134d2620 0x5622134cd750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.765+0000 7f77c524f640 1 -- 172.21.15.32:0/338856156 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56221320d7d0 con 0x5622134cb030 2024-04-25T00:36:48.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.767+0000 7f77c524f640 1 --2- 172.21.15.32:0/338856156 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134cb030 0x5622134e1450 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f77b000c700 tx=0x7f77b0004750 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a6bda4a6be1583da server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:48.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.768+0000 7f77aaffd640 1 -- 172.21.15.32:0/338856156 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f77b0005970 con 0x5622134cb030 2024-04-25T00:36:48.768 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.768+0000 7f77aaffd640 1 -- 172.21.15.32:0/338856156 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f77b001c040 con 0x5622134cb030 2024-04-25T00:36:48.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.768+0000 7f77aaffd640 1 -- 172.21.15.32:0/338856156 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f77b00064d0 con 0x5622134cb030 2024-04-25T00:36:48.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.769+0000 7f77ca1c4740 1 -- 172.21.15.32:0/338856156 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134cb030 msgr2=0x5622134e1450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:48.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.769+0000 7f77ca1c4740 1 --2- 172.21.15.32:0/338856156 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134cb030 0x5622134e1450 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f77b000c700 tx=0x7f77b0004750 comp rx=0 tx=0).stop 2024-04-25T00:36:48.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.769+0000 7f77ca1c4740 1 -- 172.21.15.32:0/338856156 shutdown_connections 2024-04-25T00:36:48.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.769+0000 7f77ca1c4740 1 --2- 172.21.15.32:0/338856156 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5622134d5700 0x5622134d4920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.769+0000 7f77ca1c4740 1 --2- 172.21.15.32:0/338856156 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5622134d2620 0x5622134cd750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.769+0000 7f77ca1c4740 1 --2- 172.21.15.32:0/338856156 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134cb030 0x5622134e1450 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.770+0000 7f77ca1c4740 1 -- 172.21.15.32:0/338856156 >> 172.21.15.32:0/338856156 conn(0x5622134c6a30 msgr2=0x5622134cd040 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:48.776 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.776+0000 7f77ca1c4740 1 -- 172.21.15.32:0/338856156 shutdown_connections 2024-04-25T00:36:48.776 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.776+0000 7f77ca1c4740 1 -- 172.21.15.32:0/338856156 wait complete. 2024-04-25T00:36:48.777 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.777+0000 7f77ca1c4740 1 -- start start 2024-04-25T00:36:48.777 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.777+0000 7f77ca1c4740 1 Processor -- start 2024-04-25T00:36:48.778 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.778+0000 7f77ca1c4740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134cb030 0x5622134d5c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:48.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.778+0000 7f77ca1c4740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5622134d2620 0x5622134d6140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:48.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.778+0000 7f77c524f640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134cb030 0x5622134d5c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:48.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.778+0000 7f77c524f640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134cb030 0x5622134d5c00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:47646/0 (socket says 172.21.15.32:47646) 2024-04-25T00:36:48.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.779+0000 7f77c524f640 1 -- 172.21.15.32:0/2430312709 learned_addr learned my addr 172.21.15.32:0/2430312709 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:48.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.779+0000 7f77ca1c4740 1 --2- 172.21.15.32:0/2430312709 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5622134d5700 0x5622134dc960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:48.779 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.779+0000 7f77c4a4e640 1 --2- 172.21.15.32:0/2430312709 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5622134d2620 0x5622134d6140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:48.780 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.779+0000 7f77ca1c4740 1 -- 172.21.15.32:0/2430312709 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f77c01b36b0 con 0x5622134d5700 2024-04-25T00:36:48.782 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.779+0000 7f77ca1c4740 1 -- 172.21.15.32:0/2430312709 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f77c01b4b20 con 0x5622134cb030 2024-04-25T00:36:48.782 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.782+0000 7f77c5a50640 1 --2- 172.21.15.32:0/2430312709 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5622134d5700 0x5622134dc960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:48.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.782+0000 7f77ca1c4740 1 -- 172.21.15.32:0/2430312709 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f77c01b3d80 con 0x5622134d2620 2024-04-25T00:36:48.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.783+0000 7f77ca1c4740 10 client.0 _ll_register_callbacks cb 0x7f77b8009900 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:36:48.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.783+0000 7f77c524f640 1 -- 172.21.15.32:0/2430312709 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5622134d2620 msgr2=0x5622134d6140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:48.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.783+0000 7f77c524f640 1 --2- 172.21.15.32:0/2430312709 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5622134d2620 0x5622134d6140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.784 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.783+0000 7f77c524f640 1 -- 172.21.15.32:0/2430312709 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5622134d5700 msgr2=0x5622134dc960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:48.784 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.783+0000 7f77c524f640 1 --2- 172.21.15.32:0/2430312709 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5622134d5700 0x5622134dc960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.784 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.783+0000 7f77c524f640 1 -- 172.21.15.32:0/2430312709 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5622134d6a00 con 0x5622134cb030 2024-04-25T00:36:48.784 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.783+0000 7f77c524f640 1 --2- 172.21.15.32:0/2430312709 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134cb030 0x5622134d5c00 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f77b0015b80 tx=0x7f77b0015bb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a8e340179b7f47b8 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:48.785 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.784+0000 7f77a8ff9640 10 client.25691 ms_handle_connect on v2:172.21.15.121:3300/0 2024-04-25T00:36:48.785 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.784+0000 7f77a8ff9640 1 -- 172.21.15.32:0/2430312709 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f77b0005970 con 0x5622134cb030 2024-04-25T00:36:48.785 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.784+0000 7f77ca1c4740 1 -- 172.21.15.32:0/2430312709 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f77b001c040 con 0x5622134cb030 2024-04-25T00:36:48.785 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.784+0000 7f77a8ff9640 1 -- 172.21.15.32:0/2430312709 <== mon.2 v2:172.21.15.121:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f77b0005b10 con 0x5622134cb030 2024-04-25T00:36:48.785 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.785+0000 7f77a8ff9640 1 -- 172.21.15.32:0/2430312709 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f77b0005d10 con 0x5622134cb030 2024-04-25T00:36:48.786 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.785+0000 7f77ca1c4740 10 client.25691 fetch_fsmap learned FSMap version 27 2024-04-25T00:36:48.786 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.786+0000 7f77ca1c4740 1 -- 172.21.15.32:0/2430312709 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({fsmap=27}) -- 0x5622134dde40 con 0x5622134cb030 2024-04-25T00:36:48.787 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.786+0000 7f77a8ff9640 1 -- 172.21.15.32:0/2430312709 <== mon.2 v2:172.21.15.121:3300/0 4 ==== fsmap(e 27) ==== 8354+0+0 (secure 0 0 0) 0x7f77b00060d0 con 0x5622134cb030 2024-04-25T00:36:48.787 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.786+0000 7f77ca1c4740 10 client.25691 fetch_fsmap finished waiting for FSMap version 27 2024-04-25T00:36:48.787 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.786+0000 7f77ca1c4740 10 client.25691 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-25T00:36:48.788 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.786+0000 7f77ca1c4740 20 client.25691 populate_metadata read hostname 'smithi032' 2024-04-25T00:36:48.788 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.787+0000 7f77ca1c4740 1 --2- 172.21.15.32:0/2430312709 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x56221354bc80 0x56221354e070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:48.788 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.787+0000 7f77ca1c4740 4 client.25691 mds_command: new command op to 24476 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-25T00:36:48.788 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.787+0000 7f77ca1c4740 1 -- 172.21.15.32:0/2430312709 --> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] -- command(tid 0: {"prefix": "damage ls"}) -- 0x7f77b001c040 con 0x56221354bc80 2024-04-25T00:36:48.788 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.787+0000 7f77c4a4e640 1 --2- 172.21.15.32:0/2430312709 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x56221354bc80 0x56221354e070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:48.788 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.788+0000 7f77c4a4e640 1 --2- 172.21.15.32:0/2430312709 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x56221354bc80 0x56221354e070 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=4f359a43537079dc server_cookie=6f041f61b341b0cd in_seq=0 out_seq=0 2024-04-25T00:36:48.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.789+0000 7f77a8ff9640 10 client.25691 ms_handle_connect on v2:172.21.15.121:6836/3456022860 2024-04-25T00:36:48.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.789+0000 7f77a8ff9640 1 -- 172.21.15.32:0/2430312709 <== mds.2 v2:172.21.15.121:6836/3456022860 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x7f77b001c040 con 0x56221354bc80 2024-04-25T00:36:48.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.789+0000 7f77a8ff9640 10 client.25691 handle_command_reply: tid=0 2024-04-25T00:36:48.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.789+0000 7f77ca1c4740 1 client.25691 shutdown 2024-04-25T00:36:48.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.790+0000 7f77ca1c4740 1 -- 172.21.15.32:0/2430312709 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134cb030 msgr2=0x5622134d5c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:48.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.790+0000 7f77ca1c4740 1 --2- 172.21.15.32:0/2430312709 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134cb030 0x5622134d5c00 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f77b0015b80 tx=0x7f77b0015bb0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.790+0000 7f77ca1c4740 1 -- 172.21.15.32:0/2430312709 shutdown_connections 2024-04-25T00:36:48.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.790+0000 7f77ca1c4740 1 --2- 172.21.15.32:0/2430312709 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x56221354bc80 0x56221354e070 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.791+0000 7f77ca1c4740 1 --2- 172.21.15.32:0/2430312709 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5622134d5700 0x5622134dc960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.792 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.791+0000 7f77a8ff9640 0 client.25691 ms_handle_reset on v2:172.21.15.121:6836/3456022860 2024-04-25T00:36:48.792 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.791+0000 7f77ca1c4740 1 --2- 172.21.15.32:0/2430312709 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5622134d2620 0x5622134d6140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.792 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.791+0000 7f77ca1c4740 1 --2- 172.21.15.32:0/2430312709 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5622134cb030 0x5622134d5c00 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.792 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.791+0000 7f77ca1c4740 1 -- 172.21.15.32:0/2430312709 >> 172.21.15.32:0/2430312709 conn(0x5622134c6a30 msgr2=0x5622134d09d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:48.799 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.799+0000 7f77ca1c4740 1 -- 172.21.15.32:0/2430312709 shutdown_connections 2024-04-25T00:36:48.799 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.799+0000 7f77ca1c4740 1 -- 172.21.15.32:0/2430312709 wait complete. 2024-04-25T00:36:48.804 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.803+0000 7f77ca1c4740 20 client.25691 trim_cache size 0 max 16384 2024-04-25T00:36:48.806 INFO:teuthology.orchestra.run.smithi032.stdout:[] 2024-04-25T00:36:48.808 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.805+0000 7f77c6251640 1 -- 172.21.15.32:0/402161466 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f77ac0a2f00 msgr2=0x7f77ac0a53c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:48.808 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.805+0000 7f77c6251640 1 --2- 172.21.15.32:0/402161466 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f77ac0a2f00 0x7f77ac0a53c0 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f77b0006fd0 tx=0x7f77b0008040 comp rx=0 tx=0).stop 2024-04-25T00:36:48.808 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.805+0000 7f77c6251640 1 -- 172.21.15.32:0/402161466 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f77c0108600 msgr2=0x7f77c019eb60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:48.808 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.805+0000 7f77c6251640 1 --2- 172.21.15.32:0/402161466 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f77c0108600 0x7f77c019eb60 secure :-1 s=READY pgs=613 cs=0 l=1 rev1=1 crypto rx=0x7f77b8030870 tx=0x7f77b80308a0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.808 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.806+0000 7f77c6251640 1 -- 172.21.15.32:0/402161466 shutdown_connections 2024-04-25T00:36:48.809 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.806+0000 7f77c6251640 1 --2- 172.21.15.32:0/402161466 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f77ac0a2f00 0x7f77ac0a53c0 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.809 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.806+0000 7f77c6251640 1 --2- 172.21.15.32:0/402161466 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f77c010dcd0 0x7f77c01a2ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.810 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.806+0000 7f77c6251640 1 --2- 172.21.15.32:0/402161466 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f77c0108600 0x7f77c019eb60 unknown :-1 s=CLOSED pgs=613 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.810 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.806+0000 7f77c6251640 1 --2- 172.21.15.32:0/402161466 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f77c0107c80 0x7f77c019e620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:48.810 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.806+0000 7f77c6251640 1 -- 172.21.15.32:0/402161466 >> 172.21.15.32:0/402161466 conn(0x7f77c0077560 msgr2=0x7f77c0078520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:48.810 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.807+0000 7f77c6251640 1 -- 172.21.15.32:0/402161466 shutdown_connections 2024-04-25T00:36:48.810 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:48.807+0000 7f77c6251640 1 -- 172.21.15.32:0/402161466 wait complete. 2024-04-25T00:36:48.889 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-25T00:36:49.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.171+0000 7fed3759e640 1 -- 172.21.15.32:0/926097709 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fed38108740 msgr2=0x7fed38108ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:49.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.171+0000 7fed3759e640 1 --2- 172.21.15.32:0/926097709 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fed38108740 0x7fed38108ba0 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7fed2c009900 tx=0x7fed2c02a4b0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.172+0000 7fed3759e640 1 -- 172.21.15.32:0/926097709 shutdown_connections 2024-04-25T00:36:49.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.172+0000 7fed3759e640 1 --2- 172.21.15.32:0/926097709 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fed381091c0 0x7fed3811cf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.174 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.172+0000 7fed3759e640 1 --2- 172.21.15.32:0/926097709 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fed38108740 0x7fed38108ba0 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.175 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.172+0000 7fed3759e640 1 --2- 172.21.15.32:0/926097709 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fed3810fde0 0x7fed381101c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.175 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.172+0000 7fed3759e640 1 -- 172.21.15.32:0/926097709 >> 172.21.15.32:0/926097709 conn(0x7fed3806c6c0 msgr2=0x7fed3806cad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:49.177 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.174+0000 7fed3759e640 1 -- 172.21.15.32:0/926097709 shutdown_connections 2024-04-25T00:36:49.177 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.175+0000 7fed3759e640 1 -- 172.21.15.32:0/926097709 wait complete. 2024-04-25T00:36:49.177 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.176+0000 7fed3759e640 1 Processor -- start 2024-04-25T00:36:49.177 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.176+0000 7fed3759e640 1 -- start start 2024-04-25T00:36:49.177 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.176+0000 7fed3759e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fed38108740 0x7fed3811caf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.177 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.176+0000 7fed3759e640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fed381091c0 0x7fed38115b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.177 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.176+0000 7fed3759e640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fed3810fde0 0x7fed381160b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.176+0000 7fed3759e640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fed38120840 con 0x7fed38108740 2024-04-25T00:36:49.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.176+0000 7fed3759e640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fed381206c0 con 0x7fed381091c0 2024-04-25T00:36:49.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.176+0000 7fed3759e640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fed381209c0 con 0x7fed3810fde0 2024-04-25T00:36:49.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.177+0000 7fed36d9d640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fed3810fde0 0x7fed381160b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.177+0000 7fed35d9b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fed381091c0 0x7fed38115b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.177+0000 7fed35d9b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fed381091c0 0x7fed38115b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:47666/0 (socket says 172.21.15.32:47666) 2024-04-25T00:36:49.179 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.177+0000 7fed35d9b640 1 -- 172.21.15.32:0/3586290468 learned_addr learned my addr 172.21.15.32:0/3586290468 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:49.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.179+0000 7fed35d9b640 1 -- 172.21.15.32:0/3586290468 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fed3810fde0 msgr2=0x7fed381160b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:49.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.179+0000 7fed35d9b640 1 --2- 172.21.15.32:0/3586290468 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fed3810fde0 0x7fed381160b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.179+0000 7fed35d9b640 1 -- 172.21.15.32:0/3586290468 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fed38108740 msgr2=0x7fed3811caf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:49.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.179+0000 7fed35d9b640 1 --2- 172.21.15.32:0/3586290468 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fed38108740 0x7fed3811caf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.179+0000 7fed35d9b640 1 -- 172.21.15.32:0/3586290468 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fed38116640 con 0x7fed381091c0 2024-04-25T00:36:49.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.180+0000 7fed35d9b640 1 --2- 172.21.15.32:0/3586290468 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fed381091c0 0x7fed38115b70 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7fed2c02aaa0 tx=0x7fed2c005370 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:49.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.181+0000 7fed277fe640 1 -- 172.21.15.32:0/3586290468 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fed2c040030 con 0x7fed381091c0 2024-04-25T00:36:49.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.181+0000 7fed277fe640 1 -- 172.21.15.32:0/3586290468 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fed2c03c920 con 0x7fed381091c0 2024-04-25T00:36:49.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.181+0000 7fed277fe640 1 -- 172.21.15.32:0/3586290468 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fed2c031b80 con 0x7fed381091c0 2024-04-25T00:36:49.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.181+0000 7fed3759e640 1 -- 172.21.15.32:0/3586290468 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fed381168d0 con 0x7fed381091c0 2024-04-25T00:36:49.183 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.181+0000 7fed3759e640 1 -- 172.21.15.32:0/3586290468 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fed3811a850 con 0x7fed381091c0 2024-04-25T00:36:49.188 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.188+0000 7fed277fe640 1 -- 172.21.15.32:0/3586290468 <== mon.2 v2:172.21.15.121:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7fed2c043030 con 0x7fed381091c0 2024-04-25T00:36:49.188 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.188+0000 7fed3f671740 1 Processor -- start 2024-04-25T00:36:49.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.192+0000 7fed3f671740 1 -- start start 2024-04-25T00:36:49.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.192+0000 7fed3f671740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557015eaf6f0 0x557015eafad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.193 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.192+0000 7fed3f671740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557015eb0010 0x557015eba1a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.192+0000 7fed3f671740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557015eba6e0 0x557015ebcad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.192+0000 7fed3f671740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x557015e14670 con 0x557015eaf6f0 2024-04-25T00:36:49.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.192+0000 7fed3f671740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x557015dbc970 con 0x557015eba6e0 2024-04-25T00:36:49.194 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.192+0000 7fed3f671740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x557015e10630 con 0x557015eb0010 2024-04-25T00:36:49.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.194+0000 7fed35d9b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557015eba6e0 0x557015ebcad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.194+0000 7fed3659c640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557015eb0010 0x557015eba1a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.194+0000 7fed3659c640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557015eb0010 0x557015eba1a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:56432/0 (socket says 172.21.15.32:56432) 2024-04-25T00:36:49.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.194+0000 7fed3659c640 1 -- 172.21.15.32:0/4249430074 learned_addr learned my addr 172.21.15.32:0/4249430074 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:49.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.194+0000 7fed3659c640 1 -- 172.21.15.32:0/4249430074 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557015eba6e0 msgr2=0x557015ebcad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:49.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.194+0000 7fed3659c640 1 --2- 172.21.15.32:0/4249430074 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557015eba6e0 0x557015ebcad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.194+0000 7fed3659c640 1 -- 172.21.15.32:0/4249430074 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557015eaf6f0 msgr2=0x557015eafad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:49.195 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.194+0000 7fed3659c640 1 --2- 172.21.15.32:0/4249430074 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557015eaf6f0 0x557015eafad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.196 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.194+0000 7fed3659c640 1 -- 172.21.15.32:0/4249430074 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557015de0400 con 0x557015eb0010 2024-04-25T00:36:49.196 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.195+0000 7fed3659c640 1 --2- 172.21.15.32:0/4249430074 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557015eb0010 0x557015eba1a0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7fed2800bb10 tx=0x7fed2800bf10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=67eb8c99930ae3b6 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:49.196 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.196+0000 7fed24ff9640 1 -- 172.21.15.32:0/4249430074 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fed28016070 con 0x557015eb0010 2024-04-25T00:36:49.196 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.196+0000 7fed24ff9640 1 -- 172.21.15.32:0/4249430074 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fed28005500 con 0x557015eb0010 2024-04-25T00:36:49.198 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.197+0000 7fed3f671740 1 -- 172.21.15.32:0/4249430074 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557015eb0010 msgr2=0x557015eba1a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:49.198 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.197+0000 7fed3f671740 1 --2- 172.21.15.32:0/4249430074 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557015eb0010 0x557015eba1a0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7fed2800bb10 tx=0x7fed2800bf10 comp rx=0 tx=0).stop 2024-04-25T00:36:49.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.200+0000 7fed277fe640 1 --2- 172.21.15.32:0/3586290468 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fed200a2f00 0x7fed200a53c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.202 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.201+0000 7fed277fe640 1 -- 172.21.15.32:0/3586290468 <== mon.2 v2:172.21.15.121:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fed2c0f1400 con 0x7fed381091c0 2024-04-25T00:36:49.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.202+0000 7fed36d9d640 1 --2- 172.21.15.32:0/3586290468 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fed200a2f00 0x7fed200a53c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.204+0000 7fed3f671740 1 -- 172.21.15.32:0/4249430074 shutdown_connections 2024-04-25T00:36:49.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.204+0000 7fed3f671740 1 --2- 172.21.15.32:0/4249430074 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557015eba6e0 0x557015ebcad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.204+0000 7fed3f671740 1 --2- 172.21.15.32:0/4249430074 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557015eb0010 0x557015eba1a0 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.204+0000 7fed3f671740 1 --2- 172.21.15.32:0/4249430074 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557015eaf6f0 0x557015eafad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.204+0000 7fed3f671740 1 -- 172.21.15.32:0/4249430074 >> 172.21.15.32:0/4249430074 conn(0x557015eaba10 msgr2=0x557015eabdf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:49.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.204+0000 7fed36d9d640 1 --2- 172.21.15.32:0/3586290468 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fed200a2f00 0x7fed200a53c0 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7fed3000a400 tx=0x7fed3000d040 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:49.213 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.210+0000 7fed3f671740 1 -- 172.21.15.32:0/4249430074 shutdown_connections 2024-04-25T00:36:49.213 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.210+0000 7fed3f671740 1 -- 172.21.15.32:0/4249430074 wait complete. 2024-04-25T00:36:49.213 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.211+0000 7fed3f671740 1 -- start start 2024-04-25T00:36:49.213 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.211+0000 7fed3f671740 1 Processor -- start 2024-04-25T00:36:49.213 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.211+0000 7fed3f671740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557015eaf6f0 0x557015eb9f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.214 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.211+0000 7fed3f671740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557015eb0010 0x557015eb7260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.214 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.211+0000 7fed3f671740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557015eba6e0 0x557015eb77a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.214 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.211+0000 7fed3f671740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x557015ebd520 con 0x557015eba6e0 2024-04-25T00:36:49.214 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.211+0000 7fed3f671740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x557015dd0050 con 0x557015eb0010 2024-04-25T00:36:49.214 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.211+0000 7fed3f671740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x557015ebd6a0 con 0x557015eaf6f0 2024-04-25T00:36:49.214 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.211+0000 7fed3f671740 10 client.0 _ll_register_callbacks cb 0x557015b27be0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:36:49.214 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.213+0000 7fed35d9b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557015eb0010 0x557015eb7260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.214 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.213+0000 7fed35d9b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557015eb0010 0x557015eb7260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:47694/0 (socket says 172.21.15.32:47694) 2024-04-25T00:36:49.214 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.213+0000 7fed35d9b640 1 -- 172.21.15.32:0/402895000 learned_addr learned my addr 172.21.15.32:0/402895000 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:49.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.213+0000 7fed35d9b640 1 -- 172.21.15.32:0/402895000 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557015eaf6f0 msgr2=0x557015eb9f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:49.218 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.213+0000 7fed35d9b640 1 --2- 172.21.15.32:0/402895000 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557015eaf6f0 0x557015eb9f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.218 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.213+0000 7fed35d9b640 1 -- 172.21.15.32:0/402895000 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557015eba6e0 msgr2=0x557015eb77a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:49.218 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.213+0000 7fed35d9b640 1 --2- 172.21.15.32:0/402895000 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557015eba6e0 0x557015eb77a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.218 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.213+0000 7fed35d9b640 1 -- 172.21.15.32:0/402895000 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557015da1dc0 con 0x557015eb0010 2024-04-25T00:36:49.218 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.213+0000 7fed35d9b640 1 --2- 172.21.15.32:0/402895000 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557015eb0010 0x557015eb7260 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7fed2c0bd210 tx=0x7fed2c03e000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=bdfa6d8d9076251c server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:49.218 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.217+0000 7fed1dffb640 10 client.0 ms_handle_connect on v2:172.21.15.121:3300/0 2024-04-25T00:36:49.218 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.217+0000 7fed1dffb640 1 -- 172.21.15.32:0/402895000 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fed2c04a040 con 0x557015eb0010 2024-04-25T00:36:49.223 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.221+0000 7fed3f671740 1 -- 172.21.15.32:0/402895000 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fed28005500 con 0x557015eb0010 2024-04-25T00:36:49.223 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.221+0000 7fed1dffb640 1 -- 172.21.15.32:0/402895000 <== mon.2 v2:172.21.15.121:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fed2c0b9af0 con 0x557015eb0010 2024-04-25T00:36:49.223 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.221+0000 7fed1dffb640 1 -- 172.21.15.32:0/402895000 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fed2c0bb650 con 0x557015eb0010 2024-04-25T00:36:49.223 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.223+0000 7fed3f671740 10 client.25703 fetch_fsmap learned FSMap version 27 2024-04-25T00:36:49.223 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.223+0000 7fed3f671740 1 -- 172.21.15.32:0/402895000 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x557015b283f0 con 0x557015eb0010 2024-04-25T00:36:49.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.224+0000 7fed1dffb640 1 -- 172.21.15.32:0/402895000 <== mon.2 v2:172.21.15.121:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fed28005500 con 0x557015eb0010 2024-04-25T00:36:49.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.225+0000 7fed1dffb640 1 -- 172.21.15.32:0/402895000 <== mon.2 v2:172.21.15.121:3300/0 5 ==== fsmap(e 27) ==== 8354+0+0 (secure 0 0 0) 0x7fed2c0bd770 con 0x557015eb0010 2024-04-25T00:36:49.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.227+0000 7fed3f671740 10 client.25703 fetch_fsmap finished waiting for FSMap version 27 2024-04-25T00:36:49.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.227+0000 7fed3f671740 10 client.25703 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-25T00:36:49.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.227+0000 7fed3f671740 20 client.25703 populate_metadata read hostname 'smithi032' 2024-04-25T00:36:49.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.227+0000 7fed3f671740 1 --2- 172.21.15.32:0/402895000 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x557015eec580 0x557015eeea20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.227+0000 7fed3f671740 4 client.25703 mds_command: new command op to 24481 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:36:49.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.227+0000 7fed3f671740 1 -- 172.21.15.32:0/402895000 --> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x557015e21bb0 con 0x557015eec580 2024-04-25T00:36:49.233 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.231+0000 7fed3659c640 1 --2- 172.21.15.32:0/402895000 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x557015eec580 0x557015eeea20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.240 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.239+0000 7fed1dffb640 10 client.25703 ms_handle_connect on v2:172.21.15.130:6838/4267125809 2024-04-25T00:36:49.240 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.239+0000 7fed3659c640 1 --2- 172.21.15.32:0/402895000 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x557015eec580 0x557015eeea20 crc :-1 s=READY pgs=39 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=2b2cc8c7e5b11166 server_cookie=ee10ee0b7ab87828 in_seq=0 out_seq=0 2024-04-25T00:36:49.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.245+0000 7fed1dffb640 1 -- 172.21.15.32:0/402895000 <== mds.1 v2:172.21.15.130:6838/4267125809 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x557015e21bb0 con 0x557015eec580 2024-04-25T00:36:49.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.245+0000 7fed1dffb640 10 client.25703 handle_command_reply: tid=0 2024-04-25T00:36:49.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.245+0000 7fed3f671740 1 client.25703 shutdown 2024-04-25T00:36:49.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.250+0000 7fed3f671740 1 -- 172.21.15.32:0/402895000 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557015eb0010 msgr2=0x557015eb7260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:49.255 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.250+0000 7fed3f671740 1 --2- 172.21.15.32:0/402895000 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557015eb0010 0x557015eb7260 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7fed2c0bd210 tx=0x7fed2c03e000 comp rx=0 tx=0).stop 2024-04-25T00:36:49.255 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.251+0000 7fed3f671740 1 -- 172.21.15.32:0/402895000 shutdown_connections 2024-04-25T00:36:49.255 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.251+0000 7fed3f671740 1 --2- 172.21.15.32:0/402895000 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x557015eec580 0x557015eeea20 crc :-1 s=READY pgs=39 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.255 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.251+0000 7fed3f671740 1 --2- 172.21.15.32:0/402895000 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557015eba6e0 0x557015eb77a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.256 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.251+0000 7fed3f671740 1 --2- 172.21.15.32:0/402895000 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557015eb0010 0x557015eb7260 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.256 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.251+0000 7fed3f671740 1 --2- 172.21.15.32:0/402895000 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557015eaf6f0 0x557015eb9f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.256 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.251+0000 7fed1dffb640 0 client.25703 ms_handle_reset on v2:172.21.15.130:6838/4267125809 2024-04-25T00:36:49.256 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.251+0000 7fed3f671740 1 -- 172.21.15.32:0/402895000 >> 172.21.15.32:0/402895000 conn(0x557015eaba10 msgr2=0x557015eaaca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:49.257 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.255+0000 7fed3f671740 1 -- 172.21.15.32:0/402895000 shutdown_connections 2024-04-25T00:36:49.257 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.256+0000 7fed3f671740 1 -- 172.21.15.32:0/402895000 wait complete. 2024-04-25T00:36:49.257 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.257+0000 7fed3f671740 20 client.25703 trim_cache size 0 max 16384 2024-04-25T00:36:49.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.294+0000 7fed3f671740 1 Processor -- start 2024-04-25T00:36:49.295 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.294+0000 7fed3f671740 1 -- start start 2024-04-25T00:36:49.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.295+0000 7fed3f671740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557015eaf6f0 0x557015eb7780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.296+0000 7fed3f671740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557015eb0010 0x557015ec6430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.296+0000 7fed3f671740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557015eba6e0 0x557015ef42f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.297 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.296+0000 7fed3659c640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557015eaf6f0 0x557015eb7780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.297 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.296+0000 7fed3659c640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557015eaf6f0 0x557015eb7780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:47698/0 (socket says 172.21.15.32:47698) 2024-04-25T00:36:49.297 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.296+0000 7fed3f671740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fed3807f360 con 0x557015eba6e0 2024-04-25T00:36:49.297 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.296+0000 7fed3f671740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fed3807cc30 con 0x557015eaf6f0 2024-04-25T00:36:49.297 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.296+0000 7fed3f671740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fed3807ab30 con 0x557015eb0010 2024-04-25T00:36:49.297 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.296+0000 7fed3659c640 1 -- 172.21.15.32:0/3189700447 learned_addr learned my addr 172.21.15.32:0/3189700447 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:49.297 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.296+0000 7fed36d9d640 1 --2- 172.21.15.32:0/3189700447 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557015eba6e0 0x557015ef42f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.298 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.296+0000 7fed35d9b640 1 --2- 172.21.15.32:0/3189700447 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557015eb0010 0x557015ec6430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.299 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.297+0000 7fed3659c640 1 -- 172.21.15.32:0/3189700447 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557015eb0010 msgr2=0x557015ec6430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:49.299 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.297+0000 7fed3659c640 1 --2- 172.21.15.32:0/3189700447 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557015eb0010 0x557015ec6430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.299 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.297+0000 7fed3659c640 1 -- 172.21.15.32:0/3189700447 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557015eba6e0 msgr2=0x557015ef42f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:49.299 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.297+0000 7fed3659c640 1 --2- 172.21.15.32:0/3189700447 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557015eba6e0 0x557015ef42f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.299 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.297+0000 7fed3659c640 1 -- 172.21.15.32:0/3189700447 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557015eafe40 con 0x557015eaf6f0 2024-04-25T00:36:49.299 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.298+0000 7fed3659c640 1 --2- 172.21.15.32:0/3189700447 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557015eaf6f0 0x557015eb7780 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7fed28005d00 tx=0x7fed28004b30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e89aa0cc5cfe9fb2 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:49.299 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.298+0000 7fed1ffff640 1 -- 172.21.15.32:0/3189700447 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fed28016040 con 0x557015eaf6f0 2024-04-25T00:36:49.299 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.298+0000 7fed1ffff640 1 -- 172.21.15.32:0/3189700447 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fed28024330 con 0x557015eaf6f0 2024-04-25T00:36:49.299 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.298+0000 7fed1ffff640 1 -- 172.21.15.32:0/3189700447 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fed28005240 con 0x557015eaf6f0 2024-04-25T00:36:49.301 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.300+0000 7fed3f671740 1 -- 172.21.15.32:0/3189700447 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557015eaf6f0 msgr2=0x557015eb7780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:49.301 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.300+0000 7fed3f671740 1 --2- 172.21.15.32:0/3189700447 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557015eaf6f0 0x557015eb7780 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7fed28005d00 tx=0x7fed28004b30 comp rx=0 tx=0).stop 2024-04-25T00:36:49.301 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.300+0000 7fed3f671740 1 -- 172.21.15.32:0/3189700447 shutdown_connections 2024-04-25T00:36:49.301 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.300+0000 7fed3f671740 1 --2- 172.21.15.32:0/3189700447 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557015eba6e0 0x557015ef42f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.301+0000 7fed3f671740 1 --2- 172.21.15.32:0/3189700447 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557015eb0010 0x557015ec6430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.301+0000 7fed3f671740 1 --2- 172.21.15.32:0/3189700447 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557015eaf6f0 0x557015eb7780 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.301+0000 7fed3f671740 1 -- 172.21.15.32:0/3189700447 >> 172.21.15.32:0/3189700447 conn(0x557015eaba10 msgr2=0x557015eaaca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:49.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.301+0000 7fed3f671740 1 -- 172.21.15.32:0/3189700447 shutdown_connections 2024-04-25T00:36:49.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.301+0000 7fed3f671740 1 -- 172.21.15.32:0/3189700447 wait complete. 2024-04-25T00:36:49.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.302+0000 7fed3f671740 1 -- start start 2024-04-25T00:36:49.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.302+0000 7fed3f671740 1 Processor -- start 2024-04-25T00:36:49.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.302+0000 7fed3f671740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557015eaf6f0 0x557015ec0690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.305 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.303+0000 7fed3f671740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557015eb0010 0x557015ec0bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.305 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.303+0000 7fed3f671740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557015eba6e0 0x557015f51b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.305 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.303+0000 7fed3f671740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fed3807b0f0 con 0x557015eb0010 2024-04-25T00:36:49.305 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.303+0000 7fed3f671740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fed3807c560 con 0x557015eba6e0 2024-04-25T00:36:49.305 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.303+0000 7fed3f671740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fed3807b7c0 con 0x557015eaf6f0 2024-04-25T00:36:49.305 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.303+0000 7fed3f671740 10 client.0 _ll_register_callbacks cb 0x7fed2c005770 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:36:49.305 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.303+0000 7fed35d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557015eb0010 0x557015ec0bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.305 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.303+0000 7fed35d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557015eb0010 0x557015ec0bd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44184/0 (socket says 172.21.15.32:44184) 2024-04-25T00:36:49.305 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.303+0000 7fed35d9b640 1 -- 172.21.15.32:0/869780886 learned_addr learned my addr 172.21.15.32:0/869780886 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:49.306 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.304+0000 7fed3659c640 1 --2- 172.21.15.32:0/869780886 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557015eaf6f0 0x557015ec0690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.306 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.305+0000 7fed36d9d640 1 --2- 172.21.15.32:0/869780886 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557015eba6e0 0x557015f51b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.306 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.305+0000 7fed3659c640 1 -- 172.21.15.32:0/869780886 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557015eba6e0 msgr2=0x557015f51b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:49.306 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.305+0000 7fed3659c640 1 --2- 172.21.15.32:0/869780886 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557015eba6e0 0x557015f51b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.306 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.305+0000 7fed3659c640 1 -- 172.21.15.32:0/869780886 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557015eb0010 msgr2=0x557015ec0bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:49.306 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.305+0000 7fed3659c640 1 --2- 172.21.15.32:0/869780886 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557015eb0010 0x557015ec0bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.306 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.305+0000 7fed3659c640 1 -- 172.21.15.32:0/869780886 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557015bf27d0 con 0x557015eaf6f0 2024-04-25T00:36:49.311 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.310+0000 7fed3659c640 1 --2- 172.21.15.32:0/869780886 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557015eaf6f0 0x557015ec0690 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7fed2800ba30 tx=0x7fed2800bf10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=479752aa210c332c server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:49.311 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.311+0000 7fed1dffb640 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:36:49.312 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.311+0000 7fed1dffb640 1 -- 172.21.15.32:0/869780886 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fed28033030 con 0x557015eaf6f0 2024-04-25T00:36:49.312 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.311+0000 7fed1dffb640 1 -- 172.21.15.32:0/869780886 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fed2802a040 con 0x557015eaf6f0 2024-04-25T00:36:49.312 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.311+0000 7fed3f671740 1 -- 172.21.15.32:0/869780886 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fed28024330 con 0x557015eaf6f0 2024-04-25T00:36:49.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.313+0000 7fed1dffb640 1 -- 172.21.15.32:0/869780886 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fed2803d050 con 0x557015eaf6f0 2024-04-25T00:36:49.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.313+0000 7fed3f671740 10 client.25693 fetch_fsmap learned FSMap version 27 2024-04-25T00:36:49.313 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.313+0000 7fed3f671740 1 -- 172.21.15.32:0/869780886 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x557015f52fa0 con 0x557015eaf6f0 2024-04-25T00:36:49.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.313+0000 7fed1dffb640 1 -- 172.21.15.32:0/869780886 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fed28024330 con 0x557015eaf6f0 2024-04-25T00:36:49.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.315+0000 7fed1dffb640 1 -- 172.21.15.32:0/869780886 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 27) ==== 8354+0+0 (secure 0 0 0) 0x7fed280053b0 con 0x557015eaf6f0 2024-04-25T00:36:49.316 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.315+0000 7fed3f671740 10 client.25693 fetch_fsmap finished waiting for FSMap version 27 2024-04-25T00:36:49.316 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.315+0000 7fed3f671740 10 client.25693 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-25T00:36:49.316 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.315+0000 7fed3f671740 20 client.25693 populate_metadata read hostname 'smithi032' 2024-04-25T00:36:49.316 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.315+0000 7fed3f671740 1 --2- 172.21.15.32:0/869780886 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x557015f53380 0x557015f55770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.316 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.315+0000 7fed3f671740 4 client.25693 mds_command: new command op to 24481 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-25T00:36:49.316 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.315+0000 7fed3f671740 1 -- 172.21.15.32:0/869780886 --> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] -- command(tid 0: {"prefix": "damage ls"}) -- 0x557015e1bf00 con 0x557015f53380 2024-04-25T00:36:49.316 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.316+0000 7fed35d9b640 1 --2- 172.21.15.32:0/869780886 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x557015f53380 0x557015f55770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.325 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.325+0000 7fed35d9b640 1 --2- 172.21.15.32:0/869780886 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x557015f53380 0x557015f55770 crc :-1 s=READY pgs=40 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=814bac6219184a7f server_cookie=ae5688a0acf05fe9 in_seq=0 out_seq=0 2024-04-25T00:36:49.326 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.325+0000 7fed1dffb640 10 client.25693 ms_handle_connect on v2:172.21.15.130:6838/4267125809 2024-04-25T00:36:49.328 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.326+0000 7fed1dffb640 1 -- 172.21.15.32:0/869780886 <== mds.1 v2:172.21.15.130:6838/4267125809 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x557015e1bf00 con 0x557015f53380 2024-04-25T00:36:49.328 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.326+0000 7fed1dffb640 10 client.25693 handle_command_reply: tid=0 2024-04-25T00:36:49.328 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.326+0000 7fed3f671740 1 client.25693 shutdown 2024-04-25T00:36:49.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.328+0000 7fed3f671740 1 -- 172.21.15.32:0/869780886 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557015eaf6f0 msgr2=0x557015ec0690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:49.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.328+0000 7fed3f671740 1 --2- 172.21.15.32:0/869780886 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557015eaf6f0 0x557015ec0690 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7fed2800ba30 tx=0x7fed2800bf10 comp rx=0 tx=0).stop 2024-04-25T00:36:49.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.328+0000 7fed3f671740 1 -- 172.21.15.32:0/869780886 shutdown_connections 2024-04-25T00:36:49.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.328+0000 7fed3f671740 1 --2- 172.21.15.32:0/869780886 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x557015eba6e0 0x557015f51b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.328+0000 7fed3f671740 1 --2- 172.21.15.32:0/869780886 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x557015f53380 0x557015f55770 crc :-1 s=READY pgs=40 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.328+0000 7fed3f671740 1 --2- 172.21.15.32:0/869780886 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x557015eb0010 0x557015ec0bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.328+0000 7fed3f671740 1 --2- 172.21.15.32:0/869780886 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557015eaf6f0 0x557015ec0690 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.328+0000 7fed3f671740 1 -- 172.21.15.32:0/869780886 >> 172.21.15.32:0/869780886 conn(0x557015eaba10 msgr2=0x557015ec5420 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:49.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.328+0000 7fed1dffb640 0 client.25693 ms_handle_reset on v2:172.21.15.130:6838/4267125809 2024-04-25T00:36:49.329 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.328+0000 7fed3f671740 1 -- 172.21.15.32:0/869780886 shutdown_connections 2024-04-25T00:36:49.330 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.328+0000 7fed3f671740 1 -- 172.21.15.32:0/869780886 wait complete. 2024-04-25T00:36:49.330 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.329+0000 7fed3f671740 20 client.25693 trim_cache size 0 max 16384 2024-04-25T00:36:49.330 INFO:teuthology.orchestra.run.smithi032.stdout:[] 2024-04-25T00:36:49.331 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.330+0000 7fed3759e640 1 -- 172.21.15.32:0/3586290468 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fed200a2f00 msgr2=0x7fed200a53c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:49.331 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.330+0000 7fed3759e640 1 --2- 172.21.15.32:0/3586290468 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fed200a2f00 0x7fed200a53c0 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7fed3000a400 tx=0x7fed3000d040 comp rx=0 tx=0).stop 2024-04-25T00:36:49.331 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.330+0000 7fed3759e640 1 -- 172.21.15.32:0/3586290468 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fed381091c0 msgr2=0x7fed38115b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:49.331 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.330+0000 7fed3759e640 1 --2- 172.21.15.32:0/3586290468 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fed381091c0 0x7fed38115b70 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7fed2c02aaa0 tx=0x7fed2c005370 comp rx=0 tx=0).stop 2024-04-25T00:36:49.332 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.332+0000 7fed3759e640 1 -- 172.21.15.32:0/3586290468 shutdown_connections 2024-04-25T00:36:49.332 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.332+0000 7fed3759e640 1 --2- 172.21.15.32:0/3586290468 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fed200a2f00 0x7fed200a53c0 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.332 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.332+0000 7fed3759e640 1 --2- 172.21.15.32:0/3586290468 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fed3810fde0 0x7fed381160b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.332 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.332+0000 7fed3759e640 1 --2- 172.21.15.32:0/3586290468 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fed381091c0 0x7fed38115b70 unknown :-1 s=CLOSED pgs=111 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.332 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.332+0000 7fed3759e640 1 --2- 172.21.15.32:0/3586290468 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fed38108740 0x7fed3811caf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.333 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.332+0000 7fed3759e640 1 -- 172.21.15.32:0/3586290468 >> 172.21.15.32:0/3586290468 conn(0x7fed3806c6c0 msgr2=0x7fed38070340 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:49.333 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.333+0000 7fed3759e640 1 -- 172.21.15.32:0/3586290468 shutdown_connections 2024-04-25T00:36:49.335 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.335+0000 7fed3759e640 1 -- 172.21.15.32:0/3586290468 wait complete. 2024-04-25T00:36:49.367 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-25T00:36:49.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:49 smithi121 ceph-mon[32231]: pgmap v108: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 2.5 MiB/s rd, 696 MiB/s wr, 1.41k op/s 2024-04-25T00:36:49.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:49 smithi130 ceph-mon[32525]: pgmap v108: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 2.5 MiB/s rd, 696 MiB/s wr, 1.41k op/s 2024-04-25T00:36:49.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.674+0000 7efc404a5640 1 -- 172.21.15.32:0/2422750863 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efc3810ff90 msgr2=0x7efc3810da80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:49.675 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.674+0000 7efc404a5640 1 --2- 172.21.15.32:0/2422750863 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efc3810ff90 0x7efc3810da80 secure :-1 s=READY pgs=614 cs=0 l=1 rev1=1 crypto rx=0x7efc34009930 tx=0x7efc3402a480 comp rx=0 tx=0).stop 2024-04-25T00:36:49.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.680+0000 7efc404a5640 1 -- 172.21.15.32:0/2422750863 shutdown_connections 2024-04-25T00:36:49.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.680+0000 7efc404a5640 1 --2- 172.21.15.32:0/2422750863 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efc3811af20 0x7efc3811d310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.680+0000 7efc404a5640 1 --2- 172.21.15.32:0/2422750863 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efc3810dfc0 0x7efc3810e440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.682 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.680+0000 7efc404a5640 1 --2- 172.21.15.32:0/2422750863 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efc3810ff90 0x7efc3810da80 unknown :-1 s=CLOSED pgs=614 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.682 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.680+0000 7efc404a5640 1 -- 172.21.15.32:0/2422750863 >> 172.21.15.32:0/2422750863 conn(0x7efc3806c850 msgr2=0x7efc3806cc60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:49.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.686+0000 7efc404a5640 1 -- 172.21.15.32:0/2422750863 shutdown_connections 2024-04-25T00:36:49.692 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.691+0000 7efc404a5640 1 -- 172.21.15.32:0/2422750863 wait complete. 2024-04-25T00:36:49.692 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.692+0000 7efc404a5640 1 Processor -- start 2024-04-25T00:36:49.692 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.692+0000 7efc404a5640 1 -- start start 2024-04-25T00:36:49.693 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.692+0000 7efc404a5640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efc3810dfc0 0x7efc381a6ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.693 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.692+0000 7efc404a5640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efc3810ff90 0x7efc381a7410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.692+0000 7efc404a5640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efc3811af20 0x7efc381ab790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.692+0000 7efc404a5640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7efc38120a10 con 0x7efc3811af20 2024-04-25T00:36:49.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.692+0000 7efc404a5640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7efc38120890 con 0x7efc3810dfc0 2024-04-25T00:36:49.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.692+0000 7efc404a5640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7efc38120b90 con 0x7efc3810ff90 2024-04-25T00:36:49.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.692+0000 7efc3e241640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efc3810dfc0 0x7efc381a6ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.693+0000 7efc3e241640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efc3810dfc0 0x7efc381a6ed0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:47710/0 (socket says 172.21.15.32:47710) 2024-04-25T00:36:49.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.693+0000 7efc3e241640 1 -- 172.21.15.32:0/2634855717 learned_addr learned my addr 172.21.15.32:0/2634855717 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:49.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.693+0000 7efc3e241640 1 -- 172.21.15.32:0/2634855717 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efc3810ff90 msgr2=0x7efc381a7410 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:36:49.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.693+0000 7efc3e241640 1 --2- 172.21.15.32:0/2634855717 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efc3810ff90 0x7efc381a7410 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.693+0000 7efc3e241640 1 -- 172.21.15.32:0/2634855717 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efc3811af20 msgr2=0x7efc381ab790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:49.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.693+0000 7efc3e241640 1 --2- 172.21.15.32:0/2634855717 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efc3811af20 0x7efc381ab790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.694 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.693+0000 7efc3e241640 1 -- 172.21.15.32:0/2634855717 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7efc381abec0 con 0x7efc3810dfc0 2024-04-25T00:36:49.695 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.695+0000 7efc3e241640 1 --2- 172.21.15.32:0/2634855717 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efc3810dfc0 0x7efc381a6ed0 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7efc34009900 tx=0x7efc34039520 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:49.696 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.696+0000 7efc2f7fe640 1 -- 172.21.15.32:0/2634855717 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efc34040030 con 0x7efc3810dfc0 2024-04-25T00:36:49.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.696+0000 7efc2f7fe640 1 -- 172.21.15.32:0/2634855717 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7efc34030e40 con 0x7efc3810dfc0 2024-04-25T00:36:49.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.696+0000 7efc2f7fe640 1 -- 172.21.15.32:0/2634855717 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efc34035070 con 0x7efc3810dfc0 2024-04-25T00:36:49.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.696+0000 7efc404a5640 1 -- 172.21.15.32:0/2634855717 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7efc381ac150 con 0x7efc3810dfc0 2024-04-25T00:36:49.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.696+0000 7efc404a5640 1 -- 172.21.15.32:0/2634855717 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7efc381b3a40 con 0x7efc3810dfc0 2024-04-25T00:36:49.707 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.703+0000 7efc2f7fe640 1 -- 172.21.15.32:0/2634855717 <== mon.2 v2:172.21.15.121:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7efc34031750 con 0x7efc3810dfc0 2024-04-25T00:36:49.707 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.705+0000 7efc2f7fe640 1 --2- 172.21.15.32:0/2634855717 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7efc240a2f10 0x7efc240a53d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.707 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.705+0000 7efc2f7fe640 1 -- 172.21.15.32:0/2634855717 <== mon.2 v2:172.21.15.121:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7efc340f0680 con 0x7efc3810dfc0 2024-04-25T00:36:49.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.708+0000 7efc431b4740 1 Processor -- start 2024-04-25T00:36:49.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.709+0000 7efc431b4740 1 -- start start 2024-04-25T00:36:49.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.709+0000 7efc431b4740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5565b9020580 0x5565b9020960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.710+0000 7efc431b4740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5565b9020ea0 0x5565b902b030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.710+0000 7efc431b4740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5565b902b570 0x5565b902d960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.710+0000 7efc431b4740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5565b8f85670 con 0x5565b9020580 2024-04-25T00:36:49.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.710+0000 7efc431b4740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5565b8f2d970 con 0x5565b902b570 2024-04-25T00:36:49.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.710+0000 7efc431b4740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5565b8f81630 con 0x5565b9020ea0 2024-04-25T00:36:49.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.710+0000 7efc3da40640 1 --2- 172.21.15.32:0/2634855717 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7efc240a2f10 0x7efc240a53d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.710+0000 7efc3ea42640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5565b902b570 0x5565b902d960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.710+0000 7efc3ea42640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5565b902b570 0x5565b902d960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:47716/0 (socket says 172.21.15.32:47716) 2024-04-25T00:36:49.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.710+0000 7efc3ea42640 1 -- 172.21.15.32:0/1936649960 learned_addr learned my addr 172.21.15.32:0/1936649960 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:49.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.711+0000 7efc3ea42640 1 -- 172.21.15.32:0/1936649960 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5565b9020ea0 msgr2=0x5565b902b030 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-25T00:36:49.715 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.711+0000 7efc3ea42640 1 --2- 172.21.15.32:0/1936649960 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5565b9020ea0 0x5565b902b030 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.716 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.711+0000 7efc3ea42640 1 -- 172.21.15.32:0/1936649960 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5565b9020580 msgr2=0x5565b9020960 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-25T00:36:49.716 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.711+0000 7efc3ea42640 1 --2- 172.21.15.32:0/1936649960 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5565b9020580 0x5565b9020960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.716 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.711+0000 7efc3ea42640 1 -- 172.21.15.32:0/1936649960 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5565b8f51400 con 0x5565b902b570 2024-04-25T00:36:49.716 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.711+0000 7efc3ea42640 1 --2- 172.21.15.32:0/1936649960 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5565b902b570 0x5565b902d960 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7efc28009760 tx=0x7efc28009c20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=373bcfcd92a0f5db server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:49.716 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.712+0000 7efc3da40640 1 --2- 172.21.15.32:0/2634855717 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7efc240a2f10 0x7efc240a53d0 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x5565b9021b40 tx=0x7efc30009250 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:49.716 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.714+0000 7efc2cbfe640 1 -- 172.21.15.32:0/1936649960 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efc28015020 con 0x5565b902b570 2024-04-25T00:36:49.716 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.714+0000 7efc2cbfe640 1 -- 172.21.15.32:0/1936649960 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7efc28011420 con 0x5565b902b570 2024-04-25T00:36:49.716 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.715+0000 7efc431b4740 1 -- 172.21.15.32:0/1936649960 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5565b902b570 msgr2=0x5565b902d960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:49.716 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.715+0000 7efc431b4740 1 --2- 172.21.15.32:0/1936649960 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5565b902b570 0x5565b902d960 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7efc28009760 tx=0x7efc28009c20 comp rx=0 tx=0).stop 2024-04-25T00:36:49.717 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.716+0000 7efc431b4740 1 -- 172.21.15.32:0/1936649960 shutdown_connections 2024-04-25T00:36:49.717 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.716+0000 7efc431b4740 1 --2- 172.21.15.32:0/1936649960 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5565b902b570 0x5565b902d960 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.717 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.716+0000 7efc431b4740 1 --2- 172.21.15.32:0/1936649960 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5565b9020ea0 0x5565b902b030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.717 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.716+0000 7efc431b4740 1 --2- 172.21.15.32:0/1936649960 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5565b9020580 0x5565b9020960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.717 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.716+0000 7efc431b4740 1 -- 172.21.15.32:0/1936649960 >> 172.21.15.32:0/1936649960 conn(0x5565b901c8a0 msgr2=0x5565b901cc80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:49.717 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.717+0000 7efc431b4740 1 -- 172.21.15.32:0/1936649960 shutdown_connections 2024-04-25T00:36:49.718 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.717+0000 7efc431b4740 1 -- 172.21.15.32:0/1936649960 wait complete. 2024-04-25T00:36:49.718 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.718+0000 7efc431b4740 1 -- start start 2024-04-25T00:36:49.719 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.718+0000 7efc431b4740 1 Processor -- start 2024-04-25T00:36:49.720 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.718+0000 7efc431b4740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5565b9020580 0x5565b902af00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.720 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.718+0000 7efc431b4740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5565b9020ea0 0x5565b90281f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.720 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.718+0000 7efc431b4740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5565b902b570 0x5565b9028770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.720 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.718+0000 7efc431b4740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5565b902e3b0 con 0x5565b902b570 2024-04-25T00:36:49.720 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.718+0000 7efc431b4740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5565b8f41050 con 0x5565b9020580 2024-04-25T00:36:49.720 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.718+0000 7efc431b4740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5565b902e530 con 0x5565b9020ea0 2024-04-25T00:36:49.720 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.718+0000 7efc431b4740 10 client.0 _ll_register_callbacks cb 0x5565b8c98be0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:36:49.721 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.719+0000 7efc3ea42640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5565b902b570 0x5565b9028770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.721 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.719+0000 7efc3ea42640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5565b902b570 0x5565b9028770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44222/0 (socket says 172.21.15.32:44222) 2024-04-25T00:36:49.721 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.719+0000 7efc3ea42640 1 -- 172.21.15.32:0/3159305541 learned_addr learned my addr 172.21.15.32:0/3159305541 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:49.721 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.719+0000 7efc3da40640 1 --2- 172.21.15.32:0/3159305541 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5565b9020ea0 0x5565b90281f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.721 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.719+0000 7efc3e241640 1 --2- 172.21.15.32:0/3159305541 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5565b9020580 0x5565b902af00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.720+0000 7efc3da40640 1 -- 172.21.15.32:0/3159305541 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5565b9020580 msgr2=0x5565b902af00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:49.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.720+0000 7efc3da40640 1 --2- 172.21.15.32:0/3159305541 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5565b9020580 0x5565b902af00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.721+0000 7efc3da40640 1 -- 172.21.15.32:0/3159305541 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5565b902b570 msgr2=0x5565b9028770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:49.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.721+0000 7efc3da40640 1 --2- 172.21.15.32:0/3159305541 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5565b902b570 0x5565b9028770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.721+0000 7efc3da40640 1 -- 172.21.15.32:0/3159305541 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5565b8f12dc0 con 0x5565b9020ea0 2024-04-25T00:36:49.727 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.726+0000 7efc21ffb640 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:36:49.727 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.726+0000 7efc3da40640 1 --2- 172.21.15.32:0/3159305541 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5565b9020ea0 0x5565b90281f0 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7efc30017d70 tx=0x7efc30004ad0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=634d47edc0270141 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:49.744 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.743+0000 7efc21ffb640 1 -- 172.21.15.32:0/3159305541 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efc30018820 con 0x5565b9020ea0 2024-04-25T00:36:49.744 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.744+0000 7efc431b4740 1 -- 172.21.15.32:0/3159305541 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7efc28011420 con 0x5565b9020ea0 2024-04-25T00:36:49.745 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.744+0000 7efc21ffb640 1 -- 172.21.15.32:0/3159305541 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7efc300189c0 con 0x5565b9020ea0 2024-04-25T00:36:49.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:49 smithi032 ceph-mon[29357]: pgmap v108: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 2.5 MiB/s rd, 696 MiB/s wr, 1.41k op/s 2024-04-25T00:36:49.751 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.749+0000 7efc21ffb640 1 -- 172.21.15.32:0/3159305541 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7efc30018bc0 con 0x5565b9020ea0 2024-04-25T00:36:49.751 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.749+0000 7efc431b4740 10 client.25696 fetch_fsmap learned FSMap version 27 2024-04-25T00:36:49.751 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.749+0000 7efc431b4740 1 -- 172.21.15.32:0/3159305541 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x5565b8c993f0 con 0x5565b9020ea0 2024-04-25T00:36:49.751 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.750+0000 7efc21ffb640 1 -- 172.21.15.32:0/3159305541 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7efc28011420 con 0x5565b9020ea0 2024-04-25T00:36:49.751 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.750+0000 7efc21ffb640 1 -- 172.21.15.32:0/3159305541 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 27) ==== 8354+0+0 (secure 0 0 0) 0x7efc3002d420 con 0x5565b9020ea0 2024-04-25T00:36:49.751 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.750+0000 7efc431b4740 10 client.25696 fetch_fsmap finished waiting for FSMap version 27 2024-04-25T00:36:49.752 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.750+0000 7efc431b4740 10 client.25696 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:36:49.752 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.750+0000 7efc431b4740 20 client.25696 populate_metadata read hostname 'smithi032' 2024-04-25T00:36:49.752 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.750+0000 7efc431b4740 1 --2- 172.21.15.32:0/3159305541 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5565b903bbc0 0x5565b903e020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.752 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.750+0000 7efc431b4740 4 client.25696 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:36:49.752 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.750+0000 7efc431b4740 1 -- 172.21.15.32:0/3159305541 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5565b8f92bb0 con 0x5565b903bbc0 2024-04-25T00:36:49.752 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.751+0000 7efc3e241640 1 --2- 172.21.15.32:0/3159305541 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5565b903bbc0 0x5565b903e020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.755 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.754+0000 7efc3e241640 1 --2- 172.21.15.32:0/3159305541 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5565b903bbc0 0x5565b903e020 crc :-1 s=READY pgs=81 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=39e25afc8ffe4129 server_cookie=117637338a6ba2f3 in_seq=0 out_seq=0 2024-04-25T00:36:49.755 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.755+0000 7efc21ffb640 10 client.25696 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:49.758 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.757+0000 7efc21ffb640 1 -- 172.21.15.32:0/3159305541 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x5565b8f92bb0 con 0x5565b903bbc0 2024-04-25T00:36:49.758 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.757+0000 7efc21ffb640 10 client.25696 handle_command_reply: tid=0 2024-04-25T00:36:49.758 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.757+0000 7efc431b4740 1 client.25696 shutdown 2024-04-25T00:36:49.758 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.757+0000 7efc431b4740 1 -- 172.21.15.32:0/3159305541 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5565b9020ea0 msgr2=0x5565b90281f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:49.759 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.757+0000 7efc431b4740 1 --2- 172.21.15.32:0/3159305541 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5565b9020ea0 0x5565b90281f0 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7efc30017d70 tx=0x7efc30004ad0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.766 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.766+0000 7efc431b4740 1 -- 172.21.15.32:0/3159305541 shutdown_connections 2024-04-25T00:36:49.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.766+0000 7efc431b4740 1 --2- 172.21.15.32:0/3159305541 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5565b903bbc0 0x5565b903e020 crc :-1 s=READY pgs=81 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.766+0000 7efc431b4740 1 --2- 172.21.15.32:0/3159305541 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5565b902b570 0x5565b9028770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.766+0000 7efc431b4740 1 --2- 172.21.15.32:0/3159305541 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5565b9020ea0 0x5565b90281f0 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.766+0000 7efc431b4740 1 --2- 172.21.15.32:0/3159305541 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5565b9020580 0x5565b902af00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.766+0000 7efc431b4740 1 -- 172.21.15.32:0/3159305541 >> 172.21.15.32:0/3159305541 conn(0x5565b901c8a0 msgr2=0x5565b901bb40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:49.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.766+0000 7efc21ffb640 0 client.25696 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:49.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.769+0000 7efc431b4740 1 -- 172.21.15.32:0/3159305541 shutdown_connections 2024-04-25T00:36:49.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.769+0000 7efc431b4740 1 -- 172.21.15.32:0/3159305541 wait complete. 2024-04-25T00:36:49.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.771+0000 7efc431b4740 20 client.25696 trim_cache size 0 max 16384 2024-04-25T00:36:49.826 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.823+0000 7efc431b4740 1 Processor -- start 2024-04-25T00:36:49.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.828+0000 7efc431b4740 1 -- start start 2024-04-25T00:36:49.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.828+0000 7efc431b4740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5565b9020580 0x5565b9028740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.830 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.828+0000 7efc431b4740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5565b9020ea0 0x5565b9037350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.828+0000 7efc431b4740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5565b902b570 0x5565b902aa30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.828+0000 7efc431b4740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7efc1c012980 con 0x5565b902b570 2024-04-25T00:36:49.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.828+0000 7efc431b4740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7efc1c010250 con 0x5565b9020ea0 2024-04-25T00:36:49.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.828+0000 7efc431b4740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7efc1c00e040 con 0x5565b9020580 2024-04-25T00:36:49.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.831+0000 7efc3e241640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5565b9020580 0x5565b9028740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.831 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.831+0000 7efc3e241640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5565b9020580 0x5565b9028740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:56480/0 (socket says 172.21.15.32:56480) 2024-04-25T00:36:49.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.831+0000 7efc3e241640 1 -- 172.21.15.32:0/1493044284 learned_addr learned my addr 172.21.15.32:0/1493044284 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:49.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.831+0000 7efc3da40640 1 --2- 172.21.15.32:0/1493044284 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5565b9020ea0 0x5565b9037350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.833+0000 7efc3e241640 1 -- 172.21.15.32:0/1493044284 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5565b9020ea0 msgr2=0x5565b9037350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:49.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.833+0000 7efc3e241640 1 --2- 172.21.15.32:0/1493044284 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5565b9020ea0 0x5565b9037350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.833+0000 7efc3e241640 1 -- 172.21.15.32:0/1493044284 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5565b902b570 msgr2=0x5565b902aa30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:49.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.833+0000 7efc3e241640 1 --2- 172.21.15.32:0/1493044284 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5565b902b570 0x5565b902aa30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.833+0000 7efc3e241640 1 -- 172.21.15.32:0/1493044284 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5565b8d637d0 con 0x5565b9020580 2024-04-25T00:36:49.842 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.841+0000 7efc3e241640 1 --2- 172.21.15.32:0/1493044284 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5565b9020580 0x5565b9028740 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7efc340bc490 tx=0x7efc34009260 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8981f0c9bdc90f56 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:49.843 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.842+0000 7efc23fff640 1 -- 172.21.15.32:0/1493044284 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efc340d9030 con 0x5565b9020580 2024-04-25T00:36:49.843 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.843+0000 7efc23fff640 1 -- 172.21.15.32:0/1493044284 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7efc340f0040 con 0x5565b9020580 2024-04-25T00:36:49.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.848+0000 7efc431b4740 1 -- 172.21.15.32:0/1493044284 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5565b9020580 msgr2=0x5565b9028740 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:49.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.848+0000 7efc431b4740 1 --2- 172.21.15.32:0/1493044284 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5565b9020580 0x5565b9028740 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7efc340bc490 tx=0x7efc34009260 comp rx=0 tx=0).stop 2024-04-25T00:36:49.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.857+0000 7efc431b4740 1 -- 172.21.15.32:0/1493044284 shutdown_connections 2024-04-25T00:36:49.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.857+0000 7efc431b4740 1 --2- 172.21.15.32:0/1493044284 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5565b902b570 0x5565b902aa30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.857+0000 7efc431b4740 1 --2- 172.21.15.32:0/1493044284 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5565b9020ea0 0x5565b9037350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.857+0000 7efc431b4740 1 --2- 172.21.15.32:0/1493044284 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5565b9020580 0x5565b9028740 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.857+0000 7efc431b4740 1 -- 172.21.15.32:0/1493044284 >> 172.21.15.32:0/1493044284 conn(0x5565b901c8a0 msgr2=0x5565b9023720 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:49.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.864+0000 7efc431b4740 1 -- 172.21.15.32:0/1493044284 shutdown_connections 2024-04-25T00:36:49.865 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.864+0000 7efc431b4740 1 -- 172.21.15.32:0/1493044284 wait complete. 2024-04-25T00:36:49.865 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.864+0000 7efc431b4740 1 -- start start 2024-04-25T00:36:49.865 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.864+0000 7efc431b4740 1 Processor -- start 2024-04-25T00:36:49.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.865+0000 7efc431b4740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5565b9020580 0x5565b902e820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.865+0000 7efc431b4740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5565b9020ea0 0x5565b902ed60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.865+0000 7efc431b4740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5565b902b570 0x5565b902f2a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.865+0000 7efc431b4740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7efc1c00e710 con 0x5565b902b570 2024-04-25T00:36:49.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.865+0000 7efc431b4740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7efc1c00fb80 con 0x5565b9020580 2024-04-25T00:36:49.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.865+0000 7efc431b4740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7efc1c00ede0 con 0x5565b9020ea0 2024-04-25T00:36:49.866 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.865+0000 7efc431b4740 10 client.0 _ll_register_callbacks cb 0x7efc3000a8b0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:36:49.870 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.869+0000 7efc3da40640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5565b9020ea0 0x5565b902ed60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.870 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.869+0000 7efc3e241640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5565b9020580 0x5565b902e820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.870 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.869+0000 7efc3da40640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5565b9020ea0 0x5565b902ed60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:56490/0 (socket says 172.21.15.32:56490) 2024-04-25T00:36:49.870 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.869+0000 7efc3da40640 1 -- 172.21.15.32:0/696014016 learned_addr learned my addr 172.21.15.32:0/696014016 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:49.870 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.869+0000 7efc3ea42640 1 --2- 172.21.15.32:0/696014016 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5565b902b570 0x5565b902f2a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.876 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.876+0000 7efc3da40640 1 -- 172.21.15.32:0/696014016 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5565b9020580 msgr2=0x5565b902e820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:49.876 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.876+0000 7efc3da40640 1 --2- 172.21.15.32:0/696014016 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5565b9020580 0x5565b902e820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.876 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.876+0000 7efc3da40640 1 -- 172.21.15.32:0/696014016 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5565b902b570 msgr2=0x5565b902f2a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:49.876 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.876+0000 7efc3da40640 1 --2- 172.21.15.32:0/696014016 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5565b902b570 0x5565b902f2a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.876 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.876+0000 7efc3da40640 1 -- 172.21.15.32:0/696014016 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5565b90b1160 con 0x5565b9020ea0 2024-04-25T00:36:49.881 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.880+0000 7efc3da40640 1 --2- 172.21.15.32:0/696014016 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5565b9020ea0 0x5565b902ed60 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7efc30015020 tx=0x7efc30022610 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5d4e53b10d796c1e server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:49.881 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.880+0000 7efc21ffb640 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:36:49.881 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.881+0000 7efc21ffb640 1 -- 172.21.15.32:0/696014016 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efc30018bc0 con 0x5565b9020ea0 2024-04-25T00:36:49.882 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.881+0000 7efc431b4740 1 -- 172.21.15.32:0/696014016 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7efc340f0040 con 0x5565b9020ea0 2024-04-25T00:36:49.882 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.881+0000 7efc21ffb640 1 -- 172.21.15.32:0/696014016 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7efc30018620 con 0x5565b9020ea0 2024-04-25T00:36:49.890 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.889+0000 7efc21ffb640 1 -- 172.21.15.32:0/696014016 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7efc3003d050 con 0x5565b9020ea0 2024-04-25T00:36:49.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.892+0000 7efc431b4740 10 client.25702 fetch_fsmap learned FSMap version 27 2024-04-25T00:36:49.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.892+0000 7efc431b4740 1 -- 172.21.15.32:0/696014016 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x5565b90b2420 con 0x5565b9020ea0 2024-04-25T00:36:49.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.894+0000 7efc21ffb640 1 -- 172.21.15.32:0/696014016 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7efc340f0040 con 0x5565b9020ea0 2024-04-25T00:36:49.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.896+0000 7efc21ffb640 1 -- 172.21.15.32:0/696014016 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 27) ==== 8354+0+0 (secure 0 0 0) 0x7efc3002d420 con 0x5565b9020ea0 2024-04-25T00:36:49.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.897+0000 7efc431b4740 10 client.25702 fetch_fsmap finished waiting for FSMap version 27 2024-04-25T00:36:49.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.897+0000 7efc431b4740 10 client.25702 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:36:49.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.897+0000 7efc431b4740 20 client.25702 populate_metadata read hostname 'smithi032' 2024-04-25T00:36:49.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.897+0000 7efc431b4740 1 --2- 172.21.15.32:0/696014016 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5565b90b2c30 0x5565b90b5020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:49.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.897+0000 7efc431b4740 4 client.25702 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-25T00:36:49.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.897+0000 7efc431b4740 1 -- 172.21.15.32:0/696014016 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "damage ls"}) -- 0x5565b8f8cf00 con 0x5565b90b2c30 2024-04-25T00:36:49.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.900+0000 7efc3e241640 1 --2- 172.21.15.32:0/696014016 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5565b90b2c30 0x5565b90b5020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:49.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.905+0000 7efc3e241640 1 --2- 172.21.15.32:0/696014016 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5565b90b2c30 0x5565b90b5020 crc :-1 s=READY pgs=82 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=37c7bf43569f5c2 server_cookie=bda9730d08ddb87e in_seq=0 out_seq=0 2024-04-25T00:36:49.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.905+0000 7efc21ffb640 10 client.25702 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:49.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.909+0000 7efc21ffb640 1 -- 172.21.15.32:0/696014016 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x5565b8f8cf00 con 0x5565b90b2c30 2024-04-25T00:36:49.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.909+0000 7efc21ffb640 10 client.25702 handle_command_reply: tid=0 2024-04-25T00:36:49.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.909+0000 7efc431b4740 1 client.25702 shutdown 2024-04-25T00:36:49.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.910+0000 7efc431b4740 1 -- 172.21.15.32:0/696014016 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5565b9020ea0 msgr2=0x5565b902ed60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:49.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.910+0000 7efc431b4740 1 --2- 172.21.15.32:0/696014016 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5565b9020ea0 0x5565b902ed60 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7efc30015020 tx=0x7efc30022610 comp rx=0 tx=0).stop 2024-04-25T00:36:49.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.911+0000 7efc431b4740 1 -- 172.21.15.32:0/696014016 shutdown_connections 2024-04-25T00:36:49.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.911+0000 7efc431b4740 1 --2- 172.21.15.32:0/696014016 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5565b90b2c30 0x5565b90b5020 crc :-1 s=READY pgs=82 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.911+0000 7efc431b4740 1 --2- 172.21.15.32:0/696014016 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5565b902b570 0x5565b902f2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.911+0000 7efc431b4740 1 --2- 172.21.15.32:0/696014016 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5565b9020ea0 0x5565b902ed60 unknown :-1 s=CLOSED pgs=111 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.911+0000 7efc431b4740 1 --2- 172.21.15.32:0/696014016 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5565b9020580 0x5565b902e820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.911+0000 7efc431b4740 1 -- 172.21.15.32:0/696014016 >> 172.21.15.32:0/696014016 conn(0x5565b901c8a0 msgr2=0x5565b9027730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:49.912 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.911+0000 7efc21ffb640 0 client.25702 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:49.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.914+0000 7efc431b4740 1 -- 172.21.15.32:0/696014016 shutdown_connections 2024-04-25T00:36:49.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.914+0000 7efc431b4740 1 -- 172.21.15.32:0/696014016 wait complete. 2024-04-25T00:36:49.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.914+0000 7efc431b4740 20 client.25702 trim_cache size 0 max 16384 2024-04-25T00:36:49.920 INFO:teuthology.orchestra.run.smithi032.stdout:[] 2024-04-25T00:36:49.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.925+0000 7efc404a5640 1 -- 172.21.15.32:0/2634855717 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7efc240a2f10 msgr2=0x7efc240a53d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:49.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.925+0000 7efc404a5640 1 --2- 172.21.15.32:0/2634855717 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7efc240a2f10 0x7efc240a53d0 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x5565b9021b40 tx=0x7efc30009250 comp rx=0 tx=0).stop 2024-04-25T00:36:49.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.925+0000 7efc404a5640 1 -- 172.21.15.32:0/2634855717 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efc3810dfc0 msgr2=0x7efc381a6ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:49.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.925+0000 7efc404a5640 1 --2- 172.21.15.32:0/2634855717 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efc3810dfc0 0x7efc381a6ed0 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7efc34009900 tx=0x7efc34039520 comp rx=0 tx=0).stop 2024-04-25T00:36:49.929 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.928+0000 7efc404a5640 1 -- 172.21.15.32:0/2634855717 shutdown_connections 2024-04-25T00:36:49.929 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.929+0000 7efc404a5640 1 --2- 172.21.15.32:0/2634855717 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7efc240a2f10 0x7efc240a53d0 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.929 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.929+0000 7efc404a5640 1 --2- 172.21.15.32:0/2634855717 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efc3811af20 0x7efc381ab790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.929 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.929+0000 7efc404a5640 1 --2- 172.21.15.32:0/2634855717 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efc3810ff90 0x7efc381a7410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.929 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.929+0000 7efc404a5640 1 --2- 172.21.15.32:0/2634855717 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efc3810dfc0 0x7efc381a6ed0 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:49.930 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.929+0000 7efc404a5640 1 -- 172.21.15.32:0/2634855717 >> 172.21.15.32:0/2634855717 conn(0x7efc3806c850 msgr2=0x7efc38072240 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:49.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.932+0000 7efc404a5640 1 -- 172.21.15.32:0/2634855717 shutdown_connections 2024-04-25T00:36:49.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:49.936+0000 7efc404a5640 1 -- 172.21.15.32:0/2634855717 wait complete. 2024-04-25T00:36:51.000 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-25T00:36:51.000 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:36:51.001 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-25T00:36:51.317 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.316+0000 7f6b27800640 1 -- 172.21.15.32:0/975508010 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6b2010aa00 msgr2=0x7f6b20113150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:51.317 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.316+0000 7f6b27800640 1 --2- 172.21.15.32:0/975508010 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6b2010aa00 0x7f6b20113150 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f6b10009860 tx=0x7f6b1002a410 comp rx=0 tx=0).stop 2024-04-25T00:36:51.317 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.316+0000 7f6b27800640 1 -- 172.21.15.32:0/975508010 shutdown_connections 2024-04-25T00:36:51.317 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.316+0000 7f6b27800640 1 --2- 172.21.15.32:0/975508010 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6b20113690 0x7f6b20115a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.317 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.316+0000 7f6b27800640 1 --2- 172.21.15.32:0/975508010 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6b2010aa00 0x7f6b20113150 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.317 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.316+0000 7f6b27800640 1 --2- 172.21.15.32:0/975508010 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6b2010a0e0 0x7f6b2010a4c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.317 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.316+0000 7f6b27800640 1 -- 172.21.15.32:0/975508010 >> 172.21.15.32:0/975508010 conn(0x7f6b2006c850 msgr2=0x7f6b2006cc60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:51.318 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.317+0000 7f6b27800640 1 -- 172.21.15.32:0/975508010 shutdown_connections 2024-04-25T00:36:51.319 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.317+0000 7f6b27800640 1 -- 172.21.15.32:0/975508010 wait complete. 2024-04-25T00:36:51.319 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.318+0000 7f6b27800640 1 Processor -- start 2024-04-25T00:36:51.319 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.319+0000 7f6b27800640 1 -- start start 2024-04-25T00:36:51.320 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.319+0000 7f6b27800640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6b2010a0e0 0x7f6b20117260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.320 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.319+0000 7f6b27800640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6b2010aa00 0x7f6b201177a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.320 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.319+0000 7f6b27800640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6b20113690 0x7f6b20117ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.320 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.319+0000 7f6b27800640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f6b2011a2e0 con 0x7f6b2010a0e0 2024-04-25T00:36:51.320 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.319+0000 7f6b27800640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f6b2011a160 con 0x7f6b2010aa00 2024-04-25T00:36:51.320 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.319+0000 7f6b27800640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f6b2011a460 con 0x7f6b20113690 2024-04-25T00:36:51.321 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.320+0000 7f6b25d9d640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6b20113690 0x7f6b20117ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.321 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.320+0000 7f6b25d9d640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6b20113690 0x7f6b20117ce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:56500/0 (socket says 172.21.15.32:56500) 2024-04-25T00:36:51.321 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.320+0000 7f6b25d9d640 1 -- 172.21.15.32:0/857786137 learned_addr learned my addr 172.21.15.32:0/857786137 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:51.321 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.320+0000 7f6b2559c640 1 --2- 172.21.15.32:0/857786137 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6b2010a0e0 0x7f6b20117260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.322 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.321+0000 7f6b24d9b640 1 --2- 172.21.15.32:0/857786137 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6b2010aa00 0x7f6b201177a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.322 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.322+0000 7f6b25d9d640 1 -- 172.21.15.32:0/857786137 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6b2010aa00 msgr2=0x7f6b201177a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:51.323 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.322+0000 7f6b25d9d640 1 --2- 172.21.15.32:0/857786137 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6b2010aa00 0x7f6b201177a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.323 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.322+0000 7f6b25d9d640 1 -- 172.21.15.32:0/857786137 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6b2010a0e0 msgr2=0x7f6b20117260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:51.323 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.322+0000 7f6b25d9d640 1 --2- 172.21.15.32:0/857786137 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6b2010a0e0 0x7f6b20117260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.323 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.322+0000 7f6b25d9d640 1 -- 172.21.15.32:0/857786137 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6b201a49c0 con 0x7f6b20113690 2024-04-25T00:36:51.323 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.323+0000 7f6b25d9d640 1 --2- 172.21.15.32:0/857786137 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6b20113690 0x7f6b20117ce0 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f6b1800eb10 tx=0x7f6b1800ef10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:51.327 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.326+0000 7f6b167fc640 1 -- 172.21.15.32:0/857786137 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b1800ce20 con 0x7f6b20113690 2024-04-25T00:36:51.327 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.326+0000 7f6b167fc640 1 -- 172.21.15.32:0/857786137 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6b18005020 con 0x7f6b20113690 2024-04-25T00:36:51.327 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.326+0000 7f6b167fc640 1 -- 172.21.15.32:0/857786137 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b18010690 con 0x7f6b20113690 2024-04-25T00:36:51.327 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.326+0000 7f6b27800640 1 -- 172.21.15.32:0/857786137 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6b201a4cb0 con 0x7f6b20113690 2024-04-25T00:36:51.327 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.326+0000 7f6b27800640 1 -- 172.21.15.32:0/857786137 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f6b201a5180 con 0x7f6b20113690 2024-04-25T00:36:51.339 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.334+0000 7f6b2a5ed740 1 Processor -- start 2024-04-25T00:36:51.339 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.336+0000 7f6b2a5ed740 1 -- start start 2024-04-25T00:36:51.339 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.336+0000 7f6b2a5ed740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39cf980 0x5575f39cdcc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.339 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.336+0000 7f6b2a5ed740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575f39ce290 0x5575f39ce710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.340 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.336+0000 7f6b2a5ed740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575f39d9010 0x5575f39db400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.340 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.336+0000 7f6b2a5ed740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5575f39322c0 con 0x5575f39ce290 2024-04-25T00:36:51.340 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.336+0000 7f6b2a5ed740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5575f38da910 con 0x5575f39cf980 2024-04-25T00:36:51.340 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.336+0000 7f6b2a5ed740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5575f392e280 con 0x5575f39d9010 2024-04-25T00:36:51.340 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.339+0000 7f6b24d9b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39cf980 0x5575f39cdcc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.340 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.339+0000 7f6b24d9b640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575f39d9010 0x5575f39db400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.340+0000 7f6b24d9b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39cf980 0x5575f39cdcc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:47770/0 (socket says 172.21.15.32:47770) 2024-04-25T00:36:51.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.340+0000 7f6b24d9b640 1 -- 172.21.15.32:0/1361614142 learned_addr learned my addr 172.21.15.32:0/1361614142 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:51.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.341+0000 7f6b167fc640 1 -- 172.21.15.32:0/857786137 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f6b18004a40 con 0x7f6b20113690 2024-04-25T00:36:51.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.342+0000 7f6b24d9b640 1 -- 172.21.15.32:0/1361614142 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39cf980 msgr2=0x5575f39cdcc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:51.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.342+0000 7f6b24d9b640 1 --2- 172.21.15.32:0/1361614142 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39cf980 0x5575f39cdcc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.342+0000 7f6b24d9b640 1 -- 172.21.15.32:0/1361614142 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575f39ce290 msgr2=0x5575f39ce710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:51.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.343+0000 7f6b167fc640 1 --2- 172.21.15.32:0/857786137 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f6b0c0a2ed0 0x7f6b0c0a5390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.343+0000 7f6b167fc640 1 -- 172.21.15.32:0/857786137 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f6b180cfc70 con 0x7f6b20113690 2024-04-25T00:36:51.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.343+0000 7f6b2559c640 1 --2- 172.21.15.32:0/1361614142 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575f39ce290 0x5575f39ce710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.343+0000 7f6b24d9b640 1 --2- 172.21.15.32:0/1361614142 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575f39ce290 0x5575f39ce710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.343+0000 7f6b24d9b640 1 -- 172.21.15.32:0/1361614142 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5575f38ebfb0 con 0x5575f39d9010 2024-04-25T00:36:51.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.344+0000 7f6b2559c640 1 --2- 172.21.15.32:0/1361614142 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575f39ce290 0x5575f39ce710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:36:51.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.344+0000 7f6b24d9b640 1 --2- 172.21.15.32:0/1361614142 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575f39d9010 0x5575f39db400 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f6b1002a930 tx=0x7f6b1002a410 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e0dfdced8ec71df4 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:51.347 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.346+0000 7f6b24d9b640 1 --2- 172.21.15.32:0/857786137 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f6b0c0a2ed0 0x7f6b0c0a5390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.347 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.347+0000 7f6b0b7fe640 1 -- 172.21.15.32:0/1361614142 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b100307c0 con 0x5575f39d9010 2024-04-25T00:36:51.348 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.347+0000 7f6b0b7fe640 1 -- 172.21.15.32:0/1361614142 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6b10030e60 con 0x5575f39d9010 2024-04-25T00:36:51.349 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.348+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/1361614142 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575f39d9010 msgr2=0x5575f39db400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:51.349 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.348+0000 7f6b2a5ed740 1 --2- 172.21.15.32:0/1361614142 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575f39d9010 0x5575f39db400 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f6b1002a930 tx=0x7f6b1002a410 comp rx=0 tx=0).stop 2024-04-25T00:36:51.351 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.350+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/1361614142 shutdown_connections 2024-04-25T00:36:51.351 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.350+0000 7f6b2a5ed740 1 --2- 172.21.15.32:0/1361614142 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575f39d9010 0x5575f39db400 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.351 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.350+0000 7f6b2a5ed740 1 --2- 172.21.15.32:0/1361614142 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575f39ce290 0x5575f39ce710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.351 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.350+0000 7f6b2a5ed740 1 --2- 172.21.15.32:0/1361614142 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39cf980 0x5575f39cdcc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.351 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.350+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/1361614142 >> 172.21.15.32:0/1361614142 conn(0x5575f3996950 msgr2=0x5575f39ca7f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:51.351 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.350+0000 7f6b24d9b640 1 --2- 172.21.15.32:0/857786137 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f6b0c0a2ed0 0x7f6b0c0a5390 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f6b10005860 tx=0x7f6b10004e50 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:51.351 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.350+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/1361614142 shutdown_connections 2024-04-25T00:36:51.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.354+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/1361614142 wait complete. 2024-04-25T00:36:51.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.356+0000 7f6b2a5ed740 1 -- start start 2024-04-25T00:36:51.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.356+0000 7f6b2a5ed740 1 Processor -- start 2024-04-25T00:36:51.357 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.357+0000 7f6b2a5ed740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575f39ce290 0x5575f39d5d60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.357 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.357+0000 7f6b2a5ed740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575f39cf980 0x5575f39d62a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.357+0000 7f6b2a5ed740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39d9010 0x5575f39d67e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.358+0000 7f6b2a5ed740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5575f39dbe90 con 0x5575f39cf980 2024-04-25T00:36:51.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.358+0000 7f6b2a5ed740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5575f38ede90 con 0x5575f39d9010 2024-04-25T00:36:51.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.358+0000 7f6b2a5ed740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5575f39dc010 con 0x5575f39ce290 2024-04-25T00:36:51.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.358+0000 7f6b2a5ed740 10 client.0 _ll_register_callbacks cb 0x5575f3929970 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:36:51.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.358+0000 7f6b24d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575f39cf980 0x5575f39d62a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.359+0000 7f6b25d9d640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39d9010 0x5575f39d67e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.359+0000 7f6b25d9d640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39d9010 0x5575f39d67e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:47778/0 (socket says 172.21.15.32:47778) 2024-04-25T00:36:51.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.359+0000 7f6b25d9d640 1 -- 172.21.15.32:0/2013463042 learned_addr learned my addr 172.21.15.32:0/2013463042 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:51.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.359+0000 7f6b2559c640 1 --2- 172.21.15.32:0/2013463042 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575f39ce290 0x5575f39d5d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.360+0000 7f6b24d9b640 1 -- 172.21.15.32:0/2013463042 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575f39ce290 msgr2=0x5575f39d5d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:51.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.360+0000 7f6b24d9b640 1 --2- 172.21.15.32:0/2013463042 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575f39ce290 0x5575f39d5d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.360+0000 7f6b24d9b640 1 -- 172.21.15.32:0/2013463042 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39d9010 msgr2=0x5575f39d67e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:51.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.360+0000 7f6b24d9b640 1 --2- 172.21.15.32:0/2013463042 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39d9010 0x5575f39d67e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.360+0000 7f6b24d9b640 1 -- 172.21.15.32:0/2013463042 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5575f370e7e0 con 0x5575f39cf980 2024-04-25T00:36:51.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.360+0000 7f6b2559c640 1 --2- 172.21.15.32:0/2013463042 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575f39ce290 0x5575f39d5d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:36:51.362 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.360+0000 7f6b24d9b640 1 --2- 172.21.15.32:0/2013463042 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575f39cf980 0x5575f39d62a0 secure :-1 s=READY pgs=615 cs=0 l=1 rev1=1 crypto rx=0x7f6b1002a930 tx=0x7f6b10040000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3fa8f085d9eb60b server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:51.362 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.360+0000 7f6b08ff9640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:36:51.362 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.361+0000 7f6b08ff9640 1 -- 172.21.15.32:0/2013463042 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b10060070 con 0x5575f39cf980 2024-04-25T00:36:51.362 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.361+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/2013463042 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f6b10030e60 con 0x5575f39cf980 2024-04-25T00:36:51.362 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.361+0000 7f6b08ff9640 1 -- 172.21.15.32:0/2013463042 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f6b10005310 con 0x5575f39cf980 2024-04-25T00:36:51.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.362+0000 7f6b08ff9640 1 -- 172.21.15.32:0/2013463042 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f6b1006c050 con 0x5575f39cf980 2024-04-25T00:36:51.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.362+0000 7f6b2a5ed740 10 client.16170 fetch_fsmap learned FSMap version 27 2024-04-25T00:36:51.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.362+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/2013463042 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x5575f376ef70 con 0x5575f39cf980 2024-04-25T00:36:51.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.363+0000 7f6b08ff9640 1 -- 172.21.15.32:0/2013463042 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6b10030e60 con 0x5575f39cf980 2024-04-25T00:36:51.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.364+0000 7f6b08ff9640 1 -- 172.21.15.32:0/2013463042 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 27) ==== 8354+0+0 (secure 0 0 0) 0x7f6b10005510 con 0x5575f39cf980 2024-04-25T00:36:51.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.364+0000 7f6b2a5ed740 10 client.16170 fetch_fsmap finished waiting for FSMap version 27 2024-04-25T00:36:51.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.364+0000 7f6b2a5ed740 10 client.16170 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:36:51.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.364+0000 7f6b2a5ed740 20 client.16170 populate_metadata read hostname 'smithi032' 2024-04-25T00:36:51.366 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.364+0000 7f6b2a5ed740 1 --2- 172.21.15.32:0/2013463042 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5575f39e9110 0x5575f39eb590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.366 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.364+0000 7f6b2a5ed740 4 client.16170 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:36:51.366 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.364+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/2013463042 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5575f39065a0 con 0x5575f39e9110 2024-04-25T00:36:51.366 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.365+0000 7f6b2559c640 1 --2- 172.21.15.32:0/2013463042 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5575f39e9110 0x5575f39eb590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.368 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.367+0000 7f6b2559c640 1 --2- 172.21.15.32:0/2013463042 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5575f39e9110 0x5575f39eb590 crc :-1 s=READY pgs=83 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d16c8d9f95dcf5e9 server_cookie=e683415fcc6a0ca in_seq=0 out_seq=0 2024-04-25T00:36:51.368 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.367+0000 7f6b08ff9640 10 client.16170 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:51.368 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.368+0000 7f6b08ff9640 1 -- 172.21.15.32:0/2013463042 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x5575f39065a0 con 0x5575f39e9110 2024-04-25T00:36:51.368 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.368+0000 7f6b08ff9640 10 client.16170 handle_command_reply: tid=0 2024-04-25T00:36:51.369 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.368+0000 7f6b2a5ed740 1 client.16170 shutdown 2024-04-25T00:36:51.369 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.368+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/2013463042 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575f39cf980 msgr2=0x5575f39d62a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:51.369 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.368+0000 7f6b2a5ed740 1 --2- 172.21.15.32:0/2013463042 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575f39cf980 0x5575f39d62a0 secure :-1 s=READY pgs=615 cs=0 l=1 rev1=1 crypto rx=0x7f6b1002a930 tx=0x7f6b10040000 comp rx=0 tx=0).stop 2024-04-25T00:36:51.371 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.371+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/2013463042 shutdown_connections 2024-04-25T00:36:51.371 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.371+0000 7f6b2a5ed740 1 --2- 172.21.15.32:0/2013463042 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5575f39e9110 0x5575f39eb590 crc :-1 s=READY pgs=83 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.371 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.371+0000 7f6b2a5ed740 1 --2- 172.21.15.32:0/2013463042 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39d9010 0x5575f39d67e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.371 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.371+0000 7f6b2a5ed740 1 --2- 172.21.15.32:0/2013463042 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575f39cf980 0x5575f39d62a0 unknown :-1 s=CLOSED pgs=615 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.371 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.371+0000 7f6b2a5ed740 1 --2- 172.21.15.32:0/2013463042 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575f39ce290 0x5575f39d5d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.371+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/2013463042 >> 172.21.15.32:0/2013463042 conn(0x5575f3996950 msgr2=0x5575f39d05f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:51.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.371+0000 7f6b08ff9640 0 client.16170 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:51.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.371+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/2013463042 shutdown_connections 2024-04-25T00:36:51.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.372+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/2013463042 wait complete. 2024-04-25T00:36:51.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.372+0000 7f6b2a5ed740 20 client.16170 trim_cache size 0 max 16384 2024-04-25T00:36:51.409 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.408+0000 7f6b2a5ed740 1 Processor -- start 2024-04-25T00:36:51.409 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.408+0000 7f6b2a5ed740 1 -- start start 2024-04-25T00:36:51.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.409+0000 7f6b2a5ed740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575f39ce290 0x5575f39d4ec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.409+0000 7f6b2a5ed740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39cf980 0x5575f39d7ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.409+0000 7f6b2a5ed740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575f39d9010 0x5575f39f0e60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.409+0000 7f6b2a5ed740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f6b04012880 con 0x5575f39ce290 2024-04-25T00:36:51.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.409+0000 7f6b2a5ed740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f6b04010150 con 0x5575f39cf980 2024-04-25T00:36:51.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.409+0000 7f6b24d9b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39cf980 0x5575f39d7ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.409+0000 7f6b2a5ed740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f6b0400dfa0 con 0x5575f39d9010 2024-04-25T00:36:51.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.409+0000 7f6b24d9b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39cf980 0x5575f39d7ca0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:47788/0 (socket says 172.21.15.32:47788) 2024-04-25T00:36:51.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.409+0000 7f6b24d9b640 1 -- 172.21.15.32:0/2816179464 learned_addr learned my addr 172.21.15.32:0/2816179464 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:51.411 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.410+0000 7f6b2559c640 1 --2- 172.21.15.32:0/2816179464 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575f39ce290 0x5575f39d4ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.411 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.410+0000 7f6b24d9b640 1 -- 172.21.15.32:0/2816179464 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575f39d9010 msgr2=0x5575f39f0e60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:51.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.410+0000 7f6b24d9b640 1 --2- 172.21.15.32:0/2816179464 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575f39d9010 0x5575f39f0e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.410+0000 7f6b24d9b640 1 -- 172.21.15.32:0/2816179464 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575f39ce290 msgr2=0x5575f39d4ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:51.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.410+0000 7f6b24d9b640 1 --2- 172.21.15.32:0/2816179464 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575f39ce290 0x5575f39d4ec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.410+0000 7f6b24d9b640 1 -- 172.21.15.32:0/2816179464 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5575f3606170 con 0x5575f39cf980 2024-04-25T00:36:51.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.411+0000 7f6b24d9b640 1 --2- 172.21.15.32:0/2816179464 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39cf980 0x5575f39d7ca0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f6b1003deb0 tx=0x7f6b10030c80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=29f01791fb79c51 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:51.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.411+0000 7f6b0affd640 1 -- 172.21.15.32:0/2816179464 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b10060030 con 0x5575f39cf980 2024-04-25T00:36:51.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.411+0000 7f6b0affd640 1 -- 172.21.15.32:0/2816179464 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6b1005ce50 con 0x5575f39cf980 2024-04-25T00:36:51.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.412+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/2816179464 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39cf980 msgr2=0x5575f39d7ca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:51.413 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.412+0000 7f6b2a5ed740 1 --2- 172.21.15.32:0/2816179464 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39cf980 0x5575f39d7ca0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f6b1003deb0 tx=0x7f6b10030c80 comp rx=0 tx=0).stop 2024-04-25T00:36:51.413 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.413+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/2816179464 shutdown_connections 2024-04-25T00:36:51.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.413+0000 7f6b2a5ed740 1 --2- 172.21.15.32:0/2816179464 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575f39d9010 0x5575f39f0e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.413+0000 7f6b2a5ed740 1 --2- 172.21.15.32:0/2816179464 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39cf980 0x5575f39d7ca0 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.413+0000 7f6b2a5ed740 1 --2- 172.21.15.32:0/2816179464 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575f39ce290 0x5575f39d4ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.413+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/2816179464 >> 172.21.15.32:0/2816179464 conn(0x5575f3996950 msgr2=0x5575f39d7590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:51.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.413+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/2816179464 shutdown_connections 2024-04-25T00:36:51.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.413+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/2816179464 wait complete. 2024-04-25T00:36:51.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.414+0000 7f6b2a5ed740 1 -- start start 2024-04-25T00:36:51.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.414+0000 7f6b2a5ed740 1 Processor -- start 2024-04-25T00:36:51.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.415+0000 7f6b2a5ed740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575f39ce290 0x5575f39e1640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.415+0000 7f6b2a5ed740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575f39cf980 0x5575f39dca00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.415+0000 7f6b2a5ed740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39d9010 0x5575f39dcf40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.415+0000 7f6b2a5ed740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f6b0400e610 con 0x5575f39ce290 2024-04-25T00:36:51.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.415+0000 7f6b2a5ed740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f6b0400fa80 con 0x5575f39d9010 2024-04-25T00:36:51.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.415+0000 7f6b2a5ed740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f6b0400ece0 con 0x5575f39cf980 2024-04-25T00:36:51.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.415+0000 7f6b2a5ed740 10 client.0 _ll_register_callbacks cb 0x7f6b10042020 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:36:51.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.415+0000 7f6b25d9d640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39d9010 0x5575f39dcf40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.415+0000 7f6b25d9d640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39d9010 0x5575f39dcf40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:47802/0 (socket says 172.21.15.32:47802) 2024-04-25T00:36:51.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.415+0000 7f6b25d9d640 1 -- 172.21.15.32:0/3696085749 learned_addr learned my addr 172.21.15.32:0/3696085749 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:51.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.415+0000 7f6b25d9d640 1 -- 172.21.15.32:0/3696085749 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575f39cf980 msgr2=0x5575f39dca00 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-25T00:36:51.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.415+0000 7f6b25d9d640 1 --2- 172.21.15.32:0/3696085749 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575f39cf980 0x5575f39dca00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.415+0000 7f6b25d9d640 1 -- 172.21.15.32:0/3696085749 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575f39ce290 msgr2=0x5575f39e1640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:51.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.415+0000 7f6b25d9d640 1 --2- 172.21.15.32:0/3696085749 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575f39ce290 0x5575f39e1640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.415+0000 7f6b25d9d640 1 -- 172.21.15.32:0/3696085749 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5575f39dd630 con 0x5575f39d9010 2024-04-25T00:36:51.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.417+0000 7f6b25d9d640 1 --2- 172.21.15.32:0/3696085749 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39d9010 0x5575f39dcf40 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f6b180981e0 tx=0x7f6b1809be40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=62851a3296e31766 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:51.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.417+0000 7f6b08ff9640 10 client.0 ms_handle_connect on v2:172.21.15.121:3300/0 2024-04-25T00:36:51.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.418+0000 7f6b08ff9640 1 -- 172.21.15.32:0/3696085749 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b180b8020 con 0x5575f39d9010 2024-04-25T00:36:51.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.418+0000 7f6b08ff9640 1 -- 172.21.15.32:0/3696085749 <== mon.2 v2:172.21.15.121:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f6b180a4560 con 0x5575f39d9010 2024-04-25T00:36:51.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.418+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/3696085749 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f6b1005ce50 con 0x5575f39d9010 2024-04-25T00:36:51.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.419+0000 7f6b08ff9640 1 -- 172.21.15.32:0/3696085749 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f6b180cd430 con 0x5575f39d9010 2024-04-25T00:36:51.420 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.420+0000 7f6b2a5ed740 10 client.25742 fetch_fsmap learned FSMap version 27 2024-04-25T00:36:51.420 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.420+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/3696085749 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x5575f3a4f670 con 0x5575f39d9010 2024-04-25T00:36:51.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.421+0000 7f6b08ff9640 1 -- 172.21.15.32:0/3696085749 <== mon.2 v2:172.21.15.121:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6b1005ce50 con 0x5575f39d9010 2024-04-25T00:36:51.421 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.421+0000 7f6b08ff9640 1 -- 172.21.15.32:0/3696085749 <== mon.2 v2:172.21.15.121:3300/0 5 ==== fsmap(e 27) ==== 8354+0+0 (secure 0 0 0) 0x7f6b1809ac40 con 0x5575f39d9010 2024-04-25T00:36:51.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.421+0000 7f6b2a5ed740 10 client.25742 fetch_fsmap finished waiting for FSMap version 27 2024-04-25T00:36:51.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.421+0000 7f6b2a5ed740 10 client.25742 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:36:51.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.421+0000 7f6b2a5ed740 20 client.25742 populate_metadata read hostname 'smithi032' 2024-04-25T00:36:51.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.421+0000 7f6b2a5ed740 1 --2- 172.21.15.32:0/3696085749 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5575f3a4fe30 0x5575f3a52250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.421+0000 7f6b2a5ed740 4 client.25742 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-25T00:36:51.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.421+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/3696085749 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) -- 0x5575f393f800 con 0x5575f3a4fe30 2024-04-25T00:36:51.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.422+0000 7f6b2559c640 1 --2- 172.21.15.32:0/3696085749 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5575f3a4fe30 0x5575f3a52250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.424 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.424+0000 7f6b08ff9640 10 client.25742 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:51.424 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.424+0000 7f6b2559c640 1 --2- 172.21.15.32:0/3696085749 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5575f3a4fe30 0x5575f3a52250 crc :-1 s=READY pgs=84 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=cb22b38479e812a8 server_cookie=b04b988d24ed01c8 in_seq=0 out_seq=0 2024-04-25T00:36:51.426 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.425+0000 7f6b08ff9640 1 -- 172.21.15.32:0/3696085749 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+110 (crc 0 0 0) 0x5575f393f800 con 0x5575f3a4fe30 2024-04-25T00:36:51.426 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.425+0000 7f6b08ff9640 10 client.25742 handle_command_reply: tid=0 2024-04-25T00:36:51.426 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.425+0000 7f6b2a5ed740 1 client.25742 shutdown 2024-04-25T00:36:51.426 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.425+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/3696085749 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39d9010 msgr2=0x5575f39dcf40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:51.426 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.425+0000 7f6b2a5ed740 1 --2- 172.21.15.32:0/3696085749 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39d9010 0x5575f39dcf40 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f6b180981e0 tx=0x7f6b1809be40 comp rx=0 tx=0).stop 2024-04-25T00:36:51.426 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.425+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/3696085749 shutdown_connections 2024-04-25T00:36:51.426 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.425+0000 7f6b2a5ed740 1 --2- 172.21.15.32:0/3696085749 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5575f3a4fe30 0x5575f3a52250 crc :-1 s=READY pgs=84 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.426 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.425+0000 7f6b2a5ed740 1 --2- 172.21.15.32:0/3696085749 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5575f39d9010 0x5575f39dcf40 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.426 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.425+0000 7f6b2a5ed740 1 --2- 172.21.15.32:0/3696085749 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5575f39cf980 0x5575f39dca00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.426 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.425+0000 7f6b2a5ed740 1 --2- 172.21.15.32:0/3696085749 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5575f39ce290 0x5575f39e1640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.427 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.425+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/3696085749 >> 172.21.15.32:0/3696085749 conn(0x5575f3996950 msgr2=0x5575f39c90c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:51.427 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.425+0000 7f6b08ff9640 0 client.25742 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:51.427 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.425+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/3696085749 shutdown_connections 2024-04-25T00:36:51.427 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.425+0000 7f6b2a5ed740 1 -- 172.21.15.32:0/3696085749 wait complete. 2024-04-25T00:36:51.427 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.426+0000 7f6b2a5ed740 20 client.25742 trim_cache size 0 max 16384 2024-04-25T00:36:51.427 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:36:51.427 INFO:teuthology.orchestra.run.smithi032.stdout: "return_code": 0, 2024-04-25T00:36:51.427 INFO:teuthology.orchestra.run.smithi032.stdout: "scrub_tag": "7e53acfd-17ef-40c7-bf04-0e74589c65f0", 2024-04-25T00:36:51.428 INFO:teuthology.orchestra.run.smithi032.stdout: "mode": "asynchronous" 2024-04-25T00:36:51.428 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:36:51.429 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.427+0000 7f6b27800640 1 -- 172.21.15.32:0/857786137 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f6b0c0a2ed0 msgr2=0x7f6b0c0a5390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:51.429 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.427+0000 7f6b27800640 1 --2- 172.21.15.32:0/857786137 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f6b0c0a2ed0 0x7f6b0c0a5390 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f6b10005860 tx=0x7f6b10004e50 comp rx=0 tx=0).stop 2024-04-25T00:36:51.429 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.427+0000 7f6b27800640 1 -- 172.21.15.32:0/857786137 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6b20113690 msgr2=0x7f6b20117ce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:51.429 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.427+0000 7f6b27800640 1 --2- 172.21.15.32:0/857786137 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6b20113690 0x7f6b20117ce0 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f6b1800eb10 tx=0x7f6b1800ef10 comp rx=0 tx=0).stop 2024-04-25T00:36:51.429 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.427+0000 7f6b27800640 1 -- 172.21.15.32:0/857786137 shutdown_connections 2024-04-25T00:36:51.429 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.427+0000 7f6b27800640 1 --2- 172.21.15.32:0/857786137 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f6b0c0a2ed0 0x7f6b0c0a5390 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.429 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.427+0000 7f6b27800640 1 --2- 172.21.15.32:0/857786137 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6b20113690 0x7f6b20117ce0 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.429 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.427+0000 7f6b27800640 1 --2- 172.21.15.32:0/857786137 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6b2010aa00 0x7f6b201177a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.429 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.427+0000 7f6b27800640 1 --2- 172.21.15.32:0/857786137 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6b2010a0e0 0x7f6b20117260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.429 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.427+0000 7f6b27800640 1 -- 172.21.15.32:0/857786137 >> 172.21.15.32:0/857786137 conn(0x7f6b2006c850 msgr2=0x7f6b200704b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:51.429 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.427+0000 7f6b27800640 1 -- 172.21.15.32:0/857786137 shutdown_connections 2024-04-25T00:36:51.429 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.427+0000 7f6b27800640 1 -- 172.21.15.32:0/857786137 wait complete. 2024-04-25T00:36:51.443 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-25T00:36:51.443 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-25T00:36:51.443 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:51 smithi032 ceph-mon[29357]: pgmap v109: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 2.3 MiB/s rd, 796 MiB/s wr, 1.72k op/s 2024-04-25T00:36:51.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:51 smithi121 ceph-mon[32231]: pgmap v109: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 2.3 MiB/s rd, 796 MiB/s wr, 1.72k op/s 2024-04-25T00:36:51.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:51 smithi130 ceph-mon[32525]: pgmap v109: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 2.3 MiB/s rd, 796 MiB/s wr, 1.72k op/s 2024-04-25T00:36:51.708 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.698+0000 7fbc3d161640 1 -- 172.21.15.32:0/3985837291 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbc3811af20 msgr2=0x7fbc3811d310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:51.708 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.698+0000 7fbc3d161640 1 --2- 172.21.15.32:0/3985837291 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbc3811af20 0x7fbc3811d310 secure :-1 s=READY pgs=616 cs=0 l=1 rev1=1 crypto rx=0x7fbc2c01a940 tx=0x7fbc2c03b490 comp rx=0 tx=0).stop 2024-04-25T00:36:51.716 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.712+0000 7fbc3d161640 1 -- 172.21.15.32:0/3985837291 shutdown_connections 2024-04-25T00:36:51.717 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.712+0000 7fbc3d161640 1 --2- 172.21.15.32:0/3985837291 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbc3811af20 0x7fbc3811d310 unknown :-1 s=CLOSED pgs=616 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.717 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.712+0000 7fbc3d161640 1 --2- 172.21.15.32:0/3985837291 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbc3810dfc0 0x7fbc3810e440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.717 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.712+0000 7fbc3d161640 1 --2- 172.21.15.32:0/3985837291 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbc3810ff90 0x7fbc3810da80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.717 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.712+0000 7fbc3d161640 1 -- 172.21.15.32:0/3985837291 >> 172.21.15.32:0/3985837291 conn(0x7fbc3806c850 msgr2=0x7fbc3806cc60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:51.717 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.717+0000 7fbc3d161640 1 -- 172.21.15.32:0/3985837291 shutdown_connections 2024-04-25T00:36:51.722 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.721+0000 7fbc3d161640 1 -- 172.21.15.32:0/3985837291 wait complete. 2024-04-25T00:36:51.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.722+0000 7fbc3d161640 1 Processor -- start 2024-04-25T00:36:51.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.723+0000 7fbc3d161640 1 -- start start 2024-04-25T00:36:51.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.723+0000 7fbc3d161640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbc3810dfc0 0x7fbc3811a770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.723+0000 7fbc3d161640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbc3810ff90 0x7fbc381168a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.723+0000 7fbc3d161640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbc3811af20 0x7fbc38116de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.723+0000 7fbc3d161640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fbc38113170 con 0x7fbc3811af20 2024-04-25T00:36:51.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.723+0000 7fbc3d161640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fbc38112ff0 con 0x7fbc3810dfc0 2024-04-25T00:36:51.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.723+0000 7fbc3d161640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fbc381132f0 con 0x7fbc3810ff90 2024-04-25T00:36:51.725 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.724+0000 7fbc3759e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbc3811af20 0x7fbc38116de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.726 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.724+0000 7fbc3759e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbc3811af20 0x7fbc38116de0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:43706/0 (socket says 172.21.15.32:43706) 2024-04-25T00:36:51.726 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.724+0000 7fbc3759e640 1 -- 172.21.15.32:0/368024729 learned_addr learned my addr 172.21.15.32:0/368024729 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:51.726 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.726+0000 7fbc36d9d640 1 --2- 172.21.15.32:0/368024729 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbc3810dfc0 0x7fbc3811a770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.728 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.728+0000 7fbc36d9d640 1 -- 172.21.15.32:0/368024729 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbc3810ff90 msgr2=0x7fbc381168a0 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-04-25T00:36:51.729 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.728+0000 7fbc36d9d640 1 --2- 172.21.15.32:0/368024729 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbc3810ff90 0x7fbc381168a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.729 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.728+0000 7fbc36d9d640 1 -- 172.21.15.32:0/368024729 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbc3811af20 msgr2=0x7fbc38116de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:51.729 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.728+0000 7fbc36d9d640 1 --2- 172.21.15.32:0/368024729 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbc3811af20 0x7fbc38116de0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.729 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.728+0000 7fbc36d9d640 1 -- 172.21.15.32:0/368024729 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbc381176a0 con 0x7fbc3810dfc0 2024-04-25T00:36:51.731 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.731+0000 7fbc36d9d640 1 --2- 172.21.15.32:0/368024729 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbc3810dfc0 0x7fbc3811a770 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7fbc28009fd0 tx=0x7fbc2800b870 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:51.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.733+0000 7fbc1ffff640 1 -- 172.21.15.32:0/368024729 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbc28015020 con 0x7fbc3810dfc0 2024-04-25T00:36:51.734 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.733+0000 7fbc3d161640 1 -- 172.21.15.32:0/368024729 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbc381ccd20 con 0x7fbc3810dfc0 2024-04-25T00:36:51.734 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.733+0000 7fbc3d161640 1 -- 172.21.15.32:0/368024729 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fbc381cd190 con 0x7fbc3810dfc0 2024-04-25T00:36:51.744 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.743+0000 7fbc3fe6c740 1 Processor -- start 2024-04-25T00:36:51.744 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.743+0000 7fbc3fe6c740 1 -- start start 2024-04-25T00:36:51.744 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.744+0000 7fbc3fe6c740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55a5bc6c8630 0x55a5bc6c8a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.745 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.744+0000 7fbc3fe6c740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55a5bc6c8f50 0x55a5bc6d30e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.745 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.744+0000 7fbc3fe6c740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55a5bc6d3620 0x55a5bc6d5a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.745 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.744+0000 7fbc3fe6c740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55a5bc62d6b0 con 0x55a5bc6c8630 2024-04-25T00:36:51.745 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.744+0000 7fbc3fe6c740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55a5bc5d5880 con 0x55a5bc6d3620 2024-04-25T00:36:51.746 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.744+0000 7fbc3fe6c740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55a5bc629670 con 0x55a5bc6c8f50 2024-04-25T00:36:51.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.745+0000 7fbc1ffff640 1 -- 172.21.15.32:0/368024729 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbc28011d10 con 0x7fbc3810dfc0 2024-04-25T00:36:51.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.745+0000 7fbc1ffff640 1 -- 172.21.15.32:0/368024729 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbc2800f2e0 con 0x7fbc3810dfc0 2024-04-25T00:36:51.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.745+0000 7fbc1ffff640 1 -- 172.21.15.32:0/368024729 <== mon.2 v2:172.21.15.121:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7fbc28018020 con 0x7fbc3810dfc0 2024-04-25T00:36:51.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.746+0000 7fbc3759e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55a5bc6c8630 0x55a5bc6c8a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.746+0000 7fbc3759e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55a5bc6c8630 0x55a5bc6c8a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:43710/0 (socket says 172.21.15.32:43710) 2024-04-25T00:36:51.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.746+0000 7fbc3759e640 1 -- 172.21.15.32:0/1671926630 learned_addr learned my addr 172.21.15.32:0/1671926630 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:51.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.746+0000 7fbc3659c640 1 --2- 172.21.15.32:0/1671926630 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55a5bc6d3620 0x55a5bc6d5a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.747 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.746+0000 7fbc1ffff640 1 --2- 172.21.15.32:0/368024729 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fbc200a2f30 0x7fbc200a53f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.748 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.747+0000 7fbc1ffff640 1 -- 172.21.15.32:0/368024729 <== mon.2 v2:172.21.15.121:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fbc280cffd0 con 0x7fbc3810dfc0 2024-04-25T00:36:51.748 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.747+0000 7fbc3659c640 1 -- 172.21.15.32:0/1671926630 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55a5bc6c8f50 msgr2=0x55a5bc6d30e0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-25T00:36:51.748 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.747+0000 7fbc3659c640 1 --2- 172.21.15.32:0/1671926630 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55a5bc6c8f50 0x55a5bc6d30e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.748 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.747+0000 7fbc3659c640 1 -- 172.21.15.32:0/1671926630 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55a5bc6c8630 msgr2=0x55a5bc6c8a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:51.748 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.747+0000 7fbc3659c640 1 --2- 172.21.15.32:0/1671926630 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55a5bc6c8630 0x55a5bc6c8a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.749 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.748+0000 7fbc3659c640 1 -- 172.21.15.32:0/1671926630 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a5bc40b7d0 con 0x55a5bc6d3620 2024-04-25T00:36:51.749 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.748+0000 7fbc3659c640 1 --2- 172.21.15.32:0/1671926630 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55a5bc6d3620 0x55a5bc6d5a10 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7fbc38118050 tx=0x7fbc3000b7c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f18555eb0b85e0ce server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:51.753 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.752+0000 7fbc1d7fa640 1 -- 172.21.15.32:0/1671926630 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbc30018070 con 0x55a5bc6d3620 2024-04-25T00:36:51.753 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.752+0000 7fbc1d7fa640 1 -- 172.21.15.32:0/1671926630 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbc300138f0 con 0x55a5bc6d3620 2024-04-25T00:36:51.753 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.752+0000 7fbc3759e640 1 --2- 172.21.15.32:0/368024729 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fbc200a2f30 0x7fbc200a53f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.756 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.754+0000 7fbc3759e640 1 --2- 172.21.15.32:0/368024729 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fbc200a2f30 0x7fbc200a53f0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7fbc2c005420 tx=0x7fbc2c01a270 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:51.756 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.754+0000 7fbc1d7fa640 1 -- 172.21.15.32:0/1671926630 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbc30012bb0 con 0x55a5bc6d3620 2024-04-25T00:36:51.756 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.756+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/1671926630 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55a5bc6d3620 msgr2=0x55a5bc6d5a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:51.756 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.756+0000 7fbc3fe6c740 1 --2- 172.21.15.32:0/1671926630 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55a5bc6d3620 0x55a5bc6d5a10 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7fbc38118050 tx=0x7fbc3000b7c0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.759+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/1671926630 shutdown_connections 2024-04-25T00:36:51.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.759+0000 7fbc3fe6c740 1 --2- 172.21.15.32:0/1671926630 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55a5bc6d3620 0x55a5bc6d5a10 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.759+0000 7fbc3fe6c740 1 --2- 172.21.15.32:0/1671926630 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55a5bc6c8f50 0x55a5bc6d30e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.759+0000 7fbc3fe6c740 1 --2- 172.21.15.32:0/1671926630 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55a5bc6c8630 0x55a5bc6c8a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.759+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/1671926630 >> 172.21.15.32:0/1671926630 conn(0x55a5bc6c4950 msgr2=0x55a5bc6c4d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:51.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.769+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/1671926630 shutdown_connections 2024-04-25T00:36:51.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.770+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/1671926630 wait complete. 2024-04-25T00:36:51.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.770+0000 7fbc3fe6c740 1 -- start start 2024-04-25T00:36:51.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.770+0000 7fbc3fe6c740 1 Processor -- start 2024-04-25T00:36:51.772 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.771+0000 7fbc3fe6c740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55a5bc6c8630 0x55a5bc6d0600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.772 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.771+0000 7fbc3fe6c740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55a5bc6c8f50 0x55a5bc6d0b40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.772 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.771+0000 7fbc3fe6c740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55a5bc6d3620 0x55a5bc6d1080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.772 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.771+0000 7fbc3fe6c740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55a5bc6d6410 con 0x55a5bc6c8630 2024-04-25T00:36:51.772 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.771+0000 7fbc3fe6c740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55a5bc5e8f50 con 0x55a5bc6c8f50 2024-04-25T00:36:51.772 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.771+0000 7fbc3fe6c740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55a5bc6d6590 con 0x55a5bc6d3620 2024-04-25T00:36:51.772 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.771+0000 7fbc3fe6c740 10 client.0 _ll_register_callbacks cb 0x55a5bc624000 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:36:51.772 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.772+0000 7fbc3759e640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55a5bc6d3620 0x55a5bc6d1080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.772 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.772+0000 7fbc3759e640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55a5bc6d3620 0x55a5bc6d1080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:53188/0 (socket says 172.21.15.32:53188) 2024-04-25T00:36:51.773 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.772+0000 7fbc3759e640 1 -- 172.21.15.32:0/840853426 learned_addr learned my addr 172.21.15.32:0/840853426 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:51.777 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.777+0000 7fbc3759e640 1 -- 172.21.15.32:0/840853426 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55a5bc6c8f50 msgr2=0x55a5bc6d0b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:51.777 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.777+0000 7fbc3759e640 1 --2- 172.21.15.32:0/840853426 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55a5bc6c8f50 0x55a5bc6d0b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.778 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.777+0000 7fbc3759e640 1 -- 172.21.15.32:0/840853426 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55a5bc6c8630 msgr2=0x55a5bc6d0600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:51.778 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.777+0000 7fbc3759e640 1 --2- 172.21.15.32:0/840853426 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55a5bc6c8630 0x55a5bc6d0600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.778 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.777+0000 7fbc3759e640 1 -- 172.21.15.32:0/840853426 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a5bc3413f0 con 0x55a5bc6d3620 2024-04-25T00:36:51.782 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.781+0000 7fbc3759e640 1 --2- 172.21.15.32:0/840853426 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55a5bc6d3620 0x55a5bc6d1080 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7fbc2c004a70 tx=0x7fbc2c004aa0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=193371f89c31b87b server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:51.786 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.785+0000 7fbc1aff5640 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:36:51.786 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.785+0000 7fbc1aff5640 1 -- 172.21.15.32:0/840853426 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbc2c050020 con 0x55a5bc6d3620 2024-04-25T00:36:51.786 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.785+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/840853426 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fbc300138f0 con 0x55a5bc6d3620 2024-04-25T00:36:51.786 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.785+0000 7fbc1aff5640 1 -- 172.21.15.32:0/840853426 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fbc2c040ca0 con 0x55a5bc6d3620 2024-04-25T00:36:51.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.787+0000 7fbc1aff5640 1 -- 172.21.15.32:0/840853426 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fbc2c05b050 con 0x55a5bc6d3620 2024-04-25T00:36:51.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.793+0000 7fbc3fe6c740 10 client.25714 fetch_fsmap learned FSMap version 27 2024-04-25T00:36:51.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.793+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/840853426 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x55a5bc3010f0 con 0x55a5bc6d3620 2024-04-25T00:36:51.800 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.800+0000 7fbc1aff5640 1 -- 172.21.15.32:0/840853426 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbc300138f0 con 0x55a5bc6d3620 2024-04-25T00:36:51.801 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.801+0000 7fbc1aff5640 1 -- 172.21.15.32:0/840853426 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 27) ==== 8354+0+0 (secure 0 0 0) 0x7fbc2c011440 con 0x55a5bc6d3620 2024-04-25T00:36:51.802 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.801+0000 7fbc3fe6c740 10 client.25714 fetch_fsmap finished waiting for FSMap version 27 2024-04-25T00:36:51.802 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.801+0000 7fbc3fe6c740 10 client.25714 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:36:51.802 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.801+0000 7fbc3fe6c740 20 client.25714 populate_metadata read hostname 'smithi032' 2024-04-25T00:36:51.802 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.801+0000 7fbc3fe6c740 1 --2- 172.21.15.32:0/840853426 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55a5bc6e3ae0 0x55a5bc6e5fa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.802 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.801+0000 7fbc3fe6c740 4 client.25714 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:36:51.802 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.801+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/840853426 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55a5bc63abf0 con 0x55a5bc6e3ae0 2024-04-25T00:36:51.802 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.802+0000 7fbc36d9d640 1 --2- 172.21.15.32:0/840853426 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55a5bc6e3ae0 0x55a5bc6e5fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.811 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.805+0000 7fbc36d9d640 1 --2- 172.21.15.32:0/840853426 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55a5bc6e3ae0 0x55a5bc6e5fa0 crc :-1 s=READY pgs=85 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=59297cbf80d56fe9 server_cookie=2f726b79c413333c in_seq=0 out_seq=0 2024-04-25T00:36:51.811 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.806+0000 7fbc1aff5640 10 client.25714 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:51.811 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.810+0000 7fbc1aff5640 1 -- 172.21.15.32:0/840853426 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x55a5bc63abf0 con 0x55a5bc6e3ae0 2024-04-25T00:36:51.811 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.810+0000 7fbc1aff5640 10 client.25714 handle_command_reply: tid=0 2024-04-25T00:36:51.811 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.810+0000 7fbc3fe6c740 1 client.25714 shutdown 2024-04-25T00:36:51.812 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.811+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/840853426 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55a5bc6d3620 msgr2=0x55a5bc6d1080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:51.812 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.811+0000 7fbc3fe6c740 1 --2- 172.21.15.32:0/840853426 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55a5bc6d3620 0x55a5bc6d1080 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7fbc2c004a70 tx=0x7fbc2c004aa0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.813 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.812+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/840853426 shutdown_connections 2024-04-25T00:36:51.813 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.812+0000 7fbc3fe6c740 1 --2- 172.21.15.32:0/840853426 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55a5bc6e3ae0 0x55a5bc6e5fa0 crc :-1 s=READY pgs=85 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.813 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.812+0000 7fbc3fe6c740 1 --2- 172.21.15.32:0/840853426 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55a5bc6d3620 0x55a5bc6d1080 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.813 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.812+0000 7fbc3fe6c740 1 --2- 172.21.15.32:0/840853426 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55a5bc6c8f50 0x55a5bc6d0b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.813 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.812+0000 7fbc3fe6c740 1 --2- 172.21.15.32:0/840853426 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55a5bc6c8630 0x55a5bc6d0600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.813 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.812+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/840853426 >> 172.21.15.32:0/840853426 conn(0x55a5bc6c4950 msgr2=0x55a5bc6c3af0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:51.815 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.815+0000 7fbc1aff5640 0 client.25714 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:51.818 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.816+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/840853426 shutdown_connections 2024-04-25T00:36:51.818 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.817+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/840853426 wait complete. 2024-04-25T00:36:51.818 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.817+0000 7fbc3fe6c740 20 client.25714 trim_cache size 0 max 16384 2024-04-25T00:36:51.859 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.858+0000 7fbc3fe6c740 1 Processor -- start 2024-04-25T00:36:51.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.861+0000 7fbc3fe6c740 1 -- start start 2024-04-25T00:36:51.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.861+0000 7fbc3fe6c740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55a5bc6c8630 0x55a5bc6d2d30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.861+0000 7fbc3fe6c740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55a5bc6c8f50 0x55a5bc6df360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.861+0000 7fbc3fe6c740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55a5bc6d3620 0x55a5bc6eb870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.861+0000 7fbc3fe6c740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fbc240127b0 con 0x55a5bc6c8630 2024-04-25T00:36:51.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.862+0000 7fbc3fe6c740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fbc24010080 con 0x55a5bc6c8f50 2024-04-25T00:36:51.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.862+0000 7fbc3fe6c740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fbc2400df80 con 0x55a5bc6d3620 2024-04-25T00:36:51.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.862+0000 7fbc36d9d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55a5bc6c8630 0x55a5bc6d2d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.862+0000 7fbc36d9d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55a5bc6c8630 0x55a5bc6d2d30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:43726/0 (socket says 172.21.15.32:43726) 2024-04-25T00:36:51.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.862+0000 7fbc36d9d640 1 -- 172.21.15.32:0/2824959571 learned_addr learned my addr 172.21.15.32:0/2824959571 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:51.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.863+0000 7fbc36d9d640 1 -- 172.21.15.32:0/2824959571 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55a5bc6d3620 msgr2=0x55a5bc6eb870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:51.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.863+0000 7fbc36d9d640 1 --2- 172.21.15.32:0/2824959571 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55a5bc6d3620 0x55a5bc6eb870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.863+0000 7fbc36d9d640 1 -- 172.21.15.32:0/2824959571 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55a5bc6c8f50 msgr2=0x55a5bc6df360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:51.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.863+0000 7fbc36d9d640 1 --2- 172.21.15.32:0/2824959571 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55a5bc6c8f50 0x55a5bc6df360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.865 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.863+0000 7fbc36d9d640 1 -- 172.21.15.32:0/2824959571 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a5bc6b6090 con 0x55a5bc6c8630 2024-04-25T00:36:51.865 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.864+0000 7fbc36d9d640 1 --2- 172.21.15.32:0/2824959571 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55a5bc6c8630 0x55a5bc6d2d30 secure :-1 s=READY pgs=617 cs=0 l=1 rev1=1 crypto rx=0x7fbc2800bd70 tx=0x7fbc2800bf20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a6b517c95b87734a server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:51.865 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.864+0000 7fbc1cff9640 1 -- 172.21.15.32:0/2824959571 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbc280af070 con 0x55a5bc6c8630 2024-04-25T00:36:51.865 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.865+0000 7fbc1cff9640 1 -- 172.21.15.32:0/2824959571 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbc280a4ee0 con 0x55a5bc6c8630 2024-04-25T00:36:51.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.866+0000 7fbc1cff9640 1 -- 172.21.15.32:0/2824959571 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbc280a4a50 con 0x55a5bc6c8630 2024-04-25T00:36:51.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.866+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/2824959571 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55a5bc6c8630 msgr2=0x55a5bc6d2d30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:51.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.866+0000 7fbc3fe6c740 1 --2- 172.21.15.32:0/2824959571 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55a5bc6c8630 0x55a5bc6d2d30 secure :-1 s=READY pgs=617 cs=0 l=1 rev1=1 crypto rx=0x7fbc2800bd70 tx=0x7fbc2800bf20 comp rx=0 tx=0).stop 2024-04-25T00:36:51.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.866+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/2824959571 shutdown_connections 2024-04-25T00:36:51.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.866+0000 7fbc3fe6c740 1 --2- 172.21.15.32:0/2824959571 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55a5bc6d3620 0x55a5bc6eb870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.866+0000 7fbc3fe6c740 1 --2- 172.21.15.32:0/2824959571 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55a5bc6c8f50 0x55a5bc6df360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.866+0000 7fbc3fe6c740 1 --2- 172.21.15.32:0/2824959571 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55a5bc6c8630 0x55a5bc6d2d30 unknown :-1 s=CLOSED pgs=617 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.867 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.866+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/2824959571 >> 172.21.15.32:0/2824959571 conn(0x55a5bc6c4950 msgr2=0x55a5bc6cb8a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:51.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.867+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/2824959571 shutdown_connections 2024-04-25T00:36:51.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.867+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/2824959571 wait complete. 2024-04-25T00:36:51.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.867+0000 7fbc3fe6c740 1 -- start start 2024-04-25T00:36:51.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.867+0000 7fbc3fe6c740 1 Processor -- start 2024-04-25T00:36:51.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.868+0000 7fbc3fe6c740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55a5bc6c8630 0x55a5bc6dba60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.868+0000 7fbc3fe6c740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55a5bc6c8f50 0x55a5bc6cce70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.868+0000 7fbc3fe6c740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55a5bc6d3620 0x55a5bc6cd3b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.868+0000 7fbc3fe6c740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fbc2400e540 con 0x55a5bc6d3620 2024-04-25T00:36:51.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.868+0000 7fbc3fe6c740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fbc2400f9b0 con 0x55a5bc6c8f50 2024-04-25T00:36:51.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.868+0000 7fbc3fe6c740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fbc2400ec10 con 0x55a5bc6c8630 2024-04-25T00:36:51.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.868+0000 7fbc3fe6c740 10 client.0 _ll_register_callbacks cb 0x7fbc2c0055a0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:36:51.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.868+0000 7fbc3659c640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55a5bc6c8f50 0x55a5bc6cce70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.868+0000 7fbc3659c640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55a5bc6c8f50 0x55a5bc6cce70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:58402/0 (socket says 172.21.15.32:58402) 2024-04-25T00:36:51.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.868+0000 7fbc3659c640 1 -- 172.21.15.32:0/3840893198 learned_addr learned my addr 172.21.15.32:0/3840893198 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:36:51.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.869+0000 7fbc3759e640 1 --2- 172.21.15.32:0/3840893198 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55a5bc6d3620 0x55a5bc6cd3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.869+0000 7fbc3659c640 1 -- 172.21.15.32:0/3840893198 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55a5bc6c8630 msgr2=0x55a5bc6dba60 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-25T00:36:51.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.869+0000 7fbc3659c640 1 --2- 172.21.15.32:0/3840893198 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55a5bc6c8630 0x55a5bc6dba60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.869+0000 7fbc3659c640 1 -- 172.21.15.32:0/3840893198 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55a5bc6d3620 msgr2=0x55a5bc6cd3b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:36:51.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.869+0000 7fbc3659c640 1 --2- 172.21.15.32:0/3840893198 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55a5bc6d3620 0x55a5bc6cd3b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.869+0000 7fbc3659c640 1 -- 172.21.15.32:0/3840893198 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a5bc6cd9f0 con 0x55a5bc6c8f50 2024-04-25T00:36:51.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.869+0000 7fbc3659c640 1 --2- 172.21.15.32:0/3840893198 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55a5bc6c8f50 0x55a5bc6cce70 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x55a5bc6c5ae0 tx=0x7fbc30004270 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1df0ac12cfa225c server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:36:51.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.870+0000 7fbc1aff5640 10 client.0 ms_handle_connect on v2:172.21.15.121:3300/0 2024-04-25T00:36:51.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.870+0000 7fbc1aff5640 1 -- 172.21.15.32:0/3840893198 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbc30020020 con 0x55a5bc6c8f50 2024-04-25T00:36:51.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.870+0000 7fbc1aff5640 1 -- 172.21.15.32:0/3840893198 <== mon.2 v2:172.21.15.121:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fbc300043e0 con 0x55a5bc6c8f50 2024-04-25T00:36:51.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.870+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/3840893198 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fbc280a4ee0 con 0x55a5bc6c8f50 2024-04-25T00:36:51.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.870+0000 7fbc1aff5640 1 -- 172.21.15.32:0/3840893198 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fbc3002b050 con 0x55a5bc6c8f50 2024-04-25T00:36:51.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.870+0000 7fbc3fe6c740 10 client.25751 fetch_fsmap learned FSMap version 27 2024-04-25T00:36:51.872 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.870+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/3840893198 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x55a5bc749f50 con 0x55a5bc6c8f50 2024-04-25T00:36:51.872 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.870+0000 7fbc1aff5640 1 -- 172.21.15.32:0/3840893198 <== mon.2 v2:172.21.15.121:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbc280a4ee0 con 0x55a5bc6c8f50 2024-04-25T00:36:51.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.871+0000 7fbc1aff5640 1 -- 172.21.15.32:0/3840893198 <== mon.2 v2:172.21.15.121:3300/0 5 ==== fsmap(e 27) ==== 8354+0+0 (secure 0 0 0) 0x7fbc3001b050 con 0x55a5bc6c8f50 2024-04-25T00:36:51.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.871+0000 7fbc3fe6c740 10 client.25751 fetch_fsmap finished waiting for FSMap version 27 2024-04-25T00:36:51.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.871+0000 7fbc3fe6c740 10 client.25751 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:36:51.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.871+0000 7fbc3fe6c740 20 client.25751 populate_metadata read hostname 'smithi032' 2024-04-25T00:36:51.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.871+0000 7fbc3fe6c740 1 --2- 172.21.15.32:0/3840893198 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55a5bc74a4e0 0x55a5bc74c900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:36:51.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.871+0000 7fbc3fe6c740 4 client.25751 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-25T00:36:51.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.871+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/3840893198 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "scrub status"}) -- 0x55a5bc634f40 con 0x55a5bc74a4e0 2024-04-25T00:36:51.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.871+0000 7fbc36d9d640 1 --2- 172.21.15.32:0/3840893198 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55a5bc74a4e0 0x55a5bc74c900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:36:51.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.873+0000 7fbc1aff5640 10 client.25751 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:51.876 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.873+0000 7fbc36d9d640 1 --2- 172.21.15.32:0/3840893198 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55a5bc74a4e0 0x55a5bc74c900 crc :-1 s=READY pgs=86 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1d00f2a4de70f219 server_cookie=b0faa581eb28204e in_seq=0 out_seq=0 2024-04-25T00:36:51.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.876+0000 7fbc1aff5640 1 -- 172.21.15.32:0/3840893198 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+265 (crc 0 0 0) 0x55a5bc634f40 con 0x55a5bc74a4e0 2024-04-25T00:36:51.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.876+0000 7fbc1aff5640 10 client.25751 handle_command_reply: tid=0 2024-04-25T00:36:51.877 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.876+0000 7fbc3fe6c740 1 client.25751 shutdown 2024-04-25T00:36:51.878 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.877+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/3840893198 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55a5bc6c8f50 msgr2=0x55a5bc6cce70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:51.878 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.877+0000 7fbc3fe6c740 1 --2- 172.21.15.32:0/3840893198 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55a5bc6c8f50 0x55a5bc6cce70 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x55a5bc6c5ae0 tx=0x7fbc30004270 comp rx=0 tx=0).stop 2024-04-25T00:36:51.878 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.877+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/3840893198 shutdown_connections 2024-04-25T00:36:51.878 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.877+0000 7fbc3fe6c740 1 --2- 172.21.15.32:0/3840893198 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55a5bc74a4e0 0x55a5bc74c900 crc :-1 s=READY pgs=86 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.878 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.877+0000 7fbc3fe6c740 1 --2- 172.21.15.32:0/3840893198 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55a5bc6d3620 0x55a5bc6cd3b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.878 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.877+0000 7fbc3fe6c740 1 --2- 172.21.15.32:0/3840893198 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55a5bc6c8f50 0x55a5bc6cce70 secure :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0x55a5bc6c5ae0 tx=0x7fbc30004270 comp rx=0 tx=0).stop 2024-04-25T00:36:51.878 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.877+0000 7fbc3fe6c740 1 --2- 172.21.15.32:0/3840893198 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55a5bc6c8630 0x55a5bc6dba60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.879 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.877+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/3840893198 >> 172.21.15.32:0/3840893198 conn(0x55a5bc6c4950 msgr2=0x55a5bc6c3260 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:51.879 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.879+0000 7fbc1aff5640 0 client.25751 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:36:51.882 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.881+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/3840893198 shutdown_connections 2024-04-25T00:36:51.882 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.882+0000 7fbc3fe6c740 1 -- 172.21.15.32:0/3840893198 wait complete. 2024-04-25T00:36:51.883 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.883+0000 7fbc3fe6c740 20 client.25751 trim_cache size 0 max 16384 2024-04-25T00:36:51.884 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:36:51.884 INFO:teuthology.orchestra.run.smithi032.stdout: "status": "scrub active (0 inodes in the stack)", 2024-04-25T00:36:51.884 INFO:teuthology.orchestra.run.smithi032.stdout: "scrubs": { 2024-04-25T00:36:51.884 INFO:teuthology.orchestra.run.smithi032.stdout: "7e53acfd-17ef-40c7-bf04-0e74589c65f0": { 2024-04-25T00:36:51.884 INFO:teuthology.orchestra.run.smithi032.stdout: "path": "/", 2024-04-25T00:36:51.885 INFO:teuthology.orchestra.run.smithi032.stdout: "tag": "7e53acfd-17ef-40c7-bf04-0e74589c65f0", 2024-04-25T00:36:51.885 INFO:teuthology.orchestra.run.smithi032.stdout: "options": "recursive,force" 2024-04-25T00:36:51.885 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:36:51.885 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:36:51.885 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:36:51.885 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.884+0000 7fbc3d161640 1 -- 172.21.15.32:0/368024729 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fbc200a2f30 msgr2=0x7fbc200a53f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:51.885 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.884+0000 7fbc3d161640 1 --2- 172.21.15.32:0/368024729 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fbc200a2f30 0x7fbc200a53f0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7fbc2c005420 tx=0x7fbc2c01a270 comp rx=0 tx=0).stop 2024-04-25T00:36:51.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.884+0000 7fbc3d161640 1 -- 172.21.15.32:0/368024729 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbc3810dfc0 msgr2=0x7fbc3811a770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:36:51.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.884+0000 7fbc3d161640 1 --2- 172.21.15.32:0/368024729 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbc3810dfc0 0x7fbc3811a770 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7fbc28009fd0 tx=0x7fbc2800b870 comp rx=0 tx=0).stop 2024-04-25T00:36:51.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.884+0000 7fbc3d161640 1 -- 172.21.15.32:0/368024729 shutdown_connections 2024-04-25T00:36:51.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.884+0000 7fbc3d161640 1 --2- 172.21.15.32:0/368024729 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fbc200a2f30 0x7fbc200a53f0 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.884+0000 7fbc3d161640 1 --2- 172.21.15.32:0/368024729 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fbc3811af20 0x7fbc38116de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.884+0000 7fbc3d161640 1 --2- 172.21.15.32:0/368024729 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbc3810ff90 0x7fbc381168a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.884+0000 7fbc3d161640 1 --2- 172.21.15.32:0/368024729 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fbc3810dfc0 0x7fbc3811a770 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:36:51.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.884+0000 7fbc3d161640 1 -- 172.21.15.32:0/368024729 >> 172.21.15.32:0/368024729 conn(0x7fbc3806c850 msgr2=0x7fbc3811c870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:36:51.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.885+0000 7fbc3d161640 1 -- 172.21.15.32:0/368024729 shutdown_connections 2024-04-25T00:36:51.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:36:51.887+0000 7fbc3d161640 1 -- 172.21.15.32:0/368024729 wait complete. 2024-04-25T00:36:51.915 INFO:tasks.cephfs.filesystem:scrub status for tag:7e53acfd-17ef-40c7-bf04-0e74589c65f0 - {'path': '/', 'tag': '7e53acfd-17ef-40c7-bf04-0e74589c65f0', 'options': 'recursive,force'} 2024-04-25T00:36:52.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:52 smithi121 ceph-mon[32231]: scrub summary: active paths [/] 2024-04-25T00:36:52.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:52 smithi121 ceph-mon[32231]: scrub summary: idle+waiting paths [/] 2024-04-25T00:36:52.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:52 smithi130 ceph-mon[32525]: scrub summary: active paths [/] 2024-04-25T00:36:52.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:52 smithi130 ceph-mon[32525]: scrub summary: idle+waiting paths [/] 2024-04-25T00:36:52.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:52 smithi032 ceph-mon[29357]: scrub summary: active paths [/] 2024-04-25T00:36:52.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:52 smithi032 ceph-mon[29357]: scrub summary: idle+waiting paths [/] 2024-04-25T00:36:53.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:53 smithi130 ceph-mon[32525]: scrub queued for path: / 2024-04-25T00:36:53.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:53 smithi130 ceph-mon[32525]: scrub summary: idle+waiting paths [/] 2024-04-25T00:36:53.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:53 smithi130 ceph-mon[32525]: scrub summary: active paths [/] 2024-04-25T00:36:53.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:53 smithi130 ceph-mon[32525]: scrub summary: active paths [/] 2024-04-25T00:36:53.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:53 smithi130 ceph-mon[32525]: pgmap v110: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 1.0 MiB/s rd, 611 MiB/s wr, 1.50k op/s 2024-04-25T00:36:53.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:53 smithi032 ceph-mon[29357]: scrub queued for path: / 2024-04-25T00:36:53.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:53 smithi032 ceph-mon[29357]: scrub summary: idle+waiting paths [/] 2024-04-25T00:36:53.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:53 smithi032 ceph-mon[29357]: scrub summary: active paths [/] 2024-04-25T00:36:53.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:53 smithi032 ceph-mon[29357]: scrub summary: active paths [/] 2024-04-25T00:36:53.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:53 smithi032 ceph-mon[29357]: pgmap v110: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 1.0 MiB/s rd, 611 MiB/s wr, 1.50k op/s 2024-04-25T00:36:53.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:53 smithi121 ceph-mon[32231]: scrub queued for path: / 2024-04-25T00:36:53.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:53 smithi121 ceph-mon[32231]: scrub summary: idle+waiting paths [/] 2024-04-25T00:36:53.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:53 smithi121 ceph-mon[32231]: scrub summary: active paths [/] 2024-04-25T00:36:53.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:53 smithi121 ceph-mon[32231]: scrub summary: active paths [/] 2024-04-25T00:36:53.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:53 smithi121 ceph-mon[32231]: pgmap v110: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 1.0 MiB/s rd, 611 MiB/s wr, 1.50k op/s 2024-04-25T00:36:54.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:54 smithi130 ceph-mon[32525]: scrub summary: idle+waiting paths [/] 2024-04-25T00:36:54.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:54 smithi032 ceph-mon[29357]: scrub summary: idle+waiting paths [/] 2024-04-25T00:36:54.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:54 smithi121 ceph-mon[32231]: scrub summary: idle+waiting paths [/] 2024-04-25T00:36:55.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:55 smithi130 ceph-mon[32525]: scrub summary: idle+waiting paths [/] 2024-04-25T00:36:55.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:55 smithi130 ceph-mon[32525]: pgmap v111: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 1.6 MiB/s rd, 637 MiB/s wr, 1.55k op/s 2024-04-25T00:36:55.739 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:55 smithi121 ceph-mon[32231]: scrub summary: idle+waiting paths [/] 2024-04-25T00:36:55.739 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:55 smithi121 ceph-mon[32231]: pgmap v111: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 1.6 MiB/s rd, 637 MiB/s wr, 1.55k op/s 2024-04-25T00:36:55.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:55 smithi032 ceph-mon[29357]: scrub summary: idle+waiting paths [/] 2024-04-25T00:36:55.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:55 smithi032 ceph-mon[29357]: pgmap v111: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 1.6 MiB/s rd, 637 MiB/s wr, 1.55k op/s 2024-04-25T00:36:57.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:57 smithi032 ceph-mon[29357]: pgmap v112: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 746 MiB/s wr, 1.59k op/s 2024-04-25T00:36:57.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:57 smithi121 ceph-mon[32231]: pgmap v112: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 746 MiB/s wr, 1.59k op/s 2024-04-25T00:36:57.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:57 smithi130 ceph-mon[32525]: pgmap v112: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 746 MiB/s wr, 1.59k op/s 2024-04-25T00:36:59.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:36:59 smithi032 ceph-mon[29357]: pgmap v113: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 934 KiB/s rd, 637 MiB/s wr, 1.19k op/s 2024-04-25T00:36:59.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:36:59 smithi121 ceph-mon[32231]: pgmap v113: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 934 KiB/s rd, 637 MiB/s wr, 1.19k op/s 2024-04-25T00:36:59.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:36:59 smithi130 ceph-mon[32525]: pgmap v113: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1009 GiB / 1.0 TiB avail; 934 KiB/s rd, 637 MiB/s wr, 1.19k op/s 2024-04-25T00:37:01.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:01 smithi032 ceph-mon[29357]: pgmap v114: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1008 GiB / 1.0 TiB avail; 1.7 MiB/s rd, 795 MiB/s wr, 1.45k op/s 2024-04-25T00:37:01.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:01 smithi121 ceph-mon[32231]: pgmap v114: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1008 GiB / 1.0 TiB avail; 1.7 MiB/s rd, 795 MiB/s wr, 1.45k op/s 2024-04-25T00:37:01.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:01 smithi130 ceph-mon[32525]: pgmap v114: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1008 GiB / 1.0 TiB avail; 1.7 MiB/s rd, 795 MiB/s wr, 1.45k op/s 2024-04-25T00:37:02.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:02 smithi032 ceph-mon[29357]: scrub summary: idle 2024-04-25T00:37:02.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:02 smithi121 ceph-mon[32231]: scrub summary: idle 2024-04-25T00:37:02.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:02 smithi130 ceph-mon[32525]: scrub summary: idle 2024-04-25T00:37:03.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:03 smithi032 ceph-mon[29357]: pgmap v115: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1008 GiB / 1.0 TiB avail; 1.6 MiB/s rd, 661 MiB/s wr, 970 op/s 2024-04-25T00:37:03.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:03 smithi032 ceph-mon[29357]: scrub summary: idle 2024-04-25T00:37:03.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:03 smithi032 ceph-mon[29357]: scrub summary: idle 2024-04-25T00:37:03.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:03 smithi121 ceph-mon[32231]: pgmap v115: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1008 GiB / 1.0 TiB avail; 1.6 MiB/s rd, 661 MiB/s wr, 970 op/s 2024-04-25T00:37:03.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:03 smithi121 ceph-mon[32231]: scrub summary: idle 2024-04-25T00:37:03.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:03 smithi121 ceph-mon[32231]: scrub summary: idle 2024-04-25T00:37:03.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:03 smithi130 ceph-mon[32525]: pgmap v115: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1008 GiB / 1.0 TiB avail; 1.6 MiB/s rd, 661 MiB/s wr, 970 op/s 2024-04-25T00:37:03.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:03 smithi130 ceph-mon[32525]: scrub summary: idle 2024-04-25T00:37:03.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:03 smithi130 ceph-mon[32525]: scrub summary: idle 2024-04-25T00:37:05.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:05 smithi032 ceph-mon[29357]: pgmap v116: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1008 GiB / 1.0 TiB avail; 1.7 MiB/s rd, 758 MiB/s wr, 1.39k op/s 2024-04-25T00:37:05.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:05 smithi121 ceph-mon[32231]: pgmap v116: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1008 GiB / 1.0 TiB avail; 1.7 MiB/s rd, 758 MiB/s wr, 1.39k op/s 2024-04-25T00:37:05.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:05 smithi130 ceph-mon[32525]: pgmap v116: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1008 GiB / 1.0 TiB avail; 1.7 MiB/s rd, 758 MiB/s wr, 1.39k op/s 2024-04-25T00:37:07.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:07 smithi032 ceph-mon[29357]: pgmap v117: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1008 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 753 MiB/s wr, 1.67k op/s 2024-04-25T00:37:07.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:07 smithi121 ceph-mon[32231]: pgmap v117: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1008 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 753 MiB/s wr, 1.67k op/s 2024-04-25T00:37:07.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:07 smithi130 ceph-mon[32525]: pgmap v117: 129 pgs: 129 active+clean; 30 GiB data, 64 GiB used, 1008 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 753 MiB/s wr, 1.67k op/s 2024-04-25T00:37:09.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:09 smithi032 ceph-mon[29357]: pgmap v118: 129 pgs: 129 active+clean; 30 GiB data, 65 GiB used, 1008 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 618 MiB/s wr, 1.76k op/s 2024-04-25T00:37:09.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:09 smithi121 ceph-mon[32231]: pgmap v118: 129 pgs: 129 active+clean; 30 GiB data, 65 GiB used, 1008 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 618 MiB/s wr, 1.76k op/s 2024-04-25T00:37:09.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:09 smithi130 ceph-mon[32525]: pgmap v118: 129 pgs: 129 active+clean; 30 GiB data, 65 GiB used, 1008 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 618 MiB/s wr, 1.76k op/s 2024-04-25T00:37:11.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:11 smithi032 ceph-mon[29357]: pgmap v119: 129 pgs: 129 active+clean; 31 GiB data, 66 GiB used, 1007 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 783 MiB/s wr, 3.11k op/s 2024-04-25T00:37:11.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:11 smithi121 ceph-mon[32231]: pgmap v119: 129 pgs: 129 active+clean; 31 GiB data, 66 GiB used, 1007 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 783 MiB/s wr, 3.11k op/s 2024-04-25T00:37:11.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:11 smithi130 ceph-mon[32525]: pgmap v119: 129 pgs: 129 active+clean; 31 GiB data, 66 GiB used, 1007 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 783 MiB/s wr, 3.11k op/s 2024-04-25T00:37:13.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:13 smithi032 ceph-mon[29357]: pgmap v120: 129 pgs: 129 active+clean; 31 GiB data, 66 GiB used, 1007 GiB / 1.0 TiB avail; 429 KiB/s rd, 558 MiB/s wr, 2.62k op/s 2024-04-25T00:37:13.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:13 smithi121 ceph-mon[32231]: pgmap v120: 129 pgs: 129 active+clean; 31 GiB data, 66 GiB used, 1007 GiB / 1.0 TiB avail; 429 KiB/s rd, 558 MiB/s wr, 2.62k op/s 2024-04-25T00:37:13.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:13 smithi130 ceph-mon[32525]: pgmap v120: 129 pgs: 129 active+clean; 31 GiB data, 66 GiB used, 1007 GiB / 1.0 TiB avail; 429 KiB/s rd, 558 MiB/s wr, 2.62k op/s 2024-04-25T00:37:15.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:15 smithi121 ceph-mon[32231]: pgmap v121: 129 pgs: 129 active+clean; 31 GiB data, 67 GiB used, 1006 GiB / 1.0 TiB avail; 429 KiB/s rd, 594 MiB/s wr, 3.08k op/s 2024-04-25T00:37:15.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:15 smithi130 ceph-mon[32525]: pgmap v121: 129 pgs: 129 active+clean; 31 GiB data, 67 GiB used, 1006 GiB / 1.0 TiB avail; 429 KiB/s rd, 594 MiB/s wr, 3.08k op/s 2024-04-25T00:37:15.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:15 smithi032 ceph-mon[29357]: pgmap v121: 129 pgs: 129 active+clean; 31 GiB data, 67 GiB used, 1006 GiB / 1.0 TiB avail; 429 KiB/s rd, 594 MiB/s wr, 3.08k op/s 2024-04-25T00:37:17.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:17 smithi121 ceph-mon[32231]: pgmap v122: 129 pgs: 129 active+clean; 32 GiB data, 68 GiB used, 1005 GiB / 1.0 TiB avail; 718 KiB/s rd, 571 MiB/s wr, 3.59k op/s 2024-04-25T00:37:17.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:17 smithi130 ceph-mon[32525]: pgmap v122: 129 pgs: 129 active+clean; 32 GiB data, 68 GiB used, 1005 GiB / 1.0 TiB avail; 718 KiB/s rd, 571 MiB/s wr, 3.59k op/s 2024-04-25T00:37:17.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:17 smithi032 ceph-mon[29357]: pgmap v122: 129 pgs: 129 active+clean; 32 GiB data, 68 GiB used, 1005 GiB / 1.0 TiB avail; 718 KiB/s rd, 571 MiB/s wr, 3.59k op/s 2024-04-25T00:37:19.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:19 smithi121 ceph-mon[32231]: pgmap v123: 129 pgs: 129 active+clean; 32 GiB data, 68 GiB used, 1004 GiB / 1.0 TiB avail; 548 KiB/s rd, 479 MiB/s wr, 3.21k op/s 2024-04-25T00:37:19.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:19 smithi130 ceph-mon[32525]: pgmap v123: 129 pgs: 129 active+clean; 32 GiB data, 68 GiB used, 1004 GiB / 1.0 TiB avail; 548 KiB/s rd, 479 MiB/s wr, 3.21k op/s 2024-04-25T00:37:19.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:19 smithi032 ceph-mon[29357]: pgmap v123: 129 pgs: 129 active+clean; 32 GiB data, 68 GiB used, 1004 GiB / 1.0 TiB avail; 548 KiB/s rd, 479 MiB/s wr, 3.21k op/s 2024-04-25T00:37:21.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:21 smithi121 ceph-mon[32231]: pgmap v124: 129 pgs: 129 active+clean; 35 GiB data, 73 GiB used, 1000 GiB / 1.0 TiB avail; 766 KiB/s rd, 650 MiB/s wr, 3.32k op/s 2024-04-25T00:37:21.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:21 smithi130 ceph-mon[32525]: pgmap v124: 129 pgs: 129 active+clean; 35 GiB data, 73 GiB used, 1000 GiB / 1.0 TiB avail; 766 KiB/s rd, 650 MiB/s wr, 3.32k op/s 2024-04-25T00:37:21.915 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-25T00:37:21.916 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-25T00:37:21.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:21 smithi032 ceph-mon[29357]: pgmap v124: 129 pgs: 129 active+clean; 35 GiB data, 73 GiB used, 1000 GiB / 1.0 TiB avail; 766 KiB/s rd, 650 MiB/s wr, 3.32k op/s 2024-04-25T00:37:22.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.241+0000 7fda0b74f640 1 -- 172.21.15.32:0/4216847676 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fda0410dfc0 msgr2=0x7fda0410e440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:22.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.241+0000 7fda0b74f640 1 --2- 172.21.15.32:0/4216847676 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fda0410dfc0 0x7fda0410e440 secure :-1 s=READY pgs=618 cs=0 l=1 rev1=1 crypto rx=0x7fda00009890 tx=0x7fda0002a3b0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.242+0000 7fda0b74f640 1 -- 172.21.15.32:0/4216847676 shutdown_connections 2024-04-25T00:37:22.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.242+0000 7fda0b74f640 1 --2- 172.21.15.32:0/4216847676 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fda0411af20 0x7fda0411d310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.242+0000 7fda0b74f640 1 --2- 172.21.15.32:0/4216847676 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fda0410dfc0 0x7fda0410e440 unknown :-1 s=CLOSED pgs=618 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.242+0000 7fda0b74f640 1 --2- 172.21.15.32:0/4216847676 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fda0410ff90 0x7fda0410da80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.242+0000 7fda0b74f640 1 -- 172.21.15.32:0/4216847676 >> 172.21.15.32:0/4216847676 conn(0x7fda0406c850 msgr2=0x7fda0406cc60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:22.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.248+0000 7fda0b74f640 1 -- 172.21.15.32:0/4216847676 shutdown_connections 2024-04-25T00:37:22.253 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.249+0000 7fda0b74f640 1 -- 172.21.15.32:0/4216847676 wait complete. 2024-04-25T00:37:22.253 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.250+0000 7fda0b74f640 1 Processor -- start 2024-04-25T00:37:22.257 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.254+0000 7fda0b74f640 1 -- start start 2024-04-25T00:37:22.257 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.255+0000 7fda0b74f640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fda0410dfc0 0x7fda04118620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:22.258 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.255+0000 7fda0b74f640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fda0410ff90 0x7fda041116a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:22.258 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.255+0000 7fda0b74f640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fda0411af20 0x7fda04111be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:22.258 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.255+0000 7fda0b74f640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fda041208e0 con 0x7fda0410dfc0 2024-04-25T00:37:22.260 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.255+0000 7fda0b74f640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fda04120760 con 0x7fda0411af20 2024-04-25T00:37:22.260 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.255+0000 7fda0b74f640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fda04120a60 con 0x7fda0410ff90 2024-04-25T00:37:22.260 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.255+0000 7fda08cea640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fda0410ff90 0x7fda041116a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:22.260 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.255+0000 7fda08cea640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fda0410ff90 0x7fda041116a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:60600/0 (socket says 172.21.15.32:60600) 2024-04-25T00:37:22.260 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.255+0000 7fda08cea640 1 -- 172.21.15.32:0/3445725822 learned_addr learned my addr 172.21.15.32:0/3445725822 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:22.261 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.255+0000 7fda08cea640 1 -- 172.21.15.32:0/3445725822 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fda0411af20 msgr2=0x7fda04111be0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:37:22.261 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.255+0000 7fda08cea640 1 --2- 172.21.15.32:0/3445725822 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fda0411af20 0x7fda04111be0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.261 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.256+0000 7fda08cea640 1 -- 172.21.15.32:0/3445725822 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fda0410dfc0 msgr2=0x7fda04118620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:22.261 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.256+0000 7fda08cea640 1 --2- 172.21.15.32:0/3445725822 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fda0410dfc0 0x7fda04118620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.261 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.256+0000 7fda08cea640 1 -- 172.21.15.32:0/3445725822 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fda04112310 con 0x7fda0410ff90 2024-04-25T00:37:22.261 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.256+0000 7fda08cea640 1 --2- 172.21.15.32:0/3445725822 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fda0410ff90 0x7fda041116a0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7fda00005d00 tx=0x7fda00033d10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:22.261 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.256+0000 7fd9fa7fc640 1 -- 172.21.15.32:0/3445725822 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fda0003f020 con 0x7fda0410ff90 2024-04-25T00:37:22.261 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.259+0000 7fda0b74f640 1 -- 172.21.15.32:0/3445725822 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fda04116140 con 0x7fda0410ff90 2024-04-25T00:37:22.261 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.259+0000 7fd9fa7fc640 1 -- 172.21.15.32:0/3445725822 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fda00004240 con 0x7fda0410ff90 2024-04-25T00:37:22.261 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.259+0000 7fd9fa7fc640 1 -- 172.21.15.32:0/3445725822 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fda00037040 con 0x7fda0410ff90 2024-04-25T00:37:22.261 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.259+0000 7fda0b74f640 1 -- 172.21.15.32:0/3445725822 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fda04116630 con 0x7fda0410ff90 2024-04-25T00:37:22.273 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.264+0000 7fda0e45c740 1 Processor -- start 2024-04-25T00:37:22.273 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.266+0000 7fd9fa7fc640 1 -- 172.21.15.32:0/3445725822 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7fda00042020 con 0x7fda0410ff90 2024-04-25T00:37:22.273 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.266+0000 7fda0e45c740 1 -- start start 2024-04-25T00:37:22.273 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.266+0000 7fda0e45c740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5585dfcbb630 0x5585dfcbba10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:22.273 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.266+0000 7fda0e45c740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5585dfcbbf50 0x5585dfcc60e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:22.273 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.266+0000 7fda0e45c740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcc6620 0x5585dfcc8a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:22.274 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.266+0000 7fda0e45c740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5585dfc206b0 con 0x5585dfcc6620 2024-04-25T00:37:22.274 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.266+0000 7fda0e45c740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5585dfbc8880 con 0x5585dfcbb630 2024-04-25T00:37:22.274 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.266+0000 7fda0e45c740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5585dfc1c670 con 0x5585dfcbbf50 2024-04-25T00:37:22.274 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.267+0000 7fd9fa7fc640 1 --2- 172.21.15.32:0/3445725822 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd9f00a2e80 0x7fd9f00a5340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:22.274 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.267+0000 7fd9fa7fc640 1 -- 172.21.15.32:0/3445725822 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fda000ef2f0 con 0x7fda0410ff90 2024-04-25T00:37:22.274 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.269+0000 7fda08cea640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcc6620 0x5585dfcc8a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:22.274 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.269+0000 7fda08cea640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcc6620 0x5585dfcc8a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46596/0 (socket says 172.21.15.32:46596) 2024-04-25T00:37:22.274 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.269+0000 7fda08cea640 1 -- 172.21.15.32:0/2046442702 learned_addr learned my addr 172.21.15.32:0/2046442702 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:22.274 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.269+0000 7fda09cec640 1 --2- 172.21.15.32:0/2046442702 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5585dfcbb630 0x5585dfcbba10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:22.274 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.272+0000 7fda094eb640 1 --2- 172.21.15.32:0/2046442702 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5585dfcbbf50 0x5585dfcc60e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:22.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.272+0000 7fda09cec640 1 --2- 172.21.15.32:0/3445725822 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd9f00a2e80 0x7fd9f00a5340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:22.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.272+0000 7fda09cec640 1 -- 172.21.15.32:0/2046442702 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5585dfcbbf50 msgr2=0x5585dfcc60e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:22.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.272+0000 7fda09cec640 1 --2- 172.21.15.32:0/2046442702 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5585dfcbbf50 0x5585dfcc60e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.275 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.272+0000 7fda09cec640 1 -- 172.21.15.32:0/2046442702 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcc6620 msgr2=0x5585dfcc8a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:22.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.272+0000 7fda09cec640 1 --2- 172.21.15.32:0/2046442702 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcc6620 0x5585dfcc8a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.272+0000 7fda09cec640 1 -- 172.21.15.32:0/2046442702 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5585df9fe7d0 con 0x5585dfcbb630 2024-04-25T00:37:22.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.274+0000 7fda09cec640 1 --2- 172.21.15.32:0/2046442702 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5585dfcbb630 0x5585dfcbba10 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7fd9f4014440 tx=0x7fd9f4014900 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7e6852b16c7a85b7 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:22.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.274+0000 7fd9ef7fe640 1 -- 172.21.15.32:0/2046442702 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd9f4004230 con 0x5585dfcbb630 2024-04-25T00:37:22.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.274+0000 7fd9ef7fe640 1 -- 172.21.15.32:0/2046442702 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd9f40043d0 con 0x5585dfcbb630 2024-04-25T00:37:22.277 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.275+0000 7fda0e45c740 1 -- 172.21.15.32:0/2046442702 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5585dfcbb630 msgr2=0x5585dfcbba10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:22.277 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.275+0000 7fda0e45c740 1 --2- 172.21.15.32:0/2046442702 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5585dfcbb630 0x5585dfcbba10 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7fd9f4014440 tx=0x7fd9f4014900 comp rx=0 tx=0).stop 2024-04-25T00:37:22.277 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.276+0000 7fda09cec640 1 --2- 172.21.15.32:0/3445725822 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd9f00a2e80 0x7fd9f00a5340 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7fd9f4014e00 tx=0x7fd9f401b000 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:22.282 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.282+0000 7fda0e45c740 1 -- 172.21.15.32:0/2046442702 shutdown_connections 2024-04-25T00:37:22.282 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.282+0000 7fda0e45c740 1 --2- 172.21.15.32:0/2046442702 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcc6620 0x5585dfcc8a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.282 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.282+0000 7fda0e45c740 1 --2- 172.21.15.32:0/2046442702 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5585dfcbbf50 0x5585dfcc60e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.282 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.282+0000 7fda0e45c740 1 --2- 172.21.15.32:0/2046442702 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5585dfcbb630 0x5585dfcbba10 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.282 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.282+0000 7fda0e45c740 1 -- 172.21.15.32:0/2046442702 >> 172.21.15.32:0/2046442702 conn(0x5585dfcb7950 msgr2=0x5585dfcb7d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:22.287 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.286+0000 7fda0e45c740 1 -- 172.21.15.32:0/2046442702 shutdown_connections 2024-04-25T00:37:22.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.286+0000 7fda0e45c740 1 -- 172.21.15.32:0/2046442702 wait complete. 2024-04-25T00:37:22.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.288+0000 7fda0e45c740 1 -- start start 2024-04-25T00:37:22.290 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.288+0000 7fda0e45c740 1 Processor -- start 2024-04-25T00:37:22.292 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.290+0000 7fda0e45c740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcbb630 0x5585dfcc34a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:22.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.291+0000 7fda0e45c740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5585dfcbbf50 0x5585dfcc39e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:22.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.291+0000 7fda0e45c740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5585dfcc6620 0x5585dfcc3f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:22.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.291+0000 7fda0e45c740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5585dfcc93a0 con 0x5585dfcbb630 2024-04-25T00:37:22.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.291+0000 7fda0e45c740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5585dfbdbf50 con 0x5585dfcbbf50 2024-04-25T00:37:22.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.291+0000 7fda0e45c740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5585dfcc9520 con 0x5585dfcc6620 2024-04-25T00:37:22.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.291+0000 7fda0e45c740 10 client.0 _ll_register_callbacks cb 0x5585dfc17000 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:37:22.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.292+0000 7fda094eb640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcbb630 0x5585dfcc34a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:22.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.292+0000 7fda094eb640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcbb630 0x5585dfcc34a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46608/0 (socket says 172.21.15.32:46608) 2024-04-25T00:37:22.293 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.292+0000 7fda094eb640 1 -- 172.21.15.32:0/2000527912 learned_addr learned my addr 172.21.15.32:0/2000527912 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:22.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.293+0000 7fda09cec640 1 --2- 172.21.15.32:0/2000527912 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5585dfcc6620 0x5585dfcc3f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:22.294 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.293+0000 7fda08cea640 1 --2- 172.21.15.32:0/2000527912 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5585dfcbbf50 0x5585dfcc39e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:22.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.295+0000 7fda094eb640 1 -- 172.21.15.32:0/2000527912 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5585dfcc6620 msgr2=0x5585dfcc3f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:22.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.295+0000 7fda094eb640 1 --2- 172.21.15.32:0/2000527912 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5585dfcc6620 0x5585dfcc3f20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.295+0000 7fda094eb640 1 -- 172.21.15.32:0/2000527912 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5585dfcbbf50 msgr2=0x5585dfcc39e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:22.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.295+0000 7fda094eb640 1 --2- 172.21.15.32:0/2000527912 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5585dfcbbf50 0x5585dfcc39e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.296 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.295+0000 7fda094eb640 1 -- 172.21.15.32:0/2000527912 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5585df9343f0 con 0x5585dfcbb630 2024-04-25T00:37:22.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.301+0000 7fda094eb640 1 --2- 172.21.15.32:0/2000527912 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcbb630 0x5585dfcc34a0 secure :-1 s=READY pgs=619 cs=0 l=1 rev1=1 crypto rx=0x7fd9fc00eb90 tx=0x7fd9fc00ef90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e28261eb31cf4968 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:22.302 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.301+0000 7fd9ecff9640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:37:22.304 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.303+0000 7fd9ecff9640 1 -- 172.21.15.32:0/2000527912 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd9fc00ddf0 con 0x5585dfcbb630 2024-04-25T00:37:22.304 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.303+0000 7fda0e45c740 1 -- 172.21.15.32:0/2000527912 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd9f40043d0 con 0x5585dfcbb630 2024-04-25T00:37:22.304 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.303+0000 7fd9ecff9640 1 -- 172.21.15.32:0/2000527912 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fd9fc003970 con 0x5585dfcbb630 2024-04-25T00:37:22.308 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.308+0000 7fd9ecff9640 1 -- 172.21.15.32:0/2000527912 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fd9fc024050 con 0x5585dfcbb630 2024-04-25T00:37:22.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.314+0000 7fda0e45c740 10 client.16197 fetch_fsmap learned FSMap version 27 2024-04-25T00:37:22.315 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.314+0000 7fda0e45c740 1 -- 172.21.15.32:0/2000527912 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x5585df8f40f0 con 0x5585dfcbb630 2024-04-25T00:37:22.326 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.326+0000 7fd9ecff9640 1 -- 172.21.15.32:0/2000527912 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd9f40043d0 con 0x5585dfcbb630 2024-04-25T00:37:22.326 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.326+0000 7fd9ecff9640 1 -- 172.21.15.32:0/2000527912 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 27) ==== 8354+0+0 (secure 0 0 0) 0x7fd9fc00b040 con 0x5585dfcbb630 2024-04-25T00:37:22.327 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.326+0000 7fda0e45c740 10 client.16197 fetch_fsmap finished waiting for FSMap version 27 2024-04-25T00:37:22.327 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.326+0000 7fda0e45c740 10 client.16197 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:37:22.327 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.326+0000 7fda0e45c740 20 client.16197 populate_metadata read hostname 'smithi032' 2024-04-25T00:37:22.327 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.326+0000 7fda0e45c740 1 --2- 172.21.15.32:0/2000527912 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5585dfcd6cf0 0x5585dfcd9150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:22.327 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.326+0000 7fda0e45c740 4 client.16197 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:37:22.328 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.326+0000 7fda0e45c740 1 -- 172.21.15.32:0/2000527912 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5585dfc2dbf0 con 0x5585dfcd6cf0 2024-04-25T00:37:22.328 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.327+0000 7fda08cea640 1 --2- 172.21.15.32:0/2000527912 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5585dfcd6cf0 0x5585dfcd9150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:22.338 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.330+0000 7fda08cea640 1 --2- 172.21.15.32:0/2000527912 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5585dfcd6cf0 0x5585dfcd9150 crc :-1 s=READY pgs=87 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ee1d8c10f33af64c server_cookie=833d6c195e145c63 in_seq=0 out_seq=0 2024-04-25T00:37:22.338 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.330+0000 7fd9ecff9640 10 client.16197 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:37:22.338 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.332+0000 7fd9ecff9640 1 -- 172.21.15.32:0/2000527912 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x5585dfc2dbf0 con 0x5585dfcd6cf0 2024-04-25T00:37:22.338 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.332+0000 7fd9ecff9640 10 client.16197 handle_command_reply: tid=0 2024-04-25T00:37:22.338 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.333+0000 7fda0e45c740 1 client.16197 shutdown 2024-04-25T00:37:22.343 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.342+0000 7fda0e45c740 1 -- 172.21.15.32:0/2000527912 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcbb630 msgr2=0x5585dfcc34a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:22.343 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.342+0000 7fda0e45c740 1 --2- 172.21.15.32:0/2000527912 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcbb630 0x5585dfcc34a0 secure :-1 s=READY pgs=619 cs=0 l=1 rev1=1 crypto rx=0x7fd9fc00eb90 tx=0x7fd9fc00ef90 comp rx=0 tx=0).stop 2024-04-25T00:37:22.343 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.342+0000 7fda0e45c740 1 -- 172.21.15.32:0/2000527912 shutdown_connections 2024-04-25T00:37:22.343 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.342+0000 7fda0e45c740 1 --2- 172.21.15.32:0/2000527912 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5585dfcd6cf0 0x5585dfcd9150 crc :-1 s=READY pgs=87 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.343 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.342+0000 7fda0e45c740 1 --2- 172.21.15.32:0/2000527912 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5585dfcc6620 0x5585dfcc3f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.343 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.342+0000 7fda0e45c740 1 --2- 172.21.15.32:0/2000527912 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5585dfcbbf50 0x5585dfcc39e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.343 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.342+0000 7fda0e45c740 1 --2- 172.21.15.32:0/2000527912 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcbb630 0x5585dfcc34a0 secure :-1 s=CLOSED pgs=619 cs=0 l=1 rev1=1 crypto rx=0x7fd9fc00eb90 tx=0x7fd9fc00ef90 comp rx=0 tx=0).stop 2024-04-25T00:37:22.343 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.342+0000 7fda0e45c740 1 -- 172.21.15.32:0/2000527912 >> 172.21.15.32:0/2000527912 conn(0x5585dfcb7950 msgr2=0x5585dfcbd750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:22.343 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.342+0000 7fd9ecff9640 0 client.16197 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:37:22.348 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.346+0000 7fda0e45c740 1 -- 172.21.15.32:0/2000527912 shutdown_connections 2024-04-25T00:37:22.348 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.346+0000 7fda0e45c740 1 -- 172.21.15.32:0/2000527912 wait complete. 2024-04-25T00:37:22.348 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.347+0000 7fda0e45c740 20 client.16197 trim_cache size 0 max 16384 2024-04-25T00:37:22.424 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.422+0000 7fda0e45c740 1 Processor -- start 2024-04-25T00:37:22.425 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.424+0000 7fda0e45c740 1 -- start start 2024-04-25T00:37:22.425 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.424+0000 7fda0e45c740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5585dfcbbf50 0x5585dfcd22b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:22.425 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.424+0000 7fda0e45c740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5585dfcc6620 0x5585dfcd8d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:22.425 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.424+0000 7fda0e45c740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcc2600 0x5585dfcc3d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:22.426 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.424+0000 7fda0e45c740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fda041c8d30 con 0x5585dfcc2600 2024-04-25T00:37:22.426 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.424+0000 7fda0e45c740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fda041c6600 con 0x5585dfcc6620 2024-04-25T00:37:22.426 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.424+0000 7fda0e45c740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fda041c4450 con 0x5585dfcbbf50 2024-04-25T00:37:22.426 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.425+0000 7fda09cec640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcc2600 0x5585dfcc3d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:22.426 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.425+0000 7fda09cec640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcc2600 0x5585dfcc3d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46610/0 (socket says 172.21.15.32:46610) 2024-04-25T00:37:22.426 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.425+0000 7fda09cec640 1 -- 172.21.15.32:0/3100694664 learned_addr learned my addr 172.21.15.32:0/3100694664 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:22.426 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.425+0000 7fda094eb640 1 --2- 172.21.15.32:0/3100694664 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5585dfcbbf50 0x5585dfcd22b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:22.427 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.425+0000 7fda09cec640 1 -- 172.21.15.32:0/3100694664 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5585dfcbbf50 msgr2=0x5585dfcd22b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:22.427 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.425+0000 7fda09cec640 1 --2- 172.21.15.32:0/3100694664 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5585dfcbbf50 0x5585dfcd22b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.427 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.425+0000 7fda09cec640 1 -- 172.21.15.32:0/3100694664 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5585dfcc6620 msgr2=0x5585dfcd8d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:22.428 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.425+0000 7fda09cec640 1 --2- 172.21.15.32:0/3100694664 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5585dfcc6620 0x5585dfcd8d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.428 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.425+0000 7fda09cec640 1 -- 172.21.15.32:0/3100694664 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5585dfcb7780 con 0x5585dfcc2600 2024-04-25T00:37:22.428 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.426+0000 7fda09cec640 1 --2- 172.21.15.32:0/3100694664 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcc2600 0x5585dfcc3d70 secure :-1 s=READY pgs=620 cs=0 l=1 rev1=1 crypto rx=0x7fd9f400bc20 tx=0x7fd9f400bc50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d3d37ba3f36a132d server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:22.428 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.427+0000 7fd9eeffd640 1 -- 172.21.15.32:0/3100694664 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd9f401d990 con 0x5585dfcc2600 2024-04-25T00:37:22.428 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.427+0000 7fd9eeffd640 1 -- 172.21.15.32:0/3100694664 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd9f4005510 con 0x5585dfcc2600 2024-04-25T00:37:22.428 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.427+0000 7fd9eeffd640 1 -- 172.21.15.32:0/3100694664 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd9f4003ab0 con 0x5585dfcc2600 2024-04-25T00:37:22.429 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.429+0000 7fda0e45c740 1 -- 172.21.15.32:0/3100694664 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcc2600 msgr2=0x5585dfcc3d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:22.429 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.429+0000 7fda0e45c740 1 --2- 172.21.15.32:0/3100694664 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcc2600 0x5585dfcc3d70 secure :-1 s=READY pgs=620 cs=0 l=1 rev1=1 crypto rx=0x7fd9f400bc20 tx=0x7fd9f400bc50 comp rx=0 tx=0).stop 2024-04-25T00:37:22.435 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.434+0000 7fda0e45c740 1 -- 172.21.15.32:0/3100694664 shutdown_connections 2024-04-25T00:37:22.435 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.434+0000 7fda0e45c740 1 --2- 172.21.15.32:0/3100694664 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcc2600 0x5585dfcc3d70 unknown :-1 s=CLOSED pgs=620 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.435 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.434+0000 7fda0e45c740 1 --2- 172.21.15.32:0/3100694664 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5585dfcc6620 0x5585dfcd8d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.435 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.434+0000 7fda0e45c740 1 --2- 172.21.15.32:0/3100694664 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5585dfcbbf50 0x5585dfcd22b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.435 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.434+0000 7fda0e45c740 1 -- 172.21.15.32:0/3100694664 >> 172.21.15.32:0/3100694664 conn(0x5585dfcb7950 msgr2=0x5585dfcc4bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:22.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.451+0000 7fda0e45c740 1 -- 172.21.15.32:0/3100694664 shutdown_connections 2024-04-25T00:37:22.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.451+0000 7fda0e45c740 1 -- 172.21.15.32:0/3100694664 wait complete. 2024-04-25T00:37:22.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.452+0000 7fda0e45c740 1 -- start start 2024-04-25T00:37:22.452 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.452+0000 7fda0e45c740 1 Processor -- start 2024-04-25T00:37:22.454 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.453+0000 7fda0e45c740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5585dfcbbf50 0x5585dfcced00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:22.454 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.453+0000 7fda0e45c740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5585dfcc2600 0x5585dfcc7360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:22.454 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.453+0000 7fda0e45c740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcc6620 0x5585dfcc7a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:22.454 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.454+0000 7fda0e45c740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fda041c4ac0 con 0x5585dfcc6620 2024-04-25T00:37:22.455 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.454+0000 7fda0e45c740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fda041c5f30 con 0x5585dfcbbf50 2024-04-25T00:37:22.455 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.454+0000 7fda0e45c740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fda041c5190 con 0x5585dfcc2600 2024-04-25T00:37:22.455 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.454+0000 7fda0e45c740 10 client.0 _ll_register_callbacks cb 0x7fd9fc009ba0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:37:22.456 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.455+0000 7fda08cea640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5585dfcc2600 0x5585dfcc7360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:22.456 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.455+0000 7fda08cea640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5585dfcc2600 0x5585dfcc7360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:60648/0 (socket says 172.21.15.32:60648) 2024-04-25T00:37:22.457 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.455+0000 7fda08cea640 1 -- 172.21.15.32:0/1139143744 learned_addr learned my addr 172.21.15.32:0/1139143744 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:22.457 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.455+0000 7fda094eb640 1 --2- 172.21.15.32:0/1139143744 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5585dfcbbf50 0x5585dfcced00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:22.457 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.456+0000 7fda09cec640 1 --2- 172.21.15.32:0/1139143744 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcc6620 0x5585dfcc7a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:22.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.460+0000 7fda08cea640 1 -- 172.21.15.32:0/1139143744 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5585dfcbbf50 msgr2=0x5585dfcced00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:22.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.460+0000 7fda08cea640 1 --2- 172.21.15.32:0/1139143744 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5585dfcbbf50 0x5585dfcced00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.461 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.460+0000 7fda08cea640 1 -- 172.21.15.32:0/1139143744 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcc6620 msgr2=0x5585dfcc7a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:22.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.460+0000 7fda08cea640 1 --2- 172.21.15.32:0/1139143744 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcc6620 0x5585dfcc7a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.462 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.460+0000 7fda08cea640 1 -- 172.21.15.32:0/1139143744 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5585dfcc9520 con 0x5585dfcc2600 2024-04-25T00:37:22.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.462+0000 7fda08cea640 1 --2- 172.21.15.32:0/1139143744 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5585dfcc2600 0x5585dfcc7360 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7fda000ecc90 tx=0x7fda00030520 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8c043662210e397e server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:22.467 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.464+0000 7fd9ecff9640 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:37:22.469 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.467+0000 7fd9ecff9640 1 -- 172.21.15.32:0/1139143744 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fda000ef120 con 0x5585dfcc2600 2024-04-25T00:37:22.469 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.467+0000 7fd9ecff9640 1 -- 172.21.15.32:0/1139143744 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fda000ec910 con 0x5585dfcc2600 2024-04-25T00:37:22.469 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.469+0000 7fda0e45c740 1 -- 172.21.15.32:0/1139143744 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x5585dfc27f40 con 0x5585dfcc2600 2024-04-25T00:37:22.472 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.472+0000 7fd9ecff9640 1 -- 172.21.15.32:0/1139143744 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fda000bbc70 con 0x5585dfcc2600 2024-04-25T00:37:22.475 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.473+0000 7fda0e45c740 10 client.25732 fetch_fsmap learned FSMap version 27 2024-04-25T00:37:22.475 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.473+0000 7fda0e45c740 1 -- 172.21.15.32:0/1139143744 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x5585dfd3d2f0 con 0x5585dfcc2600 2024-04-25T00:37:22.477 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.476+0000 7fd9ecff9640 1 -- 172.21.15.32:0/1139143744 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x5585dfc27f40 con 0x5585dfcc2600 2024-04-25T00:37:22.478 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.478+0000 7fd9ecff9640 1 -- 172.21.15.32:0/1139143744 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 27) ==== 8354+0+0 (secure 0 0 0) 0x7fda0003a030 con 0x5585dfcc2600 2024-04-25T00:37:22.479 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.479+0000 7fda0e45c740 10 client.25732 fetch_fsmap finished waiting for FSMap version 27 2024-04-25T00:37:22.480 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.479+0000 7fda0e45c740 10 client.25732 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:37:22.480 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.479+0000 7fda0e45c740 20 client.25732 populate_metadata read hostname 'smithi032' 2024-04-25T00:37:22.480 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.479+0000 7fda0e45c740 1 --2- 172.21.15.32:0/1139143744 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5585dfd3daa0 0x5585dfd3fec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:22.480 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.479+0000 7fda0e45c740 4 client.25732 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-25T00:37:22.480 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.479+0000 7fda0e45c740 1 -- 172.21.15.32:0/1139143744 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "scrub status"}) -- 0x5585dfba56f0 con 0x5585dfd3daa0 2024-04-25T00:37:22.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.481+0000 7fda094eb640 1 --2- 172.21.15.32:0/1139143744 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5585dfd3daa0 0x5585dfd3fec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:22.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.484+0000 7fda094eb640 1 --2- 172.21.15.32:0/1139143744 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5585dfd3daa0 0x5585dfd3fec0 crc :-1 s=READY pgs=88 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3864ec97f23ee915 server_cookie=df3dae41180358ea in_seq=0 out_seq=0 2024-04-25T00:37:22.485 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.484+0000 7fd9ecff9640 10 client.25732 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:37:22.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.487+0000 7fd9ecff9640 1 -- 172.21.15.32:0/1139143744 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+63 (crc 0 0 0) 0x5585dfba56f0 con 0x5585dfd3daa0 2024-04-25T00:37:22.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.487+0000 7fd9ecff9640 10 client.25732 handle_command_reply: tid=0 2024-04-25T00:37:22.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.487+0000 7fda0e45c740 1 client.25732 shutdown 2024-04-25T00:37:22.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.492+0000 7fda0e45c740 1 -- 172.21.15.32:0/1139143744 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5585dfcc2600 msgr2=0x5585dfcc7360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:22.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.492+0000 7fda0e45c740 1 --2- 172.21.15.32:0/1139143744 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5585dfcc2600 0x5585dfcc7360 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7fda000ecc90 tx=0x7fda00030520 comp rx=0 tx=0).stop 2024-04-25T00:37:22.498 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.494+0000 7fda0e45c740 1 -- 172.21.15.32:0/1139143744 shutdown_connections 2024-04-25T00:37:22.498 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.494+0000 7fda0e45c740 1 --2- 172.21.15.32:0/1139143744 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5585dfd3daa0 0x5585dfd3fec0 crc :-1 s=READY pgs=88 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.498 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.494+0000 7fda0e45c740 1 --2- 172.21.15.32:0/1139143744 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5585dfcc6620 0x5585dfcc7a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.498 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.494+0000 7fda0e45c740 1 --2- 172.21.15.32:0/1139143744 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5585dfcc2600 0x5585dfcc7360 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.498 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.494+0000 7fda0e45c740 1 --2- 172.21.15.32:0/1139143744 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5585dfcbbf50 0x5585dfcced00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.498 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.494+0000 7fda0e45c740 1 -- 172.21.15.32:0/1139143744 >> 172.21.15.32:0/1139143744 conn(0x5585dfcb7950 msgr2=0x5585dfcbd610 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:22.498 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.494+0000 7fd9ecff9640 0 client.25732 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:37:22.498 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.498+0000 7fda0e45c740 1 -- 172.21.15.32:0/1139143744 shutdown_connections 2024-04-25T00:37:22.499 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.498+0000 7fda0e45c740 1 -- 172.21.15.32:0/1139143744 wait complete. 2024-04-25T00:37:22.503 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.500+0000 7fda0e45c740 20 client.25732 trim_cache size 0 max 16384 2024-04-25T00:37:22.505 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:37:22.505 INFO:teuthology.orchestra.run.smithi032.stdout: "status": "no active scrubs running", 2024-04-25T00:37:22.505 INFO:teuthology.orchestra.run.smithi032.stdout: "scrubs": {} 2024-04-25T00:37:22.505 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:37:22.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.512+0000 7fda0b74f640 1 -- 172.21.15.32:0/3445725822 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd9f00a2e80 msgr2=0x7fd9f00a5340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:22.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.512+0000 7fda0b74f640 1 --2- 172.21.15.32:0/3445725822 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd9f00a2e80 0x7fd9f00a5340 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7fd9f4014e00 tx=0x7fd9f401b000 comp rx=0 tx=0).stop 2024-04-25T00:37:22.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.512+0000 7fda0b74f640 1 -- 172.21.15.32:0/3445725822 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fda0410ff90 msgr2=0x7fda041116a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:22.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.512+0000 7fda0b74f640 1 --2- 172.21.15.32:0/3445725822 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fda0410ff90 0x7fda041116a0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7fda00005d00 tx=0x7fda00033d10 comp rx=0 tx=0).stop 2024-04-25T00:37:22.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.512+0000 7fda0b74f640 1 -- 172.21.15.32:0/3445725822 shutdown_connections 2024-04-25T00:37:22.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.512+0000 7fda0b74f640 1 --2- 172.21.15.32:0/3445725822 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd9f00a2e80 0x7fd9f00a5340 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.512+0000 7fda0b74f640 1 --2- 172.21.15.32:0/3445725822 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fda0411af20 0x7fda04111be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.514 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.512+0000 7fda0b74f640 1 --2- 172.21.15.32:0/3445725822 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fda0410ff90 0x7fda041116a0 secure :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7fda00005d00 tx=0x7fda00033d10 comp rx=0 tx=0).stop 2024-04-25T00:37:22.514 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.512+0000 7fda0b74f640 1 --2- 172.21.15.32:0/3445725822 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fda0410dfc0 0x7fda04118620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.514 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.512+0000 7fda0b74f640 1 -- 172.21.15.32:0/3445725822 >> 172.21.15.32:0/3445725822 conn(0x7fda0406c850 msgr2=0x7fda041199b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:22.516 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.515+0000 7fda0b74f640 1 -- 172.21.15.32:0/3445725822 shutdown_connections 2024-04-25T00:37:22.519 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.518+0000 7fda0b74f640 1 -- 172.21.15.32:0/3445725822 wait complete. 2024-04-25T00:37:22.547 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-25T00:37:22.547 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:37:22.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.880+0000 7fcd91729640 1 -- 172.21.15.32:0/1249043367 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcd8c110310 msgr2=0x7fcd8c10dfc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:22.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.880+0000 7fcd91729640 1 --2- 172.21.15.32:0/1249043367 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcd8c110310 0x7fcd8c10dfc0 secure :-1 s=READY pgs=621 cs=0 l=1 rev1=1 crypto rx=0x7fcd8401acb0 tx=0x7fcd8403b900 comp rx=0 tx=0).stop 2024-04-25T00:37:22.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.882+0000 7fcd91729640 1 -- 172.21.15.32:0/1249043367 shutdown_connections 2024-04-25T00:37:22.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.882+0000 7fcd91729640 1 --2- 172.21.15.32:0/1249043367 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcd8c10e6a0 0x7fcd8c11cf90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.882+0000 7fcd91729640 1 --2- 172.21.15.32:0/1249043367 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcd8c110310 0x7fcd8c10dfc0 unknown :-1 s=CLOSED pgs=621 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.882+0000 7fcd91729640 1 --2- 172.21.15.32:0/1249043367 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fcd8c10fea0 0x7fcd8c10da80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.882+0000 7fcd91729640 1 -- 172.21.15.32:0/1249043367 >> 172.21.15.32:0/1249043367 conn(0x7fcd8c06c860 msgr2=0x7fcd8c06cc70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:22.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.890+0000 7fcd91729640 1 -- 172.21.15.32:0/1249043367 shutdown_connections 2024-04-25T00:37:22.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.892+0000 7fcd91729640 1 -- 172.21.15.32:0/1249043367 wait complete. 2024-04-25T00:37:22.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.893+0000 7fcd91729640 1 Processor -- start 2024-04-25T00:37:22.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.894+0000 7fcd91729640 1 -- start start 2024-04-25T00:37:22.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.914+0000 7fcd91729640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcd8c10fea0 0x7fcd8c11a6e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:22.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.914+0000 7fcd91729640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fcd8c1167c0 0x7fcd8c116c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:22.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.914+0000 7fcd91729640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcd8c117160 0x7fcd8c1aacc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:22.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.914+0000 7fcd91729640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fcd8c113140 con 0x7fcd8c10fea0 2024-04-25T00:37:22.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.914+0000 7fcd91729640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fcd8c112fc0 con 0x7fcd8c1167c0 2024-04-25T00:37:22.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.914+0000 7fcd91729640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fcd8c1132c0 con 0x7fcd8c117160 2024-04-25T00:37:22.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.916+0000 7fcd8a7fc640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fcd8c1167c0 0x7fcd8c116c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:22.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.916+0000 7fcd8a7fc640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fcd8c1167c0 0x7fcd8c116c20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:34050/0 (socket says 172.21.15.32:34050) 2024-04-25T00:37:22.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.916+0000 7fcd8a7fc640 1 -- 172.21.15.32:0/432827295 learned_addr learned my addr 172.21.15.32:0/432827295 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:22.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.916+0000 7fcd8affd640 1 --2- 172.21.15.32:0/432827295 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcd8c10fea0 0x7fcd8c11a6e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:22.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.917+0000 7fcd8affd640 1 -- 172.21.15.32:0/432827295 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcd8c117160 msgr2=0x7fcd8c1aacc0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:37:22.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.917+0000 7fcd8affd640 1 --2- 172.21.15.32:0/432827295 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcd8c117160 0x7fcd8c1aacc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.917+0000 7fcd8affd640 1 -- 172.21.15.32:0/432827295 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fcd8c1167c0 msgr2=0x7fcd8c116c20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:22.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.917+0000 7fcd8affd640 1 --2- 172.21.15.32:0/432827295 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fcd8c1167c0 0x7fcd8c116c20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:22.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.917+0000 7fcd8affd640 1 -- 172.21.15.32:0/432827295 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fcd8c1ab200 con 0x7fcd8c10fea0 2024-04-25T00:37:22.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.917+0000 7fcd8affd640 1 --2- 172.21.15.32:0/432827295 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcd8c10fea0 0x7fcd8c11a6e0 secure :-1 s=READY pgs=622 cs=0 l=1 rev1=1 crypto rx=0x7fcd80009fd0 tx=0x7fcd8000b870 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:22.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.921+0000 7fcd73fff640 1 -- 172.21.15.32:0/432827295 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcd80015020 con 0x7fcd8c10fea0 2024-04-25T00:37:22.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.921+0000 7fcd73fff640 1 -- 172.21.15.32:0/432827295 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fcd80011d10 con 0x7fcd8c10fea0 2024-04-25T00:37:22.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.921+0000 7fcd73fff640 1 -- 172.21.15.32:0/432827295 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcd8000f2e0 con 0x7fcd8c10fea0 2024-04-25T00:37:22.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.921+0000 7fcd91729640 1 -- 172.21.15.32:0/432827295 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fcd8c1ab3d0 con 0x7fcd8c10fea0 2024-04-25T00:37:22.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.922+0000 7fcd91729640 1 -- 172.21.15.32:0/432827295 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fcd8c1ab8d0 con 0x7fcd8c10fea0 2024-04-25T00:37:22.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.933+0000 7fcd91729640 1 -- 172.21.15.32:0/432827295 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fcd8c06daf0 con 0x7fcd8c10fea0 2024-04-25T00:37:22.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.934+0000 7fcd73fff640 1 -- 172.21.15.32:0/432827295 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7fcd80018020 con 0x7fcd8c10fea0 2024-04-25T00:37:22.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.935+0000 7fcd73fff640 1 --2- 172.21.15.32:0/432827295 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fcd740a2f10 0x7fcd740a53d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:22.936 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.935+0000 7fcd73fff640 1 -- 172.21.15.32:0/432827295 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fcd800ced60 con 0x7fcd8c10fea0 2024-04-25T00:37:22.945 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.940+0000 7fcd8a7fc640 1 --2- 172.21.15.32:0/432827295 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fcd740a2f10 0x7fcd740a53d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:22.950 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.945+0000 7fcd73fff640 1 -- 172.21.15.32:0/432827295 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fcd8009b680 con 0x7fcd8c10fea0 2024-04-25T00:37:22.950 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:22.950+0000 7fcd8a7fc640 1 --2- 172.21.15.32:0/432827295 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fcd740a2f10 0x7fcd740a53d0 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7fcd8401aa70 tx=0x7fcd8403be00 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:23.491 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.490+0000 7fcd91729640 1 -- 172.21.15.32:0/432827295 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fcd8c10e6a0 con 0x7fcd8c10fea0 2024-04-25T00:37:23.493 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.493+0000 7fcd73fff640 1 -- 172.21.15.32:0/432827295 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 27 v27) ==== 94+0+11879 (secure 0 0 0) 0x7fcd8009af80 con 0x7fcd8c10fea0 2024-04-25T00:37:23.495 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:37:23.495 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":27,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14643,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6839/187492674","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6838","nonce":187492674},{"type":"v1","addr":"172.21.15.32:6839","nonce":187492674}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14655,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6841/1948785545","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6840","nonce":1948785545},{"type":"v1","addr":"172.21.15.32:6841","nonce":1948785545}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":27,"flags":306,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":true},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-25T00:30:47.866875+0000","modified":"2024-04-25T00:36:12.357299+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24506,"mds_1":24481,"mds_2":24476},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24443":{"gid":24443,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24445":{"gid":24445,"name":"f","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24457":{"gid":24457,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24476":{"gid":24476,"name":"h","rank":2,"incarnation":22,"state":"up:active","state_seq":12,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24481":{"gid":24481,"name":"i","rank":1,"incarnation":20,"state":"up:active","state_seq":11,"addr":"172.21.15.130:6839/4267125809","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":4267125809},{"type":"v1","addr":"172.21.15.130:6839","nonce":4267125809}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24506":{"gid":24506,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.121:6839/258444853","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6838","nonce":258444853},{"type":"v1","addr":"172.21.15.121:6839","nonce":258444853}]},"join_fscid":-1,"export_targets":[1,2],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24506,"qdb_cluster":[24476,24481,24506]},"id":1}]} 2024-04-25T00:37:23.495 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 27 2024-04-25T00:37:23.503 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.500+0000 7fcd91729640 1 -- 172.21.15.32:0/432827295 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fcd740a2f10 msgr2=0x7fcd740a53d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:23.503 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.500+0000 7fcd91729640 1 --2- 172.21.15.32:0/432827295 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fcd740a2f10 0x7fcd740a53d0 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7fcd8401aa70 tx=0x7fcd8403be00 comp rx=0 tx=0).stop 2024-04-25T00:37:23.503 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.500+0000 7fcd91729640 1 -- 172.21.15.32:0/432827295 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcd8c10fea0 msgr2=0x7fcd8c11a6e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:23.503 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.500+0000 7fcd91729640 1 --2- 172.21.15.32:0/432827295 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcd8c10fea0 0x7fcd8c11a6e0 secure :-1 s=READY pgs=622 cs=0 l=1 rev1=1 crypto rx=0x7fcd80009fd0 tx=0x7fcd8000b870 comp rx=0 tx=0).stop 2024-04-25T00:37:23.503 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.500+0000 7fcd91729640 1 -- 172.21.15.32:0/432827295 shutdown_connections 2024-04-25T00:37:23.504 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.500+0000 7fcd91729640 1 --2- 172.21.15.32:0/432827295 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fcd740a2f10 0x7fcd740a53d0 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:23.504 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.500+0000 7fcd91729640 1 --2- 172.21.15.32:0/432827295 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcd8c117160 0x7fcd8c1aacc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:23.504 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.500+0000 7fcd91729640 1 --2- 172.21.15.32:0/432827295 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fcd8c1167c0 0x7fcd8c116c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:23.504 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.500+0000 7fcd91729640 1 --2- 172.21.15.32:0/432827295 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fcd8c10fea0 0x7fcd8c11a6e0 secure :-1 s=CLOSED pgs=622 cs=0 l=1 rev1=1 crypto rx=0x7fcd80009fd0 tx=0x7fcd8000b870 comp rx=0 tx=0).stop 2024-04-25T00:37:23.504 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.500+0000 7fcd91729640 1 -- 172.21.15.32:0/432827295 >> 172.21.15.32:0/432827295 conn(0x7fcd8c06c860 msgr2=0x7fcd8c11ba10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:23.504 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.503+0000 7fcd91729640 1 -- 172.21.15.32:0/432827295 shutdown_connections 2024-04-25T00:37:23.505 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.505+0000 7fcd91729640 1 -- 172.21.15.32:0/432827295 wait complete. 2024-04-25T00:37:23.537 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-25T00:37:23.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:23 smithi121 ceph-mon[32231]: pgmap v125: 129 pgs: 129 active+clean; 35 GiB data, 73 GiB used, 1000 GiB / 1.0 TiB avail; 618 KiB/s rd, 434 MiB/s wr, 1.89k op/s 2024-04-25T00:37:23.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:23 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/432827295' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:37:23.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:23 smithi130 ceph-mon[32525]: pgmap v125: 129 pgs: 129 active+clean; 35 GiB data, 73 GiB used, 1000 GiB / 1.0 TiB avail; 618 KiB/s rd, 434 MiB/s wr, 1.89k op/s 2024-04-25T00:37:23.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:23 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/432827295' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:37:23.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.908+0000 7fd67025d640 1 -- 172.21.15.32:0/3634183815 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd66811af20 msgr2=0x7fd66811d310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:23.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.908+0000 7fd67025d640 1 --2- 172.21.15.32:0/3634183815 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd66811af20 0x7fd66811d310 secure :-1 s=READY pgs=623 cs=0 l=1 rev1=1 crypto rx=0x7fd664009930 tx=0x7fd66402a440 comp rx=0 tx=0).stop 2024-04-25T00:37:23.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.908+0000 7fd67025d640 1 -- 172.21.15.32:0/3634183815 shutdown_connections 2024-04-25T00:37:23.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.908+0000 7fd67025d640 1 --2- 172.21.15.32:0/3634183815 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd66811af20 0x7fd66811d310 unknown :-1 s=CLOSED pgs=623 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:23.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.908+0000 7fd67025d640 1 --2- 172.21.15.32:0/3634183815 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd66810dfc0 0x7fd66810e440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:23.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.908+0000 7fd67025d640 1 --2- 172.21.15.32:0/3634183815 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd66810ff90 0x7fd66810da80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:23.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.908+0000 7fd67025d640 1 -- 172.21.15.32:0/3634183815 >> 172.21.15.32:0/3634183815 conn(0x7fd66806c850 msgr2=0x7fd66806cc60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:23.916 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.914+0000 7fd67025d640 1 -- 172.21.15.32:0/3634183815 shutdown_connections 2024-04-25T00:37:23.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.919+0000 7fd67025d640 1 -- 172.21.15.32:0/3634183815 wait complete. 2024-04-25T00:37:23.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.920+0000 7fd67025d640 1 Processor -- start 2024-04-25T00:37:23.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.920+0000 7fd67025d640 1 -- start start 2024-04-25T00:37:23.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.921+0000 7fd67025d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd66810dfc0 0x7fd6681a6eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:23.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.921+0000 7fd67025d640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd66810ff90 0x7fd6681a73f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:23.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.921+0000 7fd67025d640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd66811af20 0x7fd6681ab770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:23.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.921+0000 7fd67025d640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fd668120990 con 0x7fd66810dfc0 2024-04-25T00:37:23.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.921+0000 7fd67025d640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fd668120810 con 0x7fd66811af20 2024-04-25T00:37:23.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.921+0000 7fd67025d640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fd668120b10 con 0x7fd66810ff90 2024-04-25T00:37:23.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.922+0000 7fd66e7fa640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd66811af20 0x7fd6681ab770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:23.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.923+0000 7fd66e7fa640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd66811af20 0x7fd6681ab770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:34068/0 (socket says 172.21.15.32:34068) 2024-04-25T00:37:23.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.923+0000 7fd66e7fa640 1 -- 172.21.15.32:0/2490678968 learned_addr learned my addr 172.21.15.32:0/2490678968 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:23.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.923+0000 7fd66d7f8640 1 --2- 172.21.15.32:0/2490678968 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd66810ff90 0x7fd6681a73f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:23.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.924+0000 7fd66e7fa640 1 -- 172.21.15.32:0/2490678968 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd66810ff90 msgr2=0x7fd6681a73f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:23.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.924+0000 7fd66e7fa640 1 --2- 172.21.15.32:0/2490678968 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd66810ff90 0x7fd6681a73f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:23.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.924+0000 7fd66e7fa640 1 -- 172.21.15.32:0/2490678968 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd66810dfc0 msgr2=0x7fd6681a6eb0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-25T00:37:23.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.924+0000 7fd66e7fa640 1 --2- 172.21.15.32:0/2490678968 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd66810dfc0 0x7fd6681a6eb0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:23.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.924+0000 7fd66e7fa640 1 -- 172.21.15.32:0/2490678968 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd6681abea0 con 0x7fd66811af20 2024-04-25T00:37:23.928 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.926+0000 7fd66e7fa640 1 --2- 172.21.15.32:0/2490678968 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd66811af20 0x7fd6681ab770 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7fd664039520 tx=0x7fd664039550 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:23.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.928+0000 7fd65affd640 1 -- 172.21.15.32:0/2490678968 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd664040030 con 0x7fd66811af20 2024-04-25T00:37:23.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.928+0000 7fd67025d640 1 -- 172.21.15.32:0/2490678968 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd6681ac130 con 0x7fd66811af20 2024-04-25T00:37:23.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.928+0000 7fd67025d640 1 -- 172.21.15.32:0/2490678968 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd6680799a0 con 0x7fd66811af20 2024-04-25T00:37:23.932 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.928+0000 7fd65affd640 1 -- 172.21.15.32:0/2490678968 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd664030e40 con 0x7fd66811af20 2024-04-25T00:37:23.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.928+0000 7fd65affd640 1 -- 172.21.15.32:0/2490678968 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd664035070 con 0x7fd66811af20 2024-04-25T00:37:23.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.936+0000 7fd65affd640 1 -- 172.21.15.32:0/2490678968 <== mon.2 v2:172.21.15.121:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7fd664043030 con 0x7fd66811af20 2024-04-25T00:37:23.946 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.943+0000 7fd672f6d740 1 Processor -- start 2024-04-25T00:37:23.946 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.944+0000 7fd672f6d740 1 -- start start 2024-04-25T00:37:23.946 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.944+0000 7fd672f6d740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e01580 0x55d5c0e01960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:23.946 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.944+0000 7fd672f6d740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d5c0e01ea0 0x55d5c0e0c030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:23.946 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.944+0000 7fd672f6d740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d5c0e0c570 0x55d5c0e0e960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:23.946 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.944+0000 7fd672f6d740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55d5c0d66670 con 0x55d5c0e01580 2024-04-25T00:37:23.946 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.945+0000 7fd672f6d740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55d5c0d0e970 con 0x55d5c0e01ea0 2024-04-25T00:37:23.946 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.945+0000 7fd672f6d740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55d5c0d62630 con 0x55d5c0e0c570 2024-04-25T00:37:23.947 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.945+0000 7fd66d7f8640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e01580 0x55d5c0e01960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:23.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.945+0000 7fd66d7f8640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e01580 0x55d5c0e01960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46664/0 (socket says 172.21.15.32:46664) 2024-04-25T00:37:23.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.945+0000 7fd66d7f8640 1 -- 172.21.15.32:0/3371129026 learned_addr learned my addr 172.21.15.32:0/3371129026 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:23.957 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.945+0000 7fd66d7f8640 1 --2- 172.21.15.32:0/3371129026 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d5c0e0c570 0x55d5c0e0e960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:23.957 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.945+0000 7fd66d7f8640 1 -- 172.21.15.32:0/3371129026 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d5c0e01ea0 msgr2=0x55d5c0e0c030 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-25T00:37:23.957 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.945+0000 7fd66d7f8640 1 --2- 172.21.15.32:0/3371129026 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d5c0e01ea0 0x55d5c0e0c030 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:23.957 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.945+0000 7fd65affd640 1 --2- 172.21.15.32:0/2490678968 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd6540a2f10 0x7fd6540a53d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:23.957 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.945+0000 7fd66d7f8640 1 -- 172.21.15.32:0/3371129026 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e01580 msgr2=0x55d5c0e01960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:23.957 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.945+0000 7fd66d7f8640 1 --2- 172.21.15.32:0/3371129026 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e01580 0x55d5c0e01960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:23.957 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.945+0000 7fd66d7f8640 1 -- 172.21.15.32:0/3371129026 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d5c0d32400 con 0x55d5c0e0c570 2024-04-25T00:37:23.957 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.946+0000 7fd66d7f8640 1 --2- 172.21.15.32:0/3371129026 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d5c0e0c570 0x55d5c0e0e960 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7fd6600129d0 tx=0x7fd660004ab0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4364af9800f6d5aa server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:23.957 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.946+0000 7fd65affd640 1 -- 172.21.15.32:0/2490678968 <== mon.2 v2:172.21.15.121:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fd6640effe0 con 0x7fd66811af20 2024-04-25T00:37:23.957 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.946+0000 7fd66e7fa640 1 --2- 172.21.15.32:0/2490678968 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd6540a2f10 0x7fd6540a53d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:23.958 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.946+0000 7fd653fff640 1 -- 172.21.15.32:0/3371129026 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd66000ea30 con 0x55d5c0e0c570 2024-04-25T00:37:23.958 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.947+0000 7fd653fff640 1 -- 172.21.15.32:0/3371129026 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd66000ebd0 con 0x55d5c0e0c570 2024-04-25T00:37:23.958 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.948+0000 7fd672f6d740 1 -- 172.21.15.32:0/3371129026 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d5c0e0c570 msgr2=0x55d5c0e0e960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:23.958 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.948+0000 7fd672f6d740 1 --2- 172.21.15.32:0/3371129026 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d5c0e0c570 0x55d5c0e0e960 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7fd6600129d0 tx=0x7fd660004ab0 comp rx=0 tx=0).stop 2024-04-25T00:37:23.958 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.949+0000 7fd66e7fa640 1 --2- 172.21.15.32:0/2490678968 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd6540a2f10 0x7fd6540a53d0 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7fd664006010 tx=0x7fd6640f0230 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:23.958 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.951+0000 7fd672f6d740 1 -- 172.21.15.32:0/3371129026 shutdown_connections 2024-04-25T00:37:23.958 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.951+0000 7fd672f6d740 1 --2- 172.21.15.32:0/3371129026 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d5c0e0c570 0x55d5c0e0e960 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:23.958 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.951+0000 7fd672f6d740 1 --2- 172.21.15.32:0/3371129026 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d5c0e01ea0 0x55d5c0e0c030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:23.958 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.951+0000 7fd672f6d740 1 --2- 172.21.15.32:0/3371129026 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e01580 0x55d5c0e01960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:23.958 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.951+0000 7fd672f6d740 1 -- 172.21.15.32:0/3371129026 >> 172.21.15.32:0/3371129026 conn(0x55d5c0dfd8a0 msgr2=0x55d5c0dfdc80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:23.963 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.956+0000 7fd672f6d740 1 -- 172.21.15.32:0/3371129026 shutdown_connections 2024-04-25T00:37:23.963 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.959+0000 7fd672f6d740 1 -- 172.21.15.32:0/3371129026 wait complete. 2024-04-25T00:37:23.963 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.959+0000 7fd672f6d740 1 -- start start 2024-04-25T00:37:23.963 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.959+0000 7fd672f6d740 1 Processor -- start 2024-04-25T00:37:23.967 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.963+0000 7fd672f6d740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d5c0e01580 0x55d5c0e0be30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:23.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.963+0000 7fd672f6d740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e01ea0 0x55d5c0e09120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:23.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.963+0000 7fd672f6d740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d5c0e0c570 0x55d5c0e09660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:23.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.963+0000 7fd672f6d740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55d5c0e0f3b0 con 0x55d5c0e01ea0 2024-04-25T00:37:23.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.963+0000 7fd672f6d740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55d5c0d22050 con 0x55d5c0e01580 2024-04-25T00:37:23.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.963+0000 7fd672f6d740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55d5c0e0f530 con 0x55d5c0e0c570 2024-04-25T00:37:23.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.963+0000 7fd672f6d740 10 client.0 _ll_register_callbacks cb 0x55d5c0a79be0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:37:23.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.965+0000 7fd66dff9640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e01ea0 0x55d5c0e09120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:23.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.965+0000 7fd66dff9640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e01ea0 0x55d5c0e09120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46680/0 (socket says 172.21.15.32:46680) 2024-04-25T00:37:23.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.965+0000 7fd66dff9640 1 -- 172.21.15.32:0/1167576285 learned_addr learned my addr 172.21.15.32:0/1167576285 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:23.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.967+0000 7fd66d7f8640 1 --2- 172.21.15.32:0/1167576285 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d5c0e01580 0x55d5c0e0be30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:23.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.967+0000 7fd66d7f8640 1 --2- 172.21.15.32:0/1167576285 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d5c0e0c570 0x55d5c0e09660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:23.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.969+0000 7fd66dff9640 1 -- 172.21.15.32:0/1167576285 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d5c0e0c570 msgr2=0x55d5c0e09660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:23.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.969+0000 7fd66dff9640 1 --2- 172.21.15.32:0/1167576285 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d5c0e0c570 0x55d5c0e09660 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:23.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.969+0000 7fd66dff9640 1 -- 172.21.15.32:0/1167576285 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d5c0e01580 msgr2=0x55d5c0e0be30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:23.971 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.969+0000 7fd66dff9640 1 --2- 172.21.15.32:0/1167576285 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d5c0e01580 0x55d5c0e0be30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:23.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.969+0000 7fd66dff9640 1 -- 172.21.15.32:0/1167576285 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d5c0cf3dc0 con 0x55d5c0e01ea0 2024-04-25T00:37:23.974 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.972+0000 7fd66dff9640 1 --2- 172.21.15.32:0/1167576285 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e01ea0 0x55d5c0e09120 secure :-1 s=READY pgs=624 cs=0 l=1 rev1=1 crypto rx=0x7fd65c009dd0 tx=0x7fd65c00b4b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=68d9a632aef795e4 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:23.974 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.972+0000 7fd6517fa640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:37:23.975 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.974+0000 7fd6517fa640 1 -- 172.21.15.32:0/1167576285 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd65c015020 con 0x55d5c0e01ea0 2024-04-25T00:37:23.975 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.974+0000 7fd672f6d740 1 -- 172.21.15.32:0/1167576285 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd66000ebd0 con 0x55d5c0e01ea0 2024-04-25T00:37:23.975 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.974+0000 7fd6517fa640 1 -- 172.21.15.32:0/1167576285 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fd65c011420 con 0x55d5c0e01ea0 2024-04-25T00:37:23.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.982+0000 7fd6517fa640 1 -- 172.21.15.32:0/1167576285 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fd65c011620 con 0x55d5c0e01ea0 2024-04-25T00:37:23.986 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.986+0000 7fd672f6d740 10 client.16218 fetch_fsmap learned FSMap version 27 2024-04-25T00:37:23.986 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.986+0000 7fd672f6d740 1 -- 172.21.15.32:0/1167576285 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x55d5c0a7a3f0 con 0x55d5c0e01ea0 2024-04-25T00:37:23.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.990+0000 7fd6517fa640 1 -- 172.21.15.32:0/1167576285 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd66000ebd0 con 0x55d5c0e01ea0 2024-04-25T00:37:23.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.990+0000 7fd6517fa640 1 -- 172.21.15.32:0/1167576285 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 27) ==== 8354+0+0 (secure 0 0 0) 0x7fd65c022020 con 0x55d5c0e01ea0 2024-04-25T00:37:23.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.990+0000 7fd672f6d740 10 client.16218 fetch_fsmap finished waiting for FSMap version 27 2024-04-25T00:37:23.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.990+0000 7fd672f6d740 10 client.16218 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-25T00:37:23.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.990+0000 7fd672f6d740 20 client.16218 populate_metadata read hostname 'smithi032' 2024-04-25T00:37:23.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.990+0000 7fd672f6d740 1 --2- 172.21.15.32:0/1167576285 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x55d5c0e2db30 0x55d5c0e2fff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:23.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.990+0000 7fd672f6d740 4 client.16218 mds_command: new command op to 24476 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:37:23.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.990+0000 7fd672f6d740 1 -- 172.21.15.32:0/1167576285 --> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55d5c0d73bb0 con 0x55d5c0e2db30 2024-04-25T00:37:23.995 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:23.995+0000 7fd66d7f8640 1 --2- 172.21.15.32:0/1167576285 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x55d5c0e2db30 0x55d5c0e2fff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:23.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:23 smithi032 ceph-mon[29357]: pgmap v125: 129 pgs: 129 active+clean; 35 GiB data, 73 GiB used, 1000 GiB / 1.0 TiB avail; 618 KiB/s rd, 434 MiB/s wr, 1.89k op/s 2024-04-25T00:37:23.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:23 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/432827295' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:37:24.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.003+0000 7fd66d7f8640 1 --2- 172.21.15.32:0/1167576285 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x55d5c0e2db30 0x55d5c0e2fff0 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=db09eca0e4f83ab5 server_cookie=2f10f7e449c24921 in_seq=0 out_seq=0 2024-04-25T00:37:24.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.003+0000 7fd6517fa640 10 client.16218 ms_handle_connect on v2:172.21.15.121:6836/3456022860 2024-04-25T00:37:24.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.006+0000 7fd6517fa640 1 -- 172.21.15.32:0/1167576285 <== mds.2 v2:172.21.15.121:6836/3456022860 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x55d5c0d73bb0 con 0x55d5c0e2db30 2024-04-25T00:37:24.006 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.006+0000 7fd6517fa640 10 client.16218 handle_command_reply: tid=0 2024-04-25T00:37:24.017 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.011+0000 7fd672f6d740 1 client.16218 shutdown 2024-04-25T00:37:24.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.019+0000 7fd672f6d740 1 -- 172.21.15.32:0/1167576285 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e01ea0 msgr2=0x55d5c0e09120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:24.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.019+0000 7fd672f6d740 1 --2- 172.21.15.32:0/1167576285 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e01ea0 0x55d5c0e09120 secure :-1 s=READY pgs=624 cs=0 l=1 rev1=1 crypto rx=0x7fd65c009dd0 tx=0x7fd65c00b4b0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.020+0000 7fd672f6d740 1 -- 172.21.15.32:0/1167576285 shutdown_connections 2024-04-25T00:37:24.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.020+0000 7fd672f6d740 1 --2- 172.21.15.32:0/1167576285 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x55d5c0e2db30 0x55d5c0e2fff0 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.020+0000 7fd672f6d740 1 --2- 172.21.15.32:0/1167576285 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d5c0e0c570 0x55d5c0e09660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.020+0000 7fd672f6d740 1 --2- 172.21.15.32:0/1167576285 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e01ea0 0x55d5c0e09120 unknown :-1 s=CLOSED pgs=624 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.020+0000 7fd672f6d740 1 --2- 172.21.15.32:0/1167576285 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d5c0e01580 0x55d5c0e0be30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.020+0000 7fd672f6d740 1 -- 172.21.15.32:0/1167576285 >> 172.21.15.32:0/1167576285 conn(0x55d5c0dfd8a0 msgr2=0x55d5c0dfcb40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:24.023 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.020+0000 7fd6517fa640 0 client.16218 ms_handle_reset on v2:172.21.15.121:6836/3456022860 2024-04-25T00:37:24.027 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.024+0000 7fd672f6d740 1 -- 172.21.15.32:0/1167576285 shutdown_connections 2024-04-25T00:37:24.027 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.024+0000 7fd672f6d740 1 -- 172.21.15.32:0/1167576285 wait complete. 2024-04-25T00:37:24.027 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.024+0000 7fd672f6d740 20 client.16218 trim_cache size 0 max 16384 2024-04-25T00:37:24.062 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.062+0000 7fd672f6d740 1 Processor -- start 2024-04-25T00:37:24.063 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.062+0000 7fd672f6d740 1 -- start start 2024-04-25T00:37:24.064 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.063+0000 7fd672f6d740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d5c0e01580 0x55d5c0e18350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.064 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.063+0000 7fd672f6d740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e01ea0 0x55d5c0e0b960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.065 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.063+0000 7fd672f6d740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d5c0e0c570 0x55d5c0e35900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.065 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.063+0000 7fd672f6d740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fd64c012990 con 0x55d5c0e01ea0 2024-04-25T00:37:24.065 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.063+0000 7fd672f6d740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fd64c010260 con 0x55d5c0e0c570 2024-04-25T00:37:24.065 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.063+0000 7fd66dff9640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e01ea0 0x55d5c0e0b960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:24.065 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.063+0000 7fd66dff9640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e01ea0 0x55d5c0e0b960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46686/0 (socket says 172.21.15.32:46686) 2024-04-25T00:37:24.065 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.063+0000 7fd66dff9640 1 -- 172.21.15.32:0/3705413645 learned_addr learned my addr 172.21.15.32:0/3705413645 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:24.065 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.063+0000 7fd672f6d740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fd64c00e0b0 con 0x55d5c0e01580 2024-04-25T00:37:24.065 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.063+0000 7fd66d7f8640 1 --2- 172.21.15.32:0/3705413645 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d5c0e01580 0x55d5c0e18350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:24.065 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.064+0000 7fd66dff9640 1 -- 172.21.15.32:0/3705413645 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d5c0e01580 msgr2=0x55d5c0e18350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:24.065 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.064+0000 7fd66dff9640 1 --2- 172.21.15.32:0/3705413645 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d5c0e01580 0x55d5c0e18350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.065 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.064+0000 7fd66dff9640 1 -- 172.21.15.32:0/3705413645 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d5c0e0c570 msgr2=0x55d5c0e35900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:24.065 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.064+0000 7fd66dff9640 1 --2- 172.21.15.32:0/3705413645 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d5c0e0c570 0x55d5c0e35900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.065 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.064+0000 7fd66dff9640 1 -- 172.21.15.32:0/3705413645 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d5c0b447d0 con 0x55d5c0e01ea0 2024-04-25T00:37:24.066 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.064+0000 7fd66dff9640 1 --2- 172.21.15.32:0/3705413645 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e01ea0 0x55d5c0e0b960 secure :-1 s=READY pgs=625 cs=0 l=1 rev1=1 crypto rx=0x7fd65c006b90 tx=0x7fd65c0115b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c9c63527c081f360 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:24.066 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.064+0000 7fd6537fe640 1 -- 172.21.15.32:0/3705413645 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd65c027070 con 0x55d5c0e01ea0 2024-04-25T00:37:24.066 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.065+0000 7fd6537fe640 1 -- 172.21.15.32:0/3705413645 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd65c00be50 con 0x55d5c0e01ea0 2024-04-25T00:37:24.066 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.065+0000 7fd6537fe640 1 -- 172.21.15.32:0/3705413645 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd65c015040 con 0x55d5c0e01ea0 2024-04-25T00:37:24.067 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.066+0000 7fd672f6d740 1 -- 172.21.15.32:0/3705413645 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e01ea0 msgr2=0x55d5c0e0b960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:24.067 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.066+0000 7fd672f6d740 1 --2- 172.21.15.32:0/3705413645 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e01ea0 0x55d5c0e0b960 secure :-1 s=READY pgs=625 cs=0 l=1 rev1=1 crypto rx=0x7fd65c006b90 tx=0x7fd65c0115b0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.067 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.066+0000 7fd672f6d740 1 -- 172.21.15.32:0/3705413645 shutdown_connections 2024-04-25T00:37:24.067 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.066+0000 7fd672f6d740 1 --2- 172.21.15.32:0/3705413645 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d5c0e0c570 0x55d5c0e35900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.067 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.066+0000 7fd672f6d740 1 --2- 172.21.15.32:0/3705413645 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e01ea0 0x55d5c0e0b960 unknown :-1 s=CLOSED pgs=625 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.066+0000 7fd672f6d740 1 --2- 172.21.15.32:0/3705413645 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d5c0e01580 0x55d5c0e18350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.066+0000 7fd672f6d740 1 -- 172.21.15.32:0/3705413645 >> 172.21.15.32:0/3705413645 conn(0x55d5c0dfd8a0 msgr2=0x55d5c0e090e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:24.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.067+0000 7fd672f6d740 1 -- 172.21.15.32:0/3705413645 shutdown_connections 2024-04-25T00:37:24.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.067+0000 7fd672f6d740 1 -- 172.21.15.32:0/3705413645 wait complete. 2024-04-25T00:37:24.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.067+0000 7fd672f6d740 1 -- start start 2024-04-25T00:37:24.068 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.068+0000 7fd672f6d740 1 Processor -- start 2024-04-25T00:37:24.069 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.069+0000 7fd672f6d740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d5c0e01580 0x55d5c0e14860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.069+0000 7fd672f6d740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d5c0e01ea0 0x55d5c0e14da0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.070 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.069+0000 7fd66d7f8640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d5c0e01580 0x55d5c0e14860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:24.071 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.070+0000 7fd66d7f8640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d5c0e01580 0x55d5c0e14860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:34114/0 (socket says 172.21.15.32:34114) 2024-04-25T00:37:24.071 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.070+0000 7fd66d7f8640 1 -- 172.21.15.32:0/830293761 learned_addr learned my addr 172.21.15.32:0/830293761 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:24.072 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.070+0000 7fd672f6d740 1 --2- 172.21.15.32:0/830293761 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e0c570 0x55d5c0e0fde0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.072 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.070+0000 7fd672f6d740 1 -- 172.21.15.32:0/830293761 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fd64c00e720 con 0x55d5c0e0c570 2024-04-25T00:37:24.072 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.070+0000 7fd672f6d740 1 -- 172.21.15.32:0/830293761 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fd64c00fb90 con 0x55d5c0e01580 2024-04-25T00:37:24.072 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.071+0000 7fd672f6d740 1 -- 172.21.15.32:0/830293761 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fd64c00edf0 con 0x55d5c0e01ea0 2024-04-25T00:37:24.072 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.072+0000 7fd66dff9640 1 --2- 172.21.15.32:0/830293761 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d5c0e01ea0 0x55d5c0e14da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:24.073 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.072+0000 7fd66d7f8640 1 --2- 172.21.15.32:0/830293761 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e0c570 0x55d5c0e0fde0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:24.074 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.073+0000 7fd66dff9640 1 -- 172.21.15.32:0/830293761 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d5c0e01580 msgr2=0x55d5c0e14860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:24.074 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.073+0000 7fd66dff9640 1 --2- 172.21.15.32:0/830293761 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d5c0e01580 0x55d5c0e14860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.074 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.073+0000 7fd66dff9640 1 -- 172.21.15.32:0/830293761 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e0c570 msgr2=0x55d5c0e0fde0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:24.074 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.073+0000 7fd66dff9640 1 --2- 172.21.15.32:0/830293761 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e0c570 0x55d5c0e0fde0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.074 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.073+0000 7fd66dff9640 1 -- 172.21.15.32:0/830293761 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d5c0e105e0 con 0x55d5c0e01ea0 2024-04-25T00:37:24.074 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.073+0000 7fd66d7f8640 1 --2- 172.21.15.32:0/830293761 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e0c570 0x55d5c0e0fde0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:37:24.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.073+0000 7fd672f6d740 10 client.0 _ll_register_callbacks cb 0x7fd65c006e80 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:37:24.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.073+0000 7fd672f6d740 1 -- 172.21.15.32:0/830293761 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd65c00be50 con 0x55d5c0e01ea0 2024-04-25T00:37:24.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.074+0000 7fd66dff9640 1 --2- 172.21.15.32:0/830293761 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d5c0e01ea0 0x55d5c0e14da0 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x55d5c0e2c3c0 tx=0x7fd65c011df0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=755a998dc8c5cdab server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:24.078 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.074+0000 7fd6517fa640 10 client.25747 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:37:24.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.076+0000 7fd6517fa640 1 -- 172.21.15.32:0/830293761 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd65c02f060 con 0x55d5c0e01ea0 2024-04-25T00:37:24.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.076+0000 7fd6517fa640 1 -- 172.21.15.32:0/830293761 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fd65c004510 con 0x55d5c0e01ea0 2024-04-25T00:37:24.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.076+0000 7fd6517fa640 1 -- 172.21.15.32:0/830293761 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fd65c015070 con 0x55d5c0e01ea0 2024-04-25T00:37:24.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.076+0000 7fd672f6d740 10 client.25747 fetch_fsmap learned FSMap version 27 2024-04-25T00:37:24.079 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.076+0000 7fd672f6d740 1 -- 172.21.15.32:0/830293761 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({fsmap=27}) -- 0x55d5c0e93b90 con 0x55d5c0e01ea0 2024-04-25T00:37:24.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.077+0000 7fd6517fa640 1 -- 172.21.15.32:0/830293761 <== mon.1 v2:172.21.15.130:3300/0 4 ==== fsmap(e 27) ==== 8354+0+0 (secure 0 0 0) 0x7fd65c022070 con 0x55d5c0e01ea0 2024-04-25T00:37:24.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.077+0000 7fd672f6d740 10 client.25747 fetch_fsmap finished waiting for FSMap version 27 2024-04-25T00:37:24.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.077+0000 7fd672f6d740 10 client.25747 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-25T00:37:24.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.077+0000 7fd672f6d740 20 client.25747 populate_metadata read hostname 'smithi032' 2024-04-25T00:37:24.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.077+0000 7fd672f6d740 1 --2- 172.21.15.32:0/830293761 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x55d5c0e94150 0x55d5c0e96540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.077+0000 7fd672f6d740 4 client.25747 mds_command: new command op to 24476 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-25T00:37:24.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.077+0000 7fd672f6d740 1 -- 172.21.15.32:0/830293761 --> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] -- command(tid 0: {"prefix": "damage ls"}) -- 0x7fd65c00be50 con 0x55d5c0e94150 2024-04-25T00:37:24.080 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.078+0000 7fd66d7f8640 1 --2- 172.21.15.32:0/830293761 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x55d5c0e94150 0x55d5c0e96540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:24.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.081+0000 7fd66d7f8640 1 --2- 172.21.15.32:0/830293761 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x55d5c0e94150 0x55d5c0e96540 crc :-1 s=READY pgs=46 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=f720244761dee2cb server_cookie=89d3bcb1b0c5b8ba in_seq=0 out_seq=0 2024-04-25T00:37:24.082 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.081+0000 7fd6517fa640 10 client.25747 ms_handle_connect on v2:172.21.15.121:6836/3456022860 2024-04-25T00:37:24.084 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.083+0000 7fd6517fa640 1 -- 172.21.15.32:0/830293761 <== mds.2 v2:172.21.15.121:6836/3456022860 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x7fd65c00be50 con 0x55d5c0e94150 2024-04-25T00:37:24.084 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.083+0000 7fd6517fa640 10 client.25747 handle_command_reply: tid=0 2024-04-25T00:37:24.084 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.083+0000 7fd672f6d740 1 client.25747 shutdown 2024-04-25T00:37:24.084 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.083+0000 7fd672f6d740 1 -- 172.21.15.32:0/830293761 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d5c0e01ea0 msgr2=0x55d5c0e14da0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:24.084 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.083+0000 7fd672f6d740 1 --2- 172.21.15.32:0/830293761 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d5c0e01ea0 0x55d5c0e14da0 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x55d5c0e2c3c0 tx=0x7fd65c011df0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.084+0000 7fd672f6d740 1 -- 172.21.15.32:0/830293761 shutdown_connections 2024-04-25T00:37:24.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.084+0000 7fd672f6d740 1 --2- 172.21.15.32:0/830293761 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x55d5c0e94150 0x55d5c0e96540 crc :-1 s=READY pgs=46 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.087 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.084+0000 7fd6517fa640 0 client.25747 ms_handle_reset on v2:172.21.15.121:6836/3456022860 2024-04-25T00:37:24.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.085+0000 7fd672f6d740 1 --2- 172.21.15.32:0/830293761 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d5c0e0c570 0x55d5c0e0fde0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.085+0000 7fd672f6d740 1 --2- 172.21.15.32:0/830293761 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d5c0e01ea0 0x55d5c0e14da0 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.085+0000 7fd672f6d740 1 --2- 172.21.15.32:0/830293761 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d5c0e01580 0x55d5c0e14860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.085+0000 7fd672f6d740 1 -- 172.21.15.32:0/830293761 >> 172.21.15.32:0/830293761 conn(0x55d5c0dfd8a0 msgr2=0x55d5c0e03fd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:24.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.085+0000 7fd672f6d740 1 -- 172.21.15.32:0/830293761 shutdown_connections 2024-04-25T00:37:24.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.085+0000 7fd672f6d740 1 -- 172.21.15.32:0/830293761 wait complete. 2024-04-25T00:37:24.088 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.085+0000 7fd672f6d740 20 client.25747 trim_cache size 0 max 16384 2024-04-25T00:37:24.088 INFO:teuthology.orchestra.run.smithi032.stdout:[] 2024-04-25T00:37:24.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.089+0000 7fd67025d640 1 -- 172.21.15.32:0/2490678968 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd6540a2f10 msgr2=0x7fd6540a53d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:24.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.089+0000 7fd67025d640 1 --2- 172.21.15.32:0/2490678968 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd6540a2f10 0x7fd6540a53d0 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7fd664006010 tx=0x7fd6640f0230 comp rx=0 tx=0).stop 2024-04-25T00:37:24.092 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.089+0000 7fd67025d640 1 -- 172.21.15.32:0/2490678968 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd66811af20 msgr2=0x7fd6681ab770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:24.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.089+0000 7fd67025d640 1 --2- 172.21.15.32:0/2490678968 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd66811af20 0x7fd6681ab770 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7fd664039520 tx=0x7fd664039550 comp rx=0 tx=0).stop 2024-04-25T00:37:24.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.091+0000 7fd67025d640 1 -- 172.21.15.32:0/2490678968 shutdown_connections 2024-04-25T00:37:24.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.092+0000 7fd67025d640 1 --2- 172.21.15.32:0/2490678968 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd6540a2f10 0x7fd6540a53d0 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.092+0000 7fd67025d640 1 --2- 172.21.15.32:0/2490678968 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd66811af20 0x7fd6681ab770 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.092+0000 7fd67025d640 1 --2- 172.21.15.32:0/2490678968 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd66810ff90 0x7fd6681a73f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.092+0000 7fd67025d640 1 --2- 172.21.15.32:0/2490678968 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd66810dfc0 0x7fd6681a6eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.093 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.092+0000 7fd67025d640 1 -- 172.21.15.32:0/2490678968 >> 172.21.15.32:0/2490678968 conn(0x7fd66806c850 msgr2=0x7fd668070230 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:24.094 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.093+0000 7fd67025d640 1 -- 172.21.15.32:0/2490678968 shutdown_connections 2024-04-25T00:37:24.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.094+0000 7fd67025d640 1 -- 172.21.15.32:0/2490678968 wait complete. 2024-04-25T00:37:24.146 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-25T00:37:24.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.447+0000 7fd368b1b640 1 -- 172.21.15.32:0/3136971544 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd36410d120 msgr2=0x7fd36410d500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:24.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.447+0000 7fd368b1b640 1 --2- 172.21.15.32:0/3136971544 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd36410d120 0x7fd36410d500 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7fd35800af30 tx=0x7fd358032a50 comp rx=0 tx=0).stop 2024-04-25T00:37:24.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.449+0000 7fd368b1b640 1 -- 172.21.15.32:0/3136971544 shutdown_connections 2024-04-25T00:37:24.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.449+0000 7fd368b1b640 1 --2- 172.21.15.32:0/3136971544 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd3641090e0 0x7fd364114f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.449+0000 7fd368b1b640 1 --2- 172.21.15.32:0/3136971544 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd364108740 0x7fd364108ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.449+0000 7fd368b1b640 1 --2- 172.21.15.32:0/3136971544 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd36410d120 0x7fd36410d500 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.449+0000 7fd368b1b640 1 -- 172.21.15.32:0/3136971544 >> 172.21.15.32:0/3136971544 conn(0x7fd36406ccb0 msgr2=0x7fd36406d0c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:24.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.449+0000 7fd368b1b640 1 -- 172.21.15.32:0/3136971544 shutdown_connections 2024-04-25T00:37:24.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.450+0000 7fd368b1b640 1 -- 172.21.15.32:0/3136971544 wait complete. 2024-04-25T00:37:24.453 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.451+0000 7fd368b1b640 1 Processor -- start 2024-04-25T00:37:24.453 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.451+0000 7fd368b1b640 1 -- start start 2024-04-25T00:37:24.453 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.451+0000 7fd368b1b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd364108740 0x7fd36419e6c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.453 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.452+0000 7fd368b1b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd3641090e0 0x7fd36419ec00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.453 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.452+0000 7fd368b1b640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd36410d120 0x7fd3641a2f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.453 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.452+0000 7fd368b1b640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fd3641181c0 con 0x7fd3641090e0 2024-04-25T00:37:24.453 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.452+0000 7fd368b1b640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fd364118040 con 0x7fd364108740 2024-04-25T00:37:24.453 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.452+0000 7fd368b1b640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fd364118340 con 0x7fd36410d120 2024-04-25T00:37:24.455 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.453+0000 7fd36259c640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd364108740 0x7fd36419e6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:24.455 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.453+0000 7fd361d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd3641090e0 0x7fd36419ec00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:24.455 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.453+0000 7fd361d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd3641090e0 0x7fd36419ec00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46702/0 (socket says 172.21.15.32:46702) 2024-04-25T00:37:24.455 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.453+0000 7fd361d9b640 1 -- 172.21.15.32:0/1475619601 learned_addr learned my addr 172.21.15.32:0/1475619601 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:24.455 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.453+0000 7fd362d9d640 1 --2- 172.21.15.32:0/1475619601 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd36410d120 0x7fd3641a2f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:24.456 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.454+0000 7fd361d9b640 1 -- 172.21.15.32:0/1475619601 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd36410d120 msgr2=0x7fd3641a2f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:24.456 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.454+0000 7fd361d9b640 1 --2- 172.21.15.32:0/1475619601 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd36410d120 0x7fd3641a2f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.456 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.454+0000 7fd361d9b640 1 -- 172.21.15.32:0/1475619601 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd364108740 msgr2=0x7fd36419e6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:24.456 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.454+0000 7fd361d9b640 1 --2- 172.21.15.32:0/1475619601 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd364108740 0x7fd36419e6c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.456 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.454+0000 7fd361d9b640 1 -- 172.21.15.32:0/1475619601 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd3641a36b0 con 0x7fd3641090e0 2024-04-25T00:37:24.466 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.465+0000 7fd361d9b640 1 --2- 172.21.15.32:0/1475619601 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd3641090e0 0x7fd36419ec00 secure :-1 s=READY pgs=626 cs=0 l=1 rev1=1 crypto rx=0x7fd35400daf0 tx=0x7fd35400af60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:24.467 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.466+0000 7fd3537fe640 1 -- 172.21.15.32:0/1475619601 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd35401a070 con 0x7fd3641090e0 2024-04-25T00:37:24.467 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.467+0000 7fd368b1b640 1 -- 172.21.15.32:0/1475619601 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd3641a39a0 con 0x7fd3641090e0 2024-04-25T00:37:24.468 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.467+0000 7fd3537fe640 1 -- 172.21.15.32:0/1475619601 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd35400cc70 con 0x7fd3641090e0 2024-04-25T00:37:24.468 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.467+0000 7fd368b1b640 1 -- 172.21.15.32:0/1475619601 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd3641bc240 con 0x7fd3641090e0 2024-04-25T00:37:24.469 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.468+0000 7fd3537fe640 1 -- 172.21.15.32:0/1475619601 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd35401a070 con 0x7fd3641090e0 2024-04-25T00:37:24.472 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.472+0000 7fd36b82d740 1 Processor -- start 2024-04-25T00:37:24.472 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.472+0000 7fd36b82d740 1 -- start start 2024-04-25T00:37:24.473 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.473+0000 7fd36b82d740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566fa580 0x5610566fa960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.474 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.473+0000 7fd36b82d740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5610566faea0 0x561056705030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.474 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.473+0000 7fd362d9d640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566fa580 0x5610566fa960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:24.475 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.473+0000 7fd362d9d640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566fa580 0x5610566fa960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:34138/0 (socket says 172.21.15.32:34138) 2024-04-25T00:37:24.475 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.473+0000 7fd362d9d640 1 -- 172.21.15.32:0/2550383619 learned_addr learned my addr 172.21.15.32:0/2550383619 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:24.475 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.473+0000 7fd36b82d740 1 --2- 172.21.15.32:0/2550383619 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561056705570 0x561056707960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.475 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.474+0000 7fd36b82d740 1 -- 172.21.15.32:0/2550383619 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x56105665f670 con 0x5610566faea0 2024-04-25T00:37:24.475 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.474+0000 7fd36b82d740 1 -- 172.21.15.32:0/2550383619 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x561056607970 con 0x5610566fa580 2024-04-25T00:37:24.475 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.474+0000 7fd36b82d740 1 -- 172.21.15.32:0/2550383619 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x56105665b630 con 0x561056705570 2024-04-25T00:37:24.475 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.474+0000 7fd362d9d640 1 -- 172.21.15.32:0/2550383619 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561056705570 msgr2=0x561056707960 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-25T00:37:24.476 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.474+0000 7fd362d9d640 1 --2- 172.21.15.32:0/2550383619 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561056705570 0x561056707960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.476 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.474+0000 7fd362d9d640 1 -- 172.21.15.32:0/2550383619 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5610566faea0 msgr2=0x561056705030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:24.476 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.474+0000 7fd362d9d640 1 --2- 172.21.15.32:0/2550383619 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5610566faea0 0x561056705030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.476 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.474+0000 7fd362d9d640 1 -- 172.21.15.32:0/2550383619 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56105662b400 con 0x5610566fa580 2024-04-25T00:37:24.476 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.475+0000 7fd362d9d640 1 --2- 172.21.15.32:0/2550383619 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566fa580 0x5610566fa960 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7fd35c009ee0 tx=0x7fd35c010740 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c6ae766a3509d4ed server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:24.476 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.475+0000 7fd350ff9640 1 -- 172.21.15.32:0/2550383619 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd35c016070 con 0x5610566fa580 2024-04-25T00:37:24.476 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.475+0000 7fd350ff9640 1 -- 172.21.15.32:0/2550383619 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd35c004d20 con 0x5610566fa580 2024-04-25T00:37:24.476 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.475+0000 7fd350ff9640 1 -- 172.21.15.32:0/2550383619 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd35c012830 con 0x5610566fa580 2024-04-25T00:37:24.477 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.476+0000 7fd36b82d740 1 -- 172.21.15.32:0/2550383619 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566fa580 msgr2=0x5610566fa960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:24.478 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.476+0000 7fd36b82d740 1 --2- 172.21.15.32:0/2550383619 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566fa580 0x5610566fa960 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7fd35c009ee0 tx=0x7fd35c010740 comp rx=0 tx=0).stop 2024-04-25T00:37:24.478 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.477+0000 7fd36b82d740 1 -- 172.21.15.32:0/2550383619 shutdown_connections 2024-04-25T00:37:24.478 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.477+0000 7fd36b82d740 1 --2- 172.21.15.32:0/2550383619 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561056705570 0x561056707960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.478 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.477+0000 7fd36b82d740 1 --2- 172.21.15.32:0/2550383619 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5610566faea0 0x561056705030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.478 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.477+0000 7fd36b82d740 1 --2- 172.21.15.32:0/2550383619 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566fa580 0x5610566fa960 unknown :-1 s=CLOSED pgs=123 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.478 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.477+0000 7fd36b82d740 1 -- 172.21.15.32:0/2550383619 >> 172.21.15.32:0/2550383619 conn(0x5610566f68a0 msgr2=0x5610566f6c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:24.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.481+0000 7fd3537fe640 1 -- 172.21.15.32:0/1475619601 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7fd35401d070 con 0x7fd3641090e0 2024-04-25T00:37:24.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.482+0000 7fd36b82d740 1 -- 172.21.15.32:0/2550383619 shutdown_connections 2024-04-25T00:37:24.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.482+0000 7fd3537fe640 1 --2- 172.21.15.32:0/1475619601 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd34c0a30a0 0x7fd34c0a5560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.483+0000 7fd3537fe640 1 -- 172.21.15.32:0/1475619601 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fd354012070 con 0x7fd3641090e0 2024-04-25T00:37:24.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.483+0000 7fd36259c640 1 --2- 172.21.15.32:0/1475619601 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd34c0a30a0 0x7fd34c0a5560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:24.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.484+0000 7fd36b82d740 1 -- 172.21.15.32:0/2550383619 wait complete. 2024-04-25T00:37:24.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.484+0000 7fd36259c640 1 --2- 172.21.15.32:0/1475619601 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd34c0a30a0 0x7fd34c0a5560 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7fd358032f70 tx=0x7fd358000bc0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:24.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.485+0000 7fd36b82d740 1 -- start start 2024-04-25T00:37:24.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.485+0000 7fd36b82d740 1 Processor -- start 2024-04-25T00:37:24.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.485+0000 7fd36b82d740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5610566fa580 0x561056704f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.485+0000 7fd36b82d740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566faea0 0x561056702210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.485+0000 7fd36b82d740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561056705570 0x5610567027b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.485+0000 7fd36b82d740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x561056708360 con 0x5610566fa580 2024-04-25T00:37:24.487 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.485+0000 7fd36b82d740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x56105661b050 con 0x5610566faea0 2024-04-25T00:37:24.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.485+0000 7fd36b82d740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5610567084e0 con 0x561056705570 2024-04-25T00:37:24.488 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.486+0000 7fd36b82d740 10 client.0 _ll_register_callbacks cb 0x561056372be0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:37:24.489 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.488+0000 7fd361d9b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566faea0 0x561056702210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:24.489 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.488+0000 7fd361d9b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566faea0 0x561056702210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:34146/0 (socket says 172.21.15.32:34146) 2024-04-25T00:37:24.489 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.488+0000 7fd361d9b640 1 -- 172.21.15.32:0/1383450257 learned_addr learned my addr 172.21.15.32:0/1383450257 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:24.489 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.488+0000 7fd362d9d640 1 --2- 172.21.15.32:0/1383450257 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561056705570 0x5610567027b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:24.491 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.489+0000 7fd36259c640 1 --2- 172.21.15.32:0/1383450257 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5610566fa580 0x561056704f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:24.491 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.490+0000 7fd36259c640 1 -- 172.21.15.32:0/1383450257 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561056705570 msgr2=0x5610567027b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:24.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.490+0000 7fd36259c640 1 --2- 172.21.15.32:0/1383450257 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561056705570 0x5610567027b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.490+0000 7fd36259c640 1 -- 172.21.15.32:0/1383450257 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566faea0 msgr2=0x561056702210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:24.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.490+0000 7fd36259c640 1 --2- 172.21.15.32:0/1383450257 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566faea0 0x561056702210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.490+0000 7fd36259c640 1 -- 172.21.15.32:0/1383450257 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5610565ecdc0 con 0x5610566fa580 2024-04-25T00:37:24.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.490+0000 7fd36259c640 1 --2- 172.21.15.32:0/1383450257 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5610566fa580 0x561056704f20 secure :-1 s=READY pgs=627 cs=0 l=1 rev1=1 crypto rx=0x5610566fbb40 tx=0x7fd358032a50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=da1620845d0ca3fe server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:24.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.490+0000 7fd349ffb640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:37:24.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.491+0000 7fd349ffb640 1 -- 172.21.15.32:0/1383450257 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd358004f50 con 0x5610566fa580 2024-04-25T00:37:24.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.491+0000 7fd36b82d740 1 -- 172.21.15.32:0/1383450257 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd35c004d20 con 0x5610566fa580 2024-04-25T00:37:24.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.491+0000 7fd349ffb640 1 -- 172.21.15.32:0/1383450257 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fd3580050f0 con 0x5610566fa580 2024-04-25T00:37:24.492 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.492+0000 7fd349ffb640 1 -- 172.21.15.32:0/1383450257 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fd358057050 con 0x5610566fa580 2024-04-25T00:37:24.497 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.497+0000 7fd36b82d740 10 client.16230 fetch_fsmap learned FSMap version 27 2024-04-25T00:37:24.497 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.497+0000 7fd36b82d740 1 -- 172.21.15.32:0/1383450257 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x5610563733f0 con 0x5610566fa580 2024-04-25T00:37:24.501 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.499+0000 7fd349ffb640 1 -- 172.21.15.32:0/1383450257 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd358037460 con 0x5610566fa580 2024-04-25T00:37:24.501 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.499+0000 7fd349ffb640 1 -- 172.21.15.32:0/1383450257 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 27) ==== 8354+0+0 (secure 0 0 0) 0x7fd35800a0d0 con 0x5610566fa580 2024-04-25T00:37:24.501 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.499+0000 7fd36b82d740 10 client.16230 fetch_fsmap finished waiting for FSMap version 27 2024-04-25T00:37:24.501 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.499+0000 7fd36b82d740 10 client.16230 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-25T00:37:24.501 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.499+0000 7fd36b82d740 20 client.16230 populate_metadata read hostname 'smithi032' 2024-04-25T00:37:24.501 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.499+0000 7fd36b82d740 1 --2- 172.21.15.32:0/1383450257 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x5610567158d0 0x561056717d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.501 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.499+0000 7fd36b82d740 4 client.16230 mds_command: new command op to 24481 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:37:24.501 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.499+0000 7fd36b82d740 1 -- 172.21.15.32:0/1383450257 --> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x7fd35c004d20 con 0x5610567158d0 2024-04-25T00:37:24.501 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.500+0000 7fd361d9b640 1 --2- 172.21.15.32:0/1383450257 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x5610567158d0 0x561056717d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:24.502 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.500+0000 7fd361d9b640 1 --2- 172.21.15.32:0/1383450257 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x5610567158d0 0x561056717d50 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=524fe0b0ab129f8c server_cookie=4c7aeb0d81d12f73 in_seq=0 out_seq=0 2024-04-25T00:37:24.502 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.500+0000 7fd349ffb640 10 client.16230 ms_handle_connect on v2:172.21.15.130:6838/4267125809 2024-04-25T00:37:24.503 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.503+0000 7fd349ffb640 1 -- 172.21.15.32:0/1383450257 <== mds.1 v2:172.21.15.130:6838/4267125809 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x7fd35c004d20 con 0x5610567158d0 2024-04-25T00:37:24.503 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.503+0000 7fd349ffb640 10 client.16230 handle_command_reply: tid=0 2024-04-25T00:37:24.503 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.503+0000 7fd36b82d740 1 client.16230 shutdown 2024-04-25T00:37:24.525 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.525+0000 7fd36b82d740 1 -- 172.21.15.32:0/1383450257 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5610566fa580 msgr2=0x561056704f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:24.525 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.525+0000 7fd36b82d740 1 --2- 172.21.15.32:0/1383450257 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5610566fa580 0x561056704f20 secure :-1 s=READY pgs=627 cs=0 l=1 rev1=1 crypto rx=0x5610566fbb40 tx=0x7fd358032a50 comp rx=0 tx=0).stop 2024-04-25T00:37:24.527 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.527+0000 7fd36b82d740 1 -- 172.21.15.32:0/1383450257 shutdown_connections 2024-04-25T00:37:24.527 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.527+0000 7fd36b82d740 1 --2- 172.21.15.32:0/1383450257 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561056705570 0x5610567027b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.527 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.527+0000 7fd36b82d740 1 --2- 172.21.15.32:0/1383450257 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566faea0 0x561056702210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.528 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.527+0000 7fd36b82d740 1 --2- 172.21.15.32:0/1383450257 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x5610567158d0 0x561056717d50 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.528 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.527+0000 7fd36b82d740 1 --2- 172.21.15.32:0/1383450257 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5610566fa580 0x561056704f20 unknown :-1 s=CLOSED pgs=627 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.528 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.527+0000 7fd36b82d740 1 -- 172.21.15.32:0/1383450257 >> 172.21.15.32:0/1383450257 conn(0x5610566f68a0 msgr2=0x5610566f5b40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:24.528 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.527+0000 7fd349ffb640 0 client.16230 ms_handle_reset on v2:172.21.15.130:6838/4267125809 2024-04-25T00:37:24.528 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.527+0000 7fd36b82d740 1 -- 172.21.15.32:0/1383450257 shutdown_connections 2024-04-25T00:37:24.528 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.527+0000 7fd36b82d740 1 -- 172.21.15.32:0/1383450257 wait complete. 2024-04-25T00:37:24.528 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.527+0000 7fd36b82d740 20 client.16230 trim_cache size 0 max 16384 2024-04-25T00:37:24.581 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.581+0000 7fd36b82d740 1 Processor -- start 2024-04-25T00:37:24.582 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.581+0000 7fd36b82d740 1 -- start start 2024-04-25T00:37:24.584 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.581+0000 7fd36b82d740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5610566fa580 0x5610567112b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.585 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.584+0000 7fd36b82d740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566faea0 0x561056700ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.589 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.588+0000 7fd36b82d740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561056705570 0x561056704a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.589 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.589+0000 7fd36b82d740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fd344012950 con 0x5610566fa580 2024-04-25T00:37:24.589 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.589+0000 7fd36b82d740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fd344010220 con 0x5610566faea0 2024-04-25T00:37:24.590 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.589+0000 7fd36b82d740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fd34400e070 con 0x561056705570 2024-04-25T00:37:24.590 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.589+0000 7fd361d9b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566faea0 0x561056700ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:24.590 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.589+0000 7fd361d9b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566faea0 0x561056700ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:34158/0 (socket says 172.21.15.32:34158) 2024-04-25T00:37:24.590 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.589+0000 7fd361d9b640 1 -- 172.21.15.32:0/1294528946 learned_addr learned my addr 172.21.15.32:0/1294528946 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:24.591 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.591+0000 7fd36259c640 1 --2- 172.21.15.32:0/1294528946 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5610566fa580 0x5610567112b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:24.591 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.591+0000 7fd362d9d640 1 --2- 172.21.15.32:0/1294528946 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561056705570 0x561056704a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:24.592 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.591+0000 7fd361d9b640 1 -- 172.21.15.32:0/1294528946 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561056705570 msgr2=0x561056704a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:24.592 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.591+0000 7fd361d9b640 1 --2- 172.21.15.32:0/1294528946 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561056705570 0x561056704a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.592 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.591+0000 7fd361d9b640 1 -- 172.21.15.32:0/1294528946 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5610566fa580 msgr2=0x5610567112b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:24.592 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.591+0000 7fd361d9b640 1 --2- 172.21.15.32:0/1294528946 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5610566fa580 0x5610567112b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.592 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.591+0000 7fd361d9b640 1 -- 172.21.15.32:0/1294528946 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5610567011c0 con 0x5610566faea0 2024-04-25T00:37:24.592 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.591+0000 7fd362d9d640 1 --2- 172.21.15.32:0/1294528946 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561056705570 0x561056704a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:37:24.592 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.592+0000 7fd361d9b640 1 --2- 172.21.15.32:0/1294528946 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566faea0 0x561056700ae0 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7fd35400afd0 tx=0x7fd354000bc0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=68b4abe5383192d7 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:24.596 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.596+0000 7fd34bfff640 1 -- 172.21.15.32:0/1294528946 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd35401a040 con 0x5610566faea0 2024-04-25T00:37:24.596 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.596+0000 7fd34bfff640 1 -- 172.21.15.32:0/1294528946 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd35409e960 con 0x5610566faea0 2024-04-25T00:37:24.597 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.596+0000 7fd34bfff640 1 -- 172.21.15.32:0/1294528946 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd354096c20 con 0x5610566faea0 2024-04-25T00:37:24.598 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.597+0000 7fd36b82d740 1 -- 172.21.15.32:0/1294528946 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566faea0 msgr2=0x561056700ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:24.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.597+0000 7fd36b82d740 1 --2- 172.21.15.32:0/1294528946 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566faea0 0x561056700ae0 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7fd35400afd0 tx=0x7fd354000bc0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.598+0000 7fd36b82d740 1 -- 172.21.15.32:0/1294528946 shutdown_connections 2024-04-25T00:37:24.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.598+0000 7fd36b82d740 1 --2- 172.21.15.32:0/1294528946 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561056705570 0x561056704a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.598+0000 7fd36b82d740 1 --2- 172.21.15.32:0/1294528946 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566faea0 0x561056700ae0 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.598+0000 7fd36b82d740 1 --2- 172.21.15.32:0/1294528946 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5610566fa580 0x5610567112b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.598+0000 7fd36b82d740 1 -- 172.21.15.32:0/1294528946 >> 172.21.15.32:0/1294528946 conn(0x5610566f68a0 msgr2=0x5610566fd740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:24.599 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.599+0000 7fd36b82d740 1 -- 172.21.15.32:0/1294528946 shutdown_connections 2024-04-25T00:37:24.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.599+0000 7fd36b82d740 1 -- 172.21.15.32:0/1294528946 wait complete. 2024-04-25T00:37:24.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.599+0000 7fd36b82d740 1 -- start start 2024-04-25T00:37:24.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.600+0000 7fd36b82d740 1 Processor -- start 2024-04-25T00:37:24.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.600+0000 7fd36b82d740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566fa580 0x561056709890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.601 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.601+0000 7fd36b82d740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5610566faea0 0x561056709dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.601+0000 7fd36b82d740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561056705570 0x56105677ae90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.601+0000 7fd36b82d740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fd34400e6e0 con 0x5610566faea0 2024-04-25T00:37:24.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.601+0000 7fd36b82d740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fd34400fb50 con 0x5610566fa580 2024-04-25T00:37:24.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.601+0000 7fd36259c640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566fa580 0x561056709890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:24.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.601+0000 7fd36259c640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566fa580 0x561056709890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:34160/0 (socket says 172.21.15.32:34160) 2024-04-25T00:37:24.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.601+0000 7fd36259c640 1 -- 172.21.15.32:0/3254730634 learned_addr learned my addr 172.21.15.32:0/3254730634 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:24.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.601+0000 7fd36b82d740 1 -- 172.21.15.32:0/3254730634 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fd34400edb0 con 0x561056705570 2024-04-25T00:37:24.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.602+0000 7fd36b82d740 10 client.0 _ll_register_callbacks cb 0x7fd3580083f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:37:24.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.602+0000 7fd36259c640 1 -- 172.21.15.32:0/3254730634 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561056705570 msgr2=0x56105677ae90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:24.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.602+0000 7fd36259c640 1 --2- 172.21.15.32:0/3254730634 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561056705570 0x56105677ae90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.602+0000 7fd36259c640 1 -- 172.21.15.32:0/3254730634 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5610566faea0 msgr2=0x561056709dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:24.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.602+0000 7fd36259c640 1 --2- 172.21.15.32:0/3254730634 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5610566faea0 0x561056709dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.603 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.602+0000 7fd36259c640 1 -- 172.21.15.32:0/3254730634 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5610566facd0 con 0x5610566fa580 2024-04-25T00:37:24.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.604+0000 7fd36259c640 1 --2- 172.21.15.32:0/3254730634 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566fa580 0x561056709890 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7fd358016830 tx=0x7fd358016860 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c925b86072440c36 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:24.605 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.604+0000 7fd349ffb640 10 client.0 ms_handle_connect on v2:172.21.15.121:3300/0 2024-04-25T00:37:24.610 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.610+0000 7fd349ffb640 1 -- 172.21.15.32:0/3254730634 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd35800a030 con 0x5610566fa580 2024-04-25T00:37:24.611 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.610+0000 7fd36b82d740 1 -- 172.21.15.32:0/3254730634 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd35409e960 con 0x5610566fa580 2024-04-25T00:37:24.611 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.610+0000 7fd349ffb640 1 -- 172.21.15.32:0/3254730634 <== mon.2 v2:172.21.15.121:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fd35800a1d0 con 0x5610566fa580 2024-04-25T00:37:24.612 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.612+0000 7fd349ffb640 1 -- 172.21.15.32:0/3254730634 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fd358004f50 con 0x5610566fa580 2024-04-25T00:37:24.613 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.612+0000 7fd36b82d740 10 client.25787 fetch_fsmap learned FSMap version 27 2024-04-25T00:37:24.613 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.612+0000 7fd36b82d740 1 -- 172.21.15.32:0/3254730634 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x56105643d7d0 con 0x5610566fa580 2024-04-25T00:37:24.613 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.613+0000 7fd349ffb640 1 -- 172.21.15.32:0/3254730634 <== mon.2 v2:172.21.15.121:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd35409e960 con 0x5610566fa580 2024-04-25T00:37:24.615 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.614+0000 7fd349ffb640 1 -- 172.21.15.32:0/3254730634 <== mon.2 v2:172.21.15.121:3300/0 5 ==== fsmap(e 27) ==== 8354+0+0 (secure 0 0 0) 0x7fd35805a070 con 0x5610566fa580 2024-04-25T00:37:24.615 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.615+0000 7fd36b82d740 10 client.25787 fetch_fsmap finished waiting for FSMap version 27 2024-04-25T00:37:24.615 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.615+0000 7fd36b82d740 10 client.25787 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-25T00:37:24.615 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.615+0000 7fd36b82d740 20 client.25787 populate_metadata read hostname 'smithi032' 2024-04-25T00:37:24.615 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.615+0000 7fd36b82d740 1 --2- 172.21.15.32:0/3254730634 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x56105677c9d0 0x56105677ee90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:24.616 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.615+0000 7fd36b82d740 4 client.25787 mds_command: new command op to 24481 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-25T00:37:24.616 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.615+0000 7fd36b82d740 1 -- 172.21.15.32:0/3254730634 --> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] -- command(tid 0: {"prefix": "damage ls"}) -- 0x56105666cbb0 con 0x56105677c9d0 2024-04-25T00:37:24.616 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.616+0000 7fd361d9b640 1 --2- 172.21.15.32:0/3254730634 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x56105677c9d0 0x56105677ee90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:24.618 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.617+0000 7fd349ffb640 10 client.25787 ms_handle_connect on v2:172.21.15.130:6838/4267125809 2024-04-25T00:37:24.618 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.617+0000 7fd361d9b640 1 --2- 172.21.15.32:0/3254730634 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x56105677c9d0 0x56105677ee90 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=6fb9eea570df6730 server_cookie=bbfe196cde78ee88 in_seq=0 out_seq=0 2024-04-25T00:37:24.618 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.618+0000 7fd349ffb640 1 -- 172.21.15.32:0/3254730634 <== mds.1 v2:172.21.15.130:6838/4267125809 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x56105666cbb0 con 0x56105677c9d0 2024-04-25T00:37:24.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.618+0000 7fd349ffb640 10 client.25787 handle_command_reply: tid=0 2024-04-25T00:37:24.619 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.618+0000 7fd36b82d740 1 client.25787 shutdown 2024-04-25T00:37:24.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.619+0000 7fd36b82d740 1 -- 172.21.15.32:0/3254730634 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566fa580 msgr2=0x561056709890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:24.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.619+0000 7fd36b82d740 1 --2- 172.21.15.32:0/3254730634 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566fa580 0x561056709890 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7fd358016830 tx=0x7fd358016860 comp rx=0 tx=0).stop 2024-04-25T00:37:24.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.619+0000 7fd36b82d740 1 -- 172.21.15.32:0/3254730634 shutdown_connections 2024-04-25T00:37:24.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.619+0000 7fd36b82d740 1 --2- 172.21.15.32:0/3254730634 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x561056705570 0x56105677ae90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.619+0000 7fd36b82d740 1 --2- 172.21.15.32:0/3254730634 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x56105677c9d0 0x56105677ee90 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.619+0000 7fd36b82d740 1 --2- 172.21.15.32:0/3254730634 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5610566faea0 0x561056709dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.619+0000 7fd36b82d740 1 --2- 172.21.15.32:0/3254730634 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5610566fa580 0x561056709890 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.619+0000 7fd36b82d740 1 -- 172.21.15.32:0/3254730634 >> 172.21.15.32:0/3254730634 conn(0x5610566f68a0 msgr2=0x5610566fbcd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:24.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.619+0000 7fd349ffb640 0 client.25787 ms_handle_reset on v2:172.21.15.130:6838/4267125809 2024-04-25T00:37:24.620 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.619+0000 7fd36b82d740 1 -- 172.21.15.32:0/3254730634 shutdown_connections 2024-04-25T00:37:24.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.620+0000 7fd36b82d740 1 -- 172.21.15.32:0/3254730634 wait complete. 2024-04-25T00:37:24.621 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.620+0000 7fd36b82d740 20 client.25787 trim_cache size 0 max 16384 2024-04-25T00:37:24.621 INFO:teuthology.orchestra.run.smithi032.stdout:[] 2024-04-25T00:37:24.633 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.633+0000 7fd368b1b640 1 -- 172.21.15.32:0/1475619601 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd34c0a30a0 msgr2=0x7fd34c0a5560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:24.633 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.633+0000 7fd368b1b640 1 --2- 172.21.15.32:0/1475619601 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd34c0a30a0 0x7fd34c0a5560 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7fd358032f70 tx=0x7fd358000bc0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.634 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.633+0000 7fd368b1b640 1 -- 172.21.15.32:0/1475619601 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd3641090e0 msgr2=0x7fd36419ec00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:24.634 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.633+0000 7fd368b1b640 1 --2- 172.21.15.32:0/1475619601 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd3641090e0 0x7fd36419ec00 secure :-1 s=READY pgs=626 cs=0 l=1 rev1=1 crypto rx=0x7fd35400daf0 tx=0x7fd35400af60 comp rx=0 tx=0).stop 2024-04-25T00:37:24.640 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.639+0000 7fd368b1b640 1 -- 172.21.15.32:0/1475619601 shutdown_connections 2024-04-25T00:37:24.640 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.639+0000 7fd368b1b640 1 --2- 172.21.15.32:0/1475619601 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd34c0a30a0 0x7fd34c0a5560 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.640 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.639+0000 7fd368b1b640 1 --2- 172.21.15.32:0/1475619601 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd36410d120 0x7fd3641a2f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.640 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.639+0000 7fd368b1b640 1 --2- 172.21.15.32:0/1475619601 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd3641090e0 0x7fd36419ec00 unknown :-1 s=CLOSED pgs=626 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.640 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.640+0000 7fd368b1b640 1 --2- 172.21.15.32:0/1475619601 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd364108740 0x7fd36419e6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:24.640 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.640+0000 7fd368b1b640 1 -- 172.21.15.32:0/1475619601 >> 172.21.15.32:0/1475619601 conn(0x7fd36406ccb0 msgr2=0x7fd3640710c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:24.642 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.641+0000 7fd368b1b640 1 -- 172.21.15.32:0/1475619601 shutdown_connections 2024-04-25T00:37:24.642 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:24.641+0000 7fd368b1b640 1 -- 172.21.15.32:0/1475619601 wait complete. 2024-04-25T00:37:24.693 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-25T00:37:25.052 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.050+0000 7fe51769a640 1 -- 172.21.15.32:0/1430083246 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe51011af20 msgr2=0x7fe51011d310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:25.053 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.050+0000 7fe51769a640 1 --2- 172.21.15.32:0/1430083246 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe51011af20 0x7fe51011d310 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7fe50c009930 tx=0x7fe50c02a460 comp rx=0 tx=0).stop 2024-04-25T00:37:25.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.055+0000 7fe51769a640 1 -- 172.21.15.32:0/1430083246 shutdown_connections 2024-04-25T00:37:25.056 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.055+0000 7fe51769a640 1 --2- 172.21.15.32:0/1430083246 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe51011af20 0x7fe51011d310 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.055+0000 7fe51769a640 1 --2- 172.21.15.32:0/1430083246 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe51010dfc0 0x7fe51010e440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.055+0000 7fe51769a640 1 --2- 172.21.15.32:0/1430083246 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe51010ff90 0x7fe51010da80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.055+0000 7fe51769a640 1 -- 172.21.15.32:0/1430083246 >> 172.21.15.32:0/1430083246 conn(0x7fe51006c850 msgr2=0x7fe51006cc60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:25.057 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.056+0000 7fe51769a640 1 -- 172.21.15.32:0/1430083246 shutdown_connections 2024-04-25T00:37:25.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.057+0000 7fe51769a640 1 -- 172.21.15.32:0/1430083246 wait complete. 2024-04-25T00:37:25.058 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.058+0000 7fe51769a640 1 Processor -- start 2024-04-25T00:37:25.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.059+0000 7fe51769a640 1 -- start start 2024-04-25T00:37:25.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.059+0000 7fe51769a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe51010dfc0 0x7fe510118540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:25.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.059+0000 7fe51769a640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe51010ff90 0x7fe510111610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:25.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.059+0000 7fe51769a640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe51011af20 0x7fe510111b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:25.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.059+0000 7fe51769a640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fe510120880 con 0x7fe51010dfc0 2024-04-25T00:37:25.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.059+0000 7fe51769a640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fe510120700 con 0x7fe51011af20 2024-04-25T00:37:25.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.059+0000 7fe51769a640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fe510120a00 con 0x7fe51010ff90 2024-04-25T00:37:25.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.059+0000 7fe515436640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe51010dfc0 0x7fe510118540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:25.060 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.059+0000 7fe515436640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe51010dfc0 0x7fe510118540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46730/0 (socket says 172.21.15.32:46730) 2024-04-25T00:37:25.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.059+0000 7fe515436640 1 -- 172.21.15.32:0/2226252877 learned_addr learned my addr 172.21.15.32:0/2226252877 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:25.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.060+0000 7fe515436640 1 -- 172.21.15.32:0/2226252877 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe51010ff90 msgr2=0x7fe510111610 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-04-25T00:37:25.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.060+0000 7fe515436640 1 --2- 172.21.15.32:0/2226252877 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe51010ff90 0x7fe510111610 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.060+0000 7fe515436640 1 -- 172.21.15.32:0/2226252877 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe51011af20 msgr2=0x7fe510111b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:25.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.060+0000 7fe515436640 1 --2- 172.21.15.32:0/2226252877 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe51011af20 0x7fe510111b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.061 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.060+0000 7fe515436640 1 -- 172.21.15.32:0/2226252877 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe510112160 con 0x7fe51010dfc0 2024-04-25T00:37:25.062 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.060+0000 7fe515436640 1 --2- 172.21.15.32:0/2226252877 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe51010dfc0 0x7fe510118540 secure :-1 s=READY pgs=628 cs=0 l=1 rev1=1 crypto rx=0x7fe50000d950 tx=0x7fe50000de10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:25.075 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.075+0000 7fe5067fc640 1 -- 172.21.15.32:0/2226252877 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe500017070 con 0x7fe51010dfc0 2024-04-25T00:37:25.076 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.075+0000 7fe51769a640 1 -- 172.21.15.32:0/2226252877 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe510112450 con 0x7fe51010dfc0 2024-04-25T00:37:25.076 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.075+0000 7fe51769a640 1 -- 172.21.15.32:0/2226252877 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe5101163e0 con 0x7fe51010dfc0 2024-04-25T00:37:25.076 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.075+0000 7fe5067fc640 1 -- 172.21.15.32:0/2226252877 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe500009c30 con 0x7fe51010dfc0 2024-04-25T00:37:25.076 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.075+0000 7fe5067fc640 1 -- 172.21.15.32:0/2226252877 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe50000f620 con 0x7fe51010dfc0 2024-04-25T00:37:25.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.081+0000 7fe5067fc640 1 -- 172.21.15.32:0/2226252877 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7fe500010460 con 0x7fe51010dfc0 2024-04-25T00:37:25.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.082+0000 7fe5067fc640 1 --2- 172.21.15.32:0/2226252877 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe4fc0a2e80 0x7fe4fc0a5340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:25.083 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.082+0000 7fe5067fc640 1 -- 172.21.15.32:0/2226252877 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fe5000cf330 con 0x7fe51010dfc0 2024-04-25T00:37:25.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.089+0000 7fe51a3a6740 1 Processor -- start 2024-04-25T00:37:25.089 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.089+0000 7fe51a3a6740 1 -- start start 2024-04-25T00:37:25.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.090+0000 7fe51a3a6740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560ac733e580 0x560ac733e960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:25.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.090+0000 7fe51a3a6740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ac733eea0 0x560ac7349030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:25.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.090+0000 7fe51a3a6740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560ac7349570 0x560ac734b960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:25.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.090+0000 7fe51a3a6740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x560ac72a3670 con 0x560ac7349570 2024-04-25T00:37:25.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.090+0000 7fe51a3a6740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x560ac724b970 con 0x560ac733e580 2024-04-25T00:37:25.091 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.091+0000 7fe51a3a6740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x560ac729f630 con 0x560ac733eea0 2024-04-25T00:37:25.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.091+0000 7fe514c35640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ac733eea0 0x560ac7349030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:25.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.091+0000 7fe514c35640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ac733eea0 0x560ac7349030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:60808/0 (socket says 172.21.15.32:60808) 2024-04-25T00:37:25.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.091+0000 7fe514c35640 1 -- 172.21.15.32:0/1984034170 learned_addr learned my addr 172.21.15.32:0/1984034170 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:25.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.091+0000 7fe515c37640 1 --2- 172.21.15.32:0/1984034170 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560ac7349570 0x560ac734b960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:25.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.091+0000 7fe514c35640 1 -- 172.21.15.32:0/1984034170 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560ac733e580 msgr2=0x560ac733e960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:25.095 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.091+0000 7fe514c35640 1 --2- 172.21.15.32:0/1984034170 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560ac733e580 0x560ac733e960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.091+0000 7fe514c35640 1 -- 172.21.15.32:0/1984034170 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560ac7349570 msgr2=0x560ac734b960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:25.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.091+0000 7fe514c35640 1 --2- 172.21.15.32:0/1984034170 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560ac7349570 0x560ac734b960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.091+0000 7fe514c35640 1 -- 172.21.15.32:0/1984034170 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x560ac726f400 con 0x560ac733eea0 2024-04-25T00:37:25.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.091+0000 7fe514c35640 1 --2- 172.21.15.32:0/1984034170 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ac733eea0 0x560ac7349030 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7fe508009e40 tx=0x7fe50800f730 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e0fdda7af0318fe server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:25.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.093+0000 7fe4fb7fe640 1 -- 172.21.15.32:0/1984034170 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe508011640 con 0x560ac733eea0 2024-04-25T00:37:25.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.093+0000 7fe4fb7fe640 1 -- 172.21.15.32:0/1984034170 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe508011cc0 con 0x560ac733eea0 2024-04-25T00:37:25.096 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.093+0000 7fe4fb7fe640 1 -- 172.21.15.32:0/1984034170 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe508019cb0 con 0x560ac733eea0 2024-04-25T00:37:25.097 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.096+0000 7fe51a3a6740 1 -- 172.21.15.32:0/1984034170 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ac733eea0 msgr2=0x560ac7349030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:25.097 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.096+0000 7fe51a3a6740 1 --2- 172.21.15.32:0/1984034170 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ac733eea0 0x560ac7349030 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7fe508009e40 tx=0x7fe50800f730 comp rx=0 tx=0).stop 2024-04-25T00:37:25.097 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.096+0000 7fe514c35640 1 --2- 172.21.15.32:0/2226252877 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe4fc0a2e80 0x7fe4fc0a5340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:25.101 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.099+0000 7fe51a3a6740 1 -- 172.21.15.32:0/1984034170 shutdown_connections 2024-04-25T00:37:25.102 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.099+0000 7fe51a3a6740 1 --2- 172.21.15.32:0/1984034170 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560ac7349570 0x560ac734b960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.102 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.099+0000 7fe51a3a6740 1 --2- 172.21.15.32:0/1984034170 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ac733eea0 0x560ac7349030 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.102 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.099+0000 7fe51a3a6740 1 --2- 172.21.15.32:0/1984034170 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560ac733e580 0x560ac733e960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.102 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.099+0000 7fe51a3a6740 1 -- 172.21.15.32:0/1984034170 >> 172.21.15.32:0/1984034170 conn(0x560ac733a8a0 msgr2=0x560ac733ac80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:25.102 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.100+0000 7fe51a3a6740 1 -- 172.21.15.32:0/1984034170 shutdown_connections 2024-04-25T00:37:25.104 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.103+0000 7fe51a3a6740 1 -- 172.21.15.32:0/1984034170 wait complete. 2024-04-25T00:37:25.104 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.103+0000 7fe51a3a6740 1 -- start start 2024-04-25T00:37:25.104 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.103+0000 7fe51a3a6740 1 Processor -- start 2024-04-25T00:37:25.105 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.104+0000 7fe51a3a6740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560ac733e580 0x560ac7348f80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:25.105 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.104+0000 7fe51a3a6740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560ac733eea0 0x560ac7346270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:25.106 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.104+0000 7fe51a3a6740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ac7349570 0x560ac7346880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:25.106 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.104+0000 7fe51a3a6740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x560ac734c360 con 0x560ac733eea0 2024-04-25T00:37:25.106 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.104+0000 7fe51a3a6740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x560ac725f050 con 0x560ac733e580 2024-04-25T00:37:25.106 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.104+0000 7fe51a3a6740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x560ac734c4e0 con 0x560ac7349570 2024-04-25T00:37:25.106 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.105+0000 7fe51a3a6740 10 client.0 _ll_register_callbacks cb 0x560ac6fb6be0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:37:25.107 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.105+0000 7fe515436640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560ac733e580 0x560ac7348f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:25.107 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.105+0000 7fe515436640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560ac733e580 0x560ac7348f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:34190/0 (socket says 172.21.15.32:34190) 2024-04-25T00:37:25.107 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.105+0000 7fe515436640 1 -- 172.21.15.32:0/2103606417 learned_addr learned my addr 172.21.15.32:0/2103606417 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:25.107 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.105+0000 7fe515c37640 1 --2- 172.21.15.32:0/2103606417 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ac7349570 0x560ac7346880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:25.107 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.106+0000 7fe515436640 1 -- 172.21.15.32:0/2103606417 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ac7349570 msgr2=0x560ac7346880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:25.107 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.106+0000 7fe515436640 1 --2- 172.21.15.32:0/2103606417 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ac7349570 0x560ac7346880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.107 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.106+0000 7fe515436640 1 -- 172.21.15.32:0/2103606417 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560ac733eea0 msgr2=0x560ac7346270 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-25T00:37:25.107 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.106+0000 7fe515436640 1 --2- 172.21.15.32:0/2103606417 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560ac733eea0 0x560ac7346270 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.107 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.106+0000 7fe515436640 1 -- 172.21.15.32:0/2103606417 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x560ac7230dc0 con 0x560ac733e580 2024-04-25T00:37:25.108 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.107+0000 7fe515436640 1 --2- 172.21.15.32:0/2103606417 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560ac733e580 0x560ac7348f80 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7fe500097900 tx=0x7fe50009bce0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a06c1d4bb240cc11 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:25.108 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.108+0000 7fe4f8ff9640 10 client.0 ms_handle_connect on v2:172.21.15.121:3300/0 2024-04-25T00:37:25.108 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.108+0000 7fe4f8ff9640 1 -- 172.21.15.32:0/2103606417 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe500020040 con 0x560ac733e580 2024-04-25T00:37:25.108 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.108+0000 7fe4f8ff9640 1 -- 172.21.15.32:0/2103606417 <== mon.2 v2:172.21.15.121:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fe50009a8f0 con 0x560ac733e580 2024-04-25T00:37:25.109 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.108+0000 7fe51a3a6740 1 -- 172.21.15.32:0/2103606417 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe508011cc0 con 0x560ac733e580 2024-04-25T00:37:25.111 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.111+0000 7fe514c35640 1 --2- 172.21.15.32:0/2226252877 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe4fc0a2e80 0x7fe4fc0a5340 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7fe508009e40 tx=0x7fe50800f730 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:25.115 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.113+0000 7fe4f8ff9640 1 -- 172.21.15.32:0/2103606417 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fe50009f850 con 0x560ac733e580 2024-04-25T00:37:25.115 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.114+0000 7fe51a3a6740 10 client.25793 fetch_fsmap learned FSMap version 27 2024-04-25T00:37:25.115 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.115+0000 7fe51a3a6740 1 -- 172.21.15.32:0/2103606417 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x560ac6fb73f0 con 0x560ac733e580 2024-04-25T00:37:25.129 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.128+0000 7fe4f8ff9640 1 -- 172.21.15.32:0/2103606417 <== mon.2 v2:172.21.15.121:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe508011cc0 con 0x560ac733e580 2024-04-25T00:37:25.129 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.128+0000 7fe4f8ff9640 1 -- 172.21.15.32:0/2103606417 <== mon.2 v2:172.21.15.121:3300/0 5 ==== fsmap(e 27) ==== 8354+0+0 (secure 0 0 0) 0x7fe500097310 con 0x560ac733e580 2024-04-25T00:37:25.129 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.129+0000 7fe51a3a6740 10 client.25793 fetch_fsmap finished waiting for FSMap version 27 2024-04-25T00:37:25.129 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.129+0000 7fe51a3a6740 10 client.25793 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:37:25.129 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.129+0000 7fe51a3a6740 20 client.25793 populate_metadata read hostname 'smithi032' 2024-04-25T00:37:25.130 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.129+0000 7fe51a3a6740 1 --2- 172.21.15.32:0/2103606417 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x560ac7359b90 0x560ac735bfb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:25.130 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.129+0000 7fe51a3a6740 4 client.25793 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:37:25.130 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.129+0000 7fe51a3a6740 1 -- 172.21.15.32:0/2103606417 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x560ac72b0bb0 con 0x560ac7359b90 2024-04-25T00:37:25.133 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.132+0000 7fe514c35640 1 --2- 172.21.15.32:0/2103606417 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x560ac7359b90 0x560ac735bfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:25.143 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.143+0000 7fe514c35640 1 --2- 172.21.15.32:0/2103606417 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x560ac7359b90 0x560ac735bfb0 crc :-1 s=READY pgs=89 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9217d74d6e13464d server_cookie=5974321a513b6d84 in_seq=0 out_seq=0 2024-04-25T00:37:25.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.145+0000 7fe4f8ff9640 10 client.25793 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:37:25.155 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.154+0000 7fe4f8ff9640 1 -- 172.21.15.32:0/2103606417 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x560ac72b0bb0 con 0x560ac7359b90 2024-04-25T00:37:25.155 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.154+0000 7fe4f8ff9640 10 client.25793 handle_command_reply: tid=0 2024-04-25T00:37:25.155 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.154+0000 7fe51a3a6740 1 client.25793 shutdown 2024-04-25T00:37:25.155 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.154+0000 7fe51a3a6740 1 -- 172.21.15.32:0/2103606417 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560ac733e580 msgr2=0x560ac7348f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:25.155 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.154+0000 7fe51a3a6740 1 --2- 172.21.15.32:0/2103606417 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560ac733e580 0x560ac7348f80 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7fe500097900 tx=0x7fe50009bce0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.155+0000 7fe51a3a6740 1 -- 172.21.15.32:0/2103606417 shutdown_connections 2024-04-25T00:37:25.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.155+0000 7fe51a3a6740 1 --2- 172.21.15.32:0/2103606417 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x560ac7359b90 0x560ac735bfb0 crc :-1 s=READY pgs=89 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.155+0000 7fe51a3a6740 1 --2- 172.21.15.32:0/2103606417 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ac7349570 0x560ac7346880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.155+0000 7fe51a3a6740 1 --2- 172.21.15.32:0/2103606417 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560ac733eea0 0x560ac7346270 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.155+0000 7fe51a3a6740 1 --2- 172.21.15.32:0/2103606417 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560ac733e580 0x560ac7348f80 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.160 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.155+0000 7fe51a3a6740 1 -- 172.21.15.32:0/2103606417 >> 172.21.15.32:0/2103606417 conn(0x560ac733a8a0 msgr2=0x560ac7339b40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:25.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.159+0000 7fe4f8ff9640 0 client.25793 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:37:25.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.160+0000 7fe51a3a6740 1 -- 172.21.15.32:0/2103606417 shutdown_connections 2024-04-25T00:37:25.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.164+0000 7fe51a3a6740 1 -- 172.21.15.32:0/2103606417 wait complete. 2024-04-25T00:37:25.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.164+0000 7fe51a3a6740 20 client.25793 trim_cache size 0 max 16384 2024-04-25T00:37:25.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.211+0000 7fe51a3a6740 1 Processor -- start 2024-04-25T00:37:25.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.211+0000 7fe51a3a6740 1 -- start start 2024-04-25T00:37:25.212 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.211+0000 7fe51a3a6740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560ac733e580 0x560ac73552b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:25.215 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.212+0000 7fe515436640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560ac733e580 0x560ac73552b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:25.215 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.212+0000 7fe515436640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560ac733e580 0x560ac73552b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46746/0 (socket says 172.21.15.32:46746) 2024-04-25T00:37:25.215 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.212+0000 7fe515436640 1 -- 172.21.15.32:0/4246893260 learned_addr learned my addr 172.21.15.32:0/4246893260 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:25.215 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.212+0000 7fe51a3a6740 1 --2- 172.21.15.32:0/4246893260 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ac733eea0 0x560ac7348ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:25.215 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.212+0000 7fe51a3a6740 1 --2- 172.21.15.32:0/4246893260 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560ac7349570 0x560ac73618a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:25.215 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.212+0000 7fe51a3a6740 1 -- 172.21.15.32:0/4246893260 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fe4f4012950 con 0x560ac733e580 2024-04-25T00:37:25.215 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.212+0000 7fe51a3a6740 1 -- 172.21.15.32:0/4246893260 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fe4f4010220 con 0x560ac7349570 2024-04-25T00:37:25.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.212+0000 7fe51a3a6740 1 -- 172.21.15.32:0/4246893260 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fe4f400e070 con 0x560ac733eea0 2024-04-25T00:37:25.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.213+0000 7fe515c37640 1 --2- 172.21.15.32:0/4246893260 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560ac7349570 0x560ac73618a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:25.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.213+0000 7fe515436640 1 -- 172.21.15.32:0/4246893260 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ac733eea0 msgr2=0x560ac7348ab0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-25T00:37:25.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.213+0000 7fe515436640 1 --2- 172.21.15.32:0/4246893260 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ac733eea0 0x560ac7348ab0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.213+0000 7fe515436640 1 -- 172.21.15.32:0/4246893260 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560ac7349570 msgr2=0x560ac73618a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:25.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.213+0000 7fe515436640 1 --2- 172.21.15.32:0/4246893260 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560ac7349570 0x560ac73618a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.213+0000 7fe515436640 1 -- 172.21.15.32:0/4246893260 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x560ac734bcd0 con 0x560ac733e580 2024-04-25T00:37:25.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.213+0000 7fe515436640 1 --2- 172.21.15.32:0/4246893260 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560ac733e580 0x560ac73552b0 secure :-1 s=READY pgs=629 cs=0 l=1 rev1=1 crypto rx=0x7fe5000cc780 tx=0x7fe500005880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6a69d6c66b815e47 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:25.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.221+0000 7fe4faffd640 1 -- 172.21.15.32:0/4246893260 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe500020070 con 0x560ac733e580 2024-04-25T00:37:25.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.221+0000 7fe4faffd640 1 -- 172.21.15.32:0/4246893260 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe5000a4110 con 0x560ac733e580 2024-04-25T00:37:25.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.221+0000 7fe4faffd640 1 -- 172.21.15.32:0/4246893260 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe50009a8f0 con 0x560ac733e580 2024-04-25T00:37:25.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.224+0000 7fe51a3a6740 1 -- 172.21.15.32:0/4246893260 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560ac733e580 msgr2=0x560ac73552b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:25.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.225+0000 7fe51a3a6740 1 --2- 172.21.15.32:0/4246893260 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560ac733e580 0x560ac73552b0 secure :-1 s=READY pgs=629 cs=0 l=1 rev1=1 crypto rx=0x7fe5000cc780 tx=0x7fe500005880 comp rx=0 tx=0).stop 2024-04-25T00:37:25.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.229+0000 7fe51a3a6740 1 -- 172.21.15.32:0/4246893260 shutdown_connections 2024-04-25T00:37:25.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.229+0000 7fe51a3a6740 1 --2- 172.21.15.32:0/4246893260 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560ac7349570 0x560ac73618a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.229+0000 7fe51a3a6740 1 --2- 172.21.15.32:0/4246893260 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ac733eea0 0x560ac7348ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.229+0000 7fe51a3a6740 1 --2- 172.21.15.32:0/4246893260 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560ac733e580 0x560ac73552b0 unknown :-1 s=CLOSED pgs=629 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.229+0000 7fe51a3a6740 1 -- 172.21.15.32:0/4246893260 >> 172.21.15.32:0/4246893260 conn(0x560ac733a8a0 msgr2=0x560ac7341860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:25.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.235+0000 7fe51a3a6740 1 -- 172.21.15.32:0/4246893260 shutdown_connections 2024-04-25T00:37:25.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.235+0000 7fe51a3a6740 1 -- 172.21.15.32:0/4246893260 wait complete. 2024-04-25T00:37:25.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.236+0000 7fe51a3a6740 1 -- start start 2024-04-25T00:37:25.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.236+0000 7fe51a3a6740 1 Processor -- start 2024-04-25T00:37:25.243 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.240+0000 7fe51a3a6740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560ac733e580 0x560ac7351c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:25.243 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.240+0000 7fe51a3a6740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560ac733eea0 0x560ac734cfb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:25.243 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.240+0000 7fe51a3a6740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ac7349570 0x560ac734d550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:25.243 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.240+0000 7fe51a3a6740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fe4f400e6e0 con 0x560ac733e580 2024-04-25T00:37:25.243 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.240+0000 7fe51a3a6740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fe4f400fb50 con 0x560ac733eea0 2024-04-25T00:37:25.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.240+0000 7fe51a3a6740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fe4f400edb0 con 0x560ac7349570 2024-04-25T00:37:25.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.240+0000 7fe51a3a6740 10 client.0 _ll_register_callbacks cb 0x7fe50000df40 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:37:25.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.240+0000 7fe514c35640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560ac733eea0 0x560ac734cfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:25.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.240+0000 7fe514c35640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560ac733eea0 0x560ac734cfb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:34194/0 (socket says 172.21.15.32:34194) 2024-04-25T00:37:25.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.240+0000 7fe514c35640 1 -- 172.21.15.32:0/4258355178 learned_addr learned my addr 172.21.15.32:0/4258355178 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:25.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.245+0000 7fe515c37640 1 --2- 172.21.15.32:0/4258355178 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ac7349570 0x560ac734d550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:25.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.245+0000 7fe515436640 1 --2- 172.21.15.32:0/4258355178 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560ac733e580 0x560ac7351c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:25.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.246+0000 7fe514c35640 1 -- 172.21.15.32:0/4258355178 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ac7349570 msgr2=0x560ac734d550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:25.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.246+0000 7fe514c35640 1 --2- 172.21.15.32:0/4258355178 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ac7349570 0x560ac734d550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.246+0000 7fe514c35640 1 -- 172.21.15.32:0/4258355178 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560ac733e580 msgr2=0x560ac7351c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:25.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.246+0000 7fe514c35640 1 --2- 172.21.15.32:0/4258355178 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560ac733e580 0x560ac7351c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.246+0000 7fe514c35640 1 -- 172.21.15.32:0/4258355178 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x560ac70817d0 con 0x560ac733eea0 2024-04-25T00:37:25.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.247+0000 7fe514c35640 1 --2- 172.21.15.32:0/4258355178 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560ac733eea0 0x560ac734cfb0 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7fe508009270 tx=0x7fe508009780 comp rx=0 tx=0).ready entity=mon.2 client_cookie=37de3b335e8933fe server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:25.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.248+0000 7fe4f8ff9640 10 client.0 ms_handle_connect on v2:172.21.15.121:3300/0 2024-04-25T00:37:25.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.248+0000 7fe4f8ff9640 1 -- 172.21.15.32:0/4258355178 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe508011640 con 0x560ac733eea0 2024-04-25T00:37:25.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.248+0000 7fe51a3a6740 1 -- 172.21.15.32:0/4258355178 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe5000a4110 con 0x560ac733eea0 2024-04-25T00:37:25.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.248+0000 7fe4f8ff9640 1 -- 172.21.15.32:0/4258355178 <== mon.2 v2:172.21.15.121:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fe508004b20 con 0x560ac733eea0 2024-04-25T00:37:25.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.250+0000 7fe4f8ff9640 1 -- 172.21.15.32:0/4258355178 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fe508009500 con 0x560ac733eea0 2024-04-25T00:37:25.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.250+0000 7fe51a3a6740 10 client.25799 fetch_fsmap learned FSMap version 27 2024-04-25T00:37:25.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.250+0000 7fe51a3a6740 1 -- 172.21.15.32:0/4258355178 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x560ac73c0030 con 0x560ac733eea0 2024-04-25T00:37:25.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.251+0000 7fe4f8ff9640 1 -- 172.21.15.32:0/4258355178 <== mon.2 v2:172.21.15.121:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe5000a4110 con 0x560ac733eea0 2024-04-25T00:37:25.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.251+0000 7fe4f8ff9640 1 -- 172.21.15.32:0/4258355178 <== mon.2 v2:172.21.15.121:3300/0 5 ==== fsmap(e 27) ==== 8354+0+0 (secure 0 0 0) 0x7fe508019520 con 0x560ac733eea0 2024-04-25T00:37:25.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.251+0000 7fe51a3a6740 10 client.25799 fetch_fsmap finished waiting for FSMap version 27 2024-04-25T00:37:25.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.251+0000 7fe51a3a6740 10 client.25799 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:37:25.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.251+0000 7fe51a3a6740 20 client.25799 populate_metadata read hostname 'smithi032' 2024-04-25T00:37:25.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.251+0000 7fe51a3a6740 1 --2- 172.21.15.32:0/4258355178 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x560ac73c06f0 0x560ac73c2b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:25.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.251+0000 7fe51a3a6740 4 client.25799 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-25T00:37:25.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.251+0000 7fe51a3a6740 1 -- 172.21.15.32:0/4258355178 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "damage ls"}) -- 0x560ac72aaf00 con 0x560ac73c06f0 2024-04-25T00:37:25.258 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.258+0000 7fe515436640 1 --2- 172.21.15.32:0/4258355178 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x560ac73c06f0 0x560ac73c2b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:25.269 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.268+0000 7fe515436640 1 --2- 172.21.15.32:0/4258355178 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x560ac73c06f0 0x560ac73c2b10 crc :-1 s=READY pgs=90 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4a1d30795c3f823e server_cookie=2b20a2a8aaabedbe in_seq=0 out_seq=0 2024-04-25T00:37:25.269 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.269+0000 7fe4f8ff9640 10 client.25799 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:37:25.273 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.272+0000 7fe4f8ff9640 1 -- 172.21.15.32:0/4258355178 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x560ac72aaf00 con 0x560ac73c06f0 2024-04-25T00:37:25.273 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.273+0000 7fe4f8ff9640 10 client.25799 handle_command_reply: tid=0 2024-04-25T00:37:25.273 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.273+0000 7fe51a3a6740 1 client.25799 shutdown 2024-04-25T00:37:25.273 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.273+0000 7fe51a3a6740 1 -- 172.21.15.32:0/4258355178 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560ac733eea0 msgr2=0x560ac734cfb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:25.274 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.273+0000 7fe51a3a6740 1 --2- 172.21.15.32:0/4258355178 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560ac733eea0 0x560ac734cfb0 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7fe508009270 tx=0x7fe508009780 comp rx=0 tx=0).stop 2024-04-25T00:37:25.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.275+0000 7fe51a3a6740 1 -- 172.21.15.32:0/4258355178 shutdown_connections 2024-04-25T00:37:25.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.275+0000 7fe51a3a6740 1 --2- 172.21.15.32:0/4258355178 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x560ac73c06f0 0x560ac73c2b10 crc :-1 s=READY pgs=90 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.275+0000 7fe51a3a6740 1 --2- 172.21.15.32:0/4258355178 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ac7349570 0x560ac734d550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.275+0000 7fe51a3a6740 1 --2- 172.21.15.32:0/4258355178 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560ac733eea0 0x560ac734cfb0 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.275+0000 7fe51a3a6740 1 --2- 172.21.15.32:0/4258355178 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560ac733e580 0x560ac7351c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.276 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.275+0000 7fe51a3a6740 1 -- 172.21.15.32:0/4258355178 >> 172.21.15.32:0/4258355178 conn(0x560ac733a8a0 msgr2=0x560ac7339b40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:25.277 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.276+0000 7fe4f8ff9640 0 client.25799 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:37:25.277 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.276+0000 7fe51a3a6740 1 -- 172.21.15.32:0/4258355178 shutdown_connections 2024-04-25T00:37:25.277 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.276+0000 7fe51a3a6740 1 -- 172.21.15.32:0/4258355178 wait complete. 2024-04-25T00:37:25.280 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.280+0000 7fe51a3a6740 20 client.25799 trim_cache size 0 max 16384 2024-04-25T00:37:25.288 INFO:teuthology.orchestra.run.smithi032.stdout:[] 2024-04-25T00:37:25.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.302+0000 7fe51769a640 1 -- 172.21.15.32:0/2226252877 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe4fc0a2e80 msgr2=0x7fe4fc0a5340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:25.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.302+0000 7fe51769a640 1 --2- 172.21.15.32:0/2226252877 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe4fc0a2e80 0x7fe4fc0a5340 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7fe508009e40 tx=0x7fe50800f730 comp rx=0 tx=0).stop 2024-04-25T00:37:25.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.302+0000 7fe51769a640 1 -- 172.21.15.32:0/2226252877 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe51010dfc0 msgr2=0x7fe510118540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:25.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.302+0000 7fe51769a640 1 --2- 172.21.15.32:0/2226252877 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe51010dfc0 0x7fe510118540 secure :-1 s=READY pgs=628 cs=0 l=1 rev1=1 crypto rx=0x7fe50000d950 tx=0x7fe50000de10 comp rx=0 tx=0).stop 2024-04-25T00:37:25.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.302+0000 7fe51769a640 1 -- 172.21.15.32:0/2226252877 shutdown_connections 2024-04-25T00:37:25.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.302+0000 7fe51769a640 1 --2- 172.21.15.32:0/2226252877 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe4fc0a2e80 0x7fe4fc0a5340 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.302+0000 7fe51769a640 1 --2- 172.21.15.32:0/2226252877 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe51011af20 0x7fe510111b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.302+0000 7fe51769a640 1 --2- 172.21.15.32:0/2226252877 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe51010ff90 0x7fe510111610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.302+0000 7fe51769a640 1 --2- 172.21.15.32:0/2226252877 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe51010dfc0 0x7fe510118540 unknown :-1 s=CLOSED pgs=628 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:25.303 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.302+0000 7fe51769a640 1 -- 172.21.15.32:0/2226252877 >> 172.21.15.32:0/2226252877 conn(0x7fe51006c850 msgr2=0x7fe51011bc00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:25.309 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.309+0000 7fe51769a640 1 -- 172.21.15.32:0/2226252877 shutdown_connections 2024-04-25T00:37:25.309 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:25.309+0000 7fe51769a640 1 -- 172.21.15.32:0/2226252877 wait complete. 2024-04-25T00:37:25.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:25 smithi032 ceph-mon[29357]: pgmap v126: 129 pgs: 129 active+clean; 35 GiB data, 75 GiB used, 997 GiB / 1.0 TiB avail; 619 KiB/s rd, 537 MiB/s wr, 1.96k op/s 2024-04-25T00:37:25.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:25 smithi130 ceph-mon[32525]: pgmap v126: 129 pgs: 129 active+clean; 35 GiB data, 75 GiB used, 997 GiB / 1.0 TiB avail; 619 KiB/s rd, 537 MiB/s wr, 1.96k op/s 2024-04-25T00:37:26.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:25 smithi121 ceph-mon[32231]: pgmap v126: 129 pgs: 129 active+clean; 35 GiB data, 75 GiB used, 997 GiB / 1.0 TiB avail; 619 KiB/s rd, 537 MiB/s wr, 1.96k op/s 2024-04-25T00:37:26.365 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-25T00:37:26.365 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:37:26.365 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-25T00:37:26.639 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.637+0000 7f785f73b640 1 -- 172.21.15.32:0/4245482990 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7858109900 msgr2=0x7f7858109ce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:26.639 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.638+0000 7f785f73b640 1 --2- 172.21.15.32:0/4245482990 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7858109900 0x7f7858109ce0 secure :-1 s=READY pgs=630 cs=0 l=1 rev1=1 crypto rx=0x7f7848009930 tx=0x7f784802a420 comp rx=0 tx=0).stop 2024-04-25T00:37:26.640 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.638+0000 7f785f73b640 1 -- 172.21.15.32:0/4245482990 shutdown_connections 2024-04-25T00:37:26.640 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.638+0000 7f785f73b640 1 --2- 172.21.15.32:0/4245482990 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f78581042b0 0x7f78581107c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.640 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.638+0000 7f785f73b640 1 --2- 172.21.15.32:0/4245482990 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f78581038f0 0x7f7858103d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.640 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.638+0000 7f785f73b640 1 --2- 172.21.15.32:0/4245482990 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7858109900 0x7f7858109ce0 unknown :-1 s=CLOSED pgs=630 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.640 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.638+0000 7f785f73b640 1 -- 172.21.15.32:0/4245482990 >> 172.21.15.32:0/4245482990 conn(0x7f78580ff3c0 msgr2=0x7f78581017e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:26.640 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.638+0000 7f785f73b640 1 -- 172.21.15.32:0/4245482990 shutdown_connections 2024-04-25T00:37:26.640 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.638+0000 7f785f73b640 1 -- 172.21.15.32:0/4245482990 wait complete. 2024-04-25T00:37:26.641 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.640+0000 7f785f73b640 1 Processor -- start 2024-04-25T00:37:26.642 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.640+0000 7f785f73b640 1 -- start start 2024-04-25T00:37:26.643 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.641+0000 7f785f73b640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f78581038f0 0x7f785819a290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:26.643 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.641+0000 7f785f73b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f78581042b0 0x7f785819a7d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:26.643 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.641+0000 7f785f73b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7858109900 0x7f785819eb50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:26.643 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.641+0000 7f785f73b640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f7858113d10 con 0x7f7858109900 2024-04-25T00:37:26.643 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.641+0000 7f785f73b640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f7858113b90 con 0x7f78581042b0 2024-04-25T00:37:26.643 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.641+0000 7f785dcd8640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7858109900 0x7f785819eb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:26.644 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.642+0000 7f785dcd8640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7858109900 0x7f785819eb50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46760/0 (socket says 172.21.15.32:46760) 2024-04-25T00:37:26.644 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.642+0000 7f785dcd8640 1 -- 172.21.15.32:0/2669849982 learned_addr learned my addr 172.21.15.32:0/2669849982 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:26.644 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.641+0000 7f785f73b640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f7858113e90 con 0x7f78581038f0 2024-04-25T00:37:26.644 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.642+0000 7f785d4d7640 1 --2- 172.21.15.32:0/2669849982 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f78581038f0 0x7f785819a290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:26.644 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.642+0000 7f785ccd6640 1 --2- 172.21.15.32:0/2669849982 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f78581042b0 0x7f785819a7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:26.644 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.642+0000 7f785dcd8640 1 -- 172.21.15.32:0/2669849982 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f78581038f0 msgr2=0x7f785819a290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:26.645 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.642+0000 7f785dcd8640 1 --2- 172.21.15.32:0/2669849982 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f78581038f0 0x7f785819a290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.645 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.642+0000 7f785dcd8640 1 -- 172.21.15.32:0/2669849982 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f78581042b0 msgr2=0x7f785819a7d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:26.645 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.642+0000 7f785dcd8640 1 --2- 172.21.15.32:0/2669849982 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f78581042b0 0x7f785819a7d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.645 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.642+0000 7f785dcd8640 1 -- 172.21.15.32:0/2669849982 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f785819f280 con 0x7f7858109900 2024-04-25T00:37:26.645 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.643+0000 7f785dcd8640 1 --2- 172.21.15.32:0/2669849982 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7858109900 0x7f785819eb50 secure :-1 s=READY pgs=631 cs=0 l=1 rev1=1 crypto rx=0x7f785000eb10 tx=0x7f785000c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:26.645 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.643+0000 7f784e7fc640 1 -- 172.21.15.32:0/2669849982 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7850004b20 con 0x7f7858109900 2024-04-25T00:37:26.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.643+0000 7f784e7fc640 1 -- 172.21.15.32:0/2669849982 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7850004cc0 con 0x7f7858109900 2024-04-25T00:37:26.646 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.643+0000 7f784e7fc640 1 -- 172.21.15.32:0/2669849982 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7850010760 con 0x7f7858109900 2024-04-25T00:37:26.647 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.645+0000 7f785f73b640 1 -- 172.21.15.32:0/2669849982 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f785819f570 con 0x7f7858109900 2024-04-25T00:37:26.647 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.645+0000 7f785f73b640 1 -- 172.21.15.32:0/2669849982 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f78581a6e00 con 0x7f7858109900 2024-04-25T00:37:26.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.649+0000 7f784e7fc640 1 -- 172.21.15.32:0/2669849982 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f7850005000 con 0x7f7858109900 2024-04-25T00:37:26.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.650+0000 7f784e7fc640 1 --2- 172.21.15.32:0/2669849982 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f78440a2ed0 0x7f78440a5390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:26.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.650+0000 7f784e7fc640 1 -- 172.21.15.32:0/2669849982 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f78500cfe00 con 0x7f7858109900 2024-04-25T00:37:26.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.652+0000 7f7862446740 1 Processor -- start 2024-04-25T00:37:26.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.652+0000 7f7862446740 1 -- start start 2024-04-25T00:37:26.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.652+0000 7f7862446740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5627dc3f9980 0x5627dc3f7cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:26.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.653+0000 7f7862446740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5627dc3f8290 0x5627dc3f8710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:26.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.653+0000 7f7862446740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5627dc3fd010 0x5627dc3ff480 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:26.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.653+0000 7f7862446740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5627dc35c2c0 con 0x5627dc3f9980 2024-04-25T00:37:26.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.653+0000 7f7862446740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5627dc304910 con 0x5627dc3fd010 2024-04-25T00:37:26.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.653+0000 7f7862446740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5627dc358280 con 0x5627dc3f8290 2024-04-25T00:37:26.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.653+0000 7f785dcd8640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5627dc3fd010 0x5627dc3ff480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:26.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.653+0000 7f785dcd8640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5627dc3fd010 0x5627dc3ff480 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:34228/0 (socket says 172.21.15.32:34228) 2024-04-25T00:37:26.655 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.653+0000 7f785dcd8640 1 -- 172.21.15.32:0/729609249 learned_addr learned my addr 172.21.15.32:0/729609249 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:26.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.654+0000 7f785d4d7640 1 --2- 172.21.15.32:0/2669849982 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f78440a2ed0 0x7f78440a5390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:26.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.654+0000 7f785d4d7640 1 --2- 172.21.15.32:0/729609249 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5627dc3f9980 0x5627dc3f7cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:26.656 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.654+0000 7f785ccd6640 1 --2- 172.21.15.32:0/729609249 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5627dc3f8290 0x5627dc3f8710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:26.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.655+0000 7f785d4d7640 1 -- 172.21.15.32:0/729609249 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5627dc3f8290 msgr2=0x5627dc3f8710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:26.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.655+0000 7f785d4d7640 1 --2- 172.21.15.32:0/729609249 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5627dc3f8290 0x5627dc3f8710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.655+0000 7f785d4d7640 1 -- 172.21.15.32:0/729609249 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5627dc3fd010 msgr2=0x5627dc3ff480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:26.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.655+0000 7f785d4d7640 1 --2- 172.21.15.32:0/729609249 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5627dc3fd010 0x5627dc3ff480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.655+0000 7f785d4d7640 1 -- 172.21.15.32:0/729609249 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5627dc315fb0 con 0x5627dc3f9980 2024-04-25T00:37:26.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.657+0000 7f785d4d7640 1 --2- 172.21.15.32:0/2669849982 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f78440a2ed0 0x7f78440a5390 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f7848006010 tx=0x7f7848005e60 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:26.659 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.657+0000 7f785d4d7640 1 --2- 172.21.15.32:0/729609249 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5627dc3f9980 0x5627dc3f7cc0 secure :-1 s=READY pgs=632 cs=0 l=1 rev1=1 crypto rx=0x7f7848009900 tx=0x7f7848045000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e1515fd5c593ad78 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:26.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.662+0000 7f78437fe640 1 -- 172.21.15.32:0/729609249 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7848035040 con 0x5627dc3f9980 2024-04-25T00:37:26.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.662+0000 7f78437fe640 1 -- 172.21.15.32:0/729609249 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f784804d070 con 0x5627dc3f9980 2024-04-25T00:37:26.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.662+0000 7f78437fe640 1 -- 172.21.15.32:0/729609249 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7848004e00 con 0x5627dc3f9980 2024-04-25T00:37:26.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.663+0000 7f7862446740 1 -- 172.21.15.32:0/729609249 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5627dc3f9980 msgr2=0x5627dc3f7cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:26.665 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.663+0000 7f7862446740 1 --2- 172.21.15.32:0/729609249 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5627dc3f9980 0x5627dc3f7cc0 secure :-1 s=READY pgs=632 cs=0 l=1 rev1=1 crypto rx=0x7f7848009900 tx=0x7f7848045000 comp rx=0 tx=0).stop 2024-04-25T00:37:26.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.667+0000 7f7862446740 1 -- 172.21.15.32:0/729609249 shutdown_connections 2024-04-25T00:37:26.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.668+0000 7f7862446740 1 --2- 172.21.15.32:0/729609249 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5627dc3fd010 0x5627dc3ff480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.668+0000 7f7862446740 1 --2- 172.21.15.32:0/729609249 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5627dc3f8290 0x5627dc3f8710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.668+0000 7f7862446740 1 --2- 172.21.15.32:0/729609249 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5627dc3f9980 0x5627dc3f7cc0 unknown :-1 s=CLOSED pgs=632 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.668+0000 7f7862446740 1 -- 172.21.15.32:0/729609249 >> 172.21.15.32:0/729609249 conn(0x5627dc3c0950 msgr2=0x5627dc3f47f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:26.677 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.676+0000 7f7862446740 1 -- 172.21.15.32:0/729609249 shutdown_connections 2024-04-25T00:37:26.681 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.679+0000 7f7862446740 1 -- 172.21.15.32:0/729609249 wait complete. 2024-04-25T00:37:26.683 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.680+0000 7f7862446740 1 -- start start 2024-04-25T00:37:26.683 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.680+0000 7f7862446740 1 Processor -- start 2024-04-25T00:37:26.685 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.683+0000 7f7862446740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5627dc3f8290 0x5627dc408ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:26.685 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.683+0000 7f7862446740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5627dc3f9980 0x5627dc4032b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:26.685 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.683+0000 7f7862446740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5627dc3fd010 0x5627dc4037f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:26.685 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.683+0000 7f7862446740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x5627dc3ffe70 con 0x5627dc3f8290 2024-04-25T00:37:26.685 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.683+0000 7f7862446740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x5627dc317e90 con 0x5627dc3fd010 2024-04-25T00:37:26.685 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.683+0000 7f7862446740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x5627dc3ffff0 con 0x5627dc3f9980 2024-04-25T00:37:26.685 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.683+0000 7f7862446740 10 client.0 _ll_register_callbacks cb 0x5627dc353970 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:37:26.688 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.687+0000 7f785ccd6640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5627dc3f9980 0x5627dc4032b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:26.688 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.687+0000 7f785dcd8640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5627dc3fd010 0x5627dc4037f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:26.688 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.687+0000 7f785ccd6640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5627dc3f9980 0x5627dc4032b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:60864/0 (socket says 172.21.15.32:60864) 2024-04-25T00:37:26.689 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.687+0000 7f785ccd6640 1 -- 172.21.15.32:0/4111492766 learned_addr learned my addr 172.21.15.32:0/4111492766 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:26.697 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.696+0000 7f785d4d7640 1 --2- 172.21.15.32:0/4111492766 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5627dc3f8290 0x5627dc408ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:26.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.697+0000 7f785ccd6640 1 -- 172.21.15.32:0/4111492766 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5627dc3fd010 msgr2=0x5627dc4037f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:26.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.697+0000 7f785ccd6640 1 --2- 172.21.15.32:0/4111492766 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5627dc3fd010 0x5627dc4037f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.697+0000 7f785ccd6640 1 -- 172.21.15.32:0/4111492766 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5627dc3f8290 msgr2=0x5627dc408ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:26.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.697+0000 7f785ccd6640 1 --2- 172.21.15.32:0/4111492766 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5627dc3f8290 0x5627dc408ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.699 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.697+0000 7f785ccd6640 1 -- 172.21.15.32:0/4111492766 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5627dc1387e0 con 0x5627dc3f9980 2024-04-25T00:37:26.705 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.701+0000 7f785ccd6640 1 --2- 172.21.15.32:0/4111492766 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5627dc3f9980 0x5627dc4032b0 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f7854012700 tx=0x7f7854012b00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=31078e758a16040e server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:26.705 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.701+0000 7f7840ff9640 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:37:26.705 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.702+0000 7f7840ff9640 1 -- 172.21.15.32:0/4111492766 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7854004260 con 0x5627dc3f9980 2024-04-25T00:37:26.705 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.702+0000 7f7862446740 1 -- 172.21.15.32:0/4111492766 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f784804d070 con 0x5627dc3f9980 2024-04-25T00:37:26.705 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.702+0000 7f7840ff9640 1 -- 172.21.15.32:0/4111492766 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f7854004400 con 0x5627dc3f9980 2024-04-25T00:37:26.705 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.703+0000 7f7840ff9640 1 -- 172.21.15.32:0/4111492766 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f7854025050 con 0x5627dc3f9980 2024-04-25T00:37:26.706 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.703+0000 7f7862446740 10 client.25777 fetch_fsmap learned FSMap version 27 2024-04-25T00:37:26.706 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.703+0000 7f7862446740 1 -- 172.21.15.32:0/4111492766 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x5627dc198f70 con 0x5627dc3f9980 2024-04-25T00:37:26.709 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.705+0000 7f7840ff9640 1 -- 172.21.15.32:0/4111492766 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f784804d070 con 0x5627dc3f9980 2024-04-25T00:37:26.709 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.705+0000 7f7840ff9640 1 -- 172.21.15.32:0/4111492766 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 27) ==== 8354+0+0 (secure 0 0 0) 0x7f7854003b80 con 0x5627dc3f9980 2024-04-25T00:37:26.709 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.705+0000 7f7862446740 10 client.25777 fetch_fsmap finished waiting for FSMap version 27 2024-04-25T00:37:26.709 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.705+0000 7f7862446740 10 client.25777 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:37:26.709 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.705+0000 7f7862446740 20 client.25777 populate_metadata read hostname 'smithi032' 2024-04-25T00:37:26.709 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.705+0000 7f7862446740 1 --2- 172.21.15.32:0/4111492766 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5627dc424520 0x5627dc426940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:26.709 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.705+0000 7f7862446740 4 client.25777 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:37:26.710 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.705+0000 7f7862446740 1 -- 172.21.15.32:0/4111492766 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5627dc3305a0 con 0x5627dc424520 2024-04-25T00:37:26.713 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.712+0000 7f785d4d7640 1 --2- 172.21.15.32:0/4111492766 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5627dc424520 0x5627dc426940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:26.731 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.730+0000 7f785d4d7640 1 --2- 172.21.15.32:0/4111492766 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5627dc424520 0x5627dc426940 crc :-1 s=READY pgs=91 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=53cdf94ea01e98c0 server_cookie=73b4a08ee0615544 in_seq=0 out_seq=0 2024-04-25T00:37:26.731 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.730+0000 7f7840ff9640 10 client.25777 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:37:26.737 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.735+0000 7f7840ff9640 1 -- 172.21.15.32:0/4111492766 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x5627dc3305a0 con 0x5627dc424520 2024-04-25T00:37:26.737 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.735+0000 7f7840ff9640 10 client.25777 handle_command_reply: tid=0 2024-04-25T00:37:26.737 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.735+0000 7f7862446740 1 client.25777 shutdown 2024-04-25T00:37:26.741 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.736+0000 7f7862446740 1 -- 172.21.15.32:0/4111492766 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5627dc3f9980 msgr2=0x5627dc4032b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:26.741 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.736+0000 7f7862446740 1 --2- 172.21.15.32:0/4111492766 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5627dc3f9980 0x5627dc4032b0 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f7854012700 tx=0x7f7854012b00 comp rx=0 tx=0).stop 2024-04-25T00:37:26.745 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.743+0000 7f7862446740 1 -- 172.21.15.32:0/4111492766 shutdown_connections 2024-04-25T00:37:26.745 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.743+0000 7f7862446740 1 --2- 172.21.15.32:0/4111492766 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5627dc424520 0x5627dc426940 crc :-1 s=READY pgs=91 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.746 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.743+0000 7f7862446740 1 --2- 172.21.15.32:0/4111492766 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5627dc3fd010 0x5627dc4037f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.746 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.743+0000 7f7862446740 1 --2- 172.21.15.32:0/4111492766 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5627dc3f9980 0x5627dc4032b0 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.746 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.743+0000 7f7862446740 1 --2- 172.21.15.32:0/4111492766 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5627dc3f8290 0x5627dc408ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.746 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.743+0000 7f7862446740 1 -- 172.21.15.32:0/4111492766 >> 172.21.15.32:0/4111492766 conn(0x5627dc3c0950 msgr2=0x5627dc3f9f10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:26.746 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.744+0000 7f7840ff9640 0 client.25777 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:37:26.750 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.748+0000 7f7862446740 1 -- 172.21.15.32:0/4111492766 shutdown_connections 2024-04-25T00:37:26.751 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.750+0000 7f7862446740 1 -- 172.21.15.32:0/4111492766 wait complete. 2024-04-25T00:37:26.752 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.750+0000 7f7862446740 20 client.25777 trim_cache size 0 max 16384 2024-04-25T00:37:26.811 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.809+0000 7f7862446740 1 Processor -- start 2024-04-25T00:37:26.813 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.811+0000 7f7862446740 1 -- start start 2024-04-25T00:37:26.813 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.811+0000 7f7862446740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5627dc3c0950 0x5627dc42c230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:26.813 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.811+0000 7f7862446740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5627dc3f8290 0x5627dc42c770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:26.813 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.811+0000 7f7862446740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5627dc3f9980 0x5627dc401b80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:26.813 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.811+0000 7f7862446740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f78581af470 con 0x5627dc3f8290 2024-04-25T00:37:26.813 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.811+0000 7f7862446740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f78581acd40 con 0x5627dc3f9980 2024-04-25T00:37:26.813 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.811+0000 7f7862446740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f78581aab90 con 0x5627dc3c0950 2024-04-25T00:37:26.817 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.816+0000 7f785d4d7640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5627dc3c0950 0x5627dc42c230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:26.817 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.816+0000 7f785dcd8640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5627dc3f9980 0x5627dc401b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:26.818 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.816+0000 7f785d4d7640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5627dc3c0950 0x5627dc42c230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:60870/0 (socket says 172.21.15.32:60870) 2024-04-25T00:37:26.818 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.816+0000 7f785d4d7640 1 -- 172.21.15.32:0/1909461698 learned_addr learned my addr 172.21.15.32:0/1909461698 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:26.818 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.817+0000 7f785ccd6640 1 --2- 172.21.15.32:0/1909461698 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5627dc3f8290 0x5627dc42c770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:26.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.820+0000 7f785d4d7640 1 -- 172.21.15.32:0/1909461698 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5627dc3f9980 msgr2=0x5627dc401b80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:26.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.820+0000 7f785d4d7640 1 --2- 172.21.15.32:0/1909461698 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5627dc3f9980 0x5627dc401b80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.820+0000 7f785d4d7640 1 -- 172.21.15.32:0/1909461698 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5627dc3f8290 msgr2=0x5627dc42c770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:26.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.820+0000 7f785d4d7640 1 --2- 172.21.15.32:0/1909461698 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5627dc3f8290 0x5627dc42c770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.822 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.820+0000 7f785d4d7640 1 -- 172.21.15.32:0/1909461698 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5627dc030170 con 0x5627dc3c0950 2024-04-25T00:37:26.828 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.827+0000 7f785d4d7640 1 --2- 172.21.15.32:0/1909461698 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5627dc3c0950 0x5627dc42c230 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f7848009930 tx=0x7f784802a420 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5775df50995a1f14 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:26.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.833+0000 7f7842ffd640 1 -- 172.21.15.32:0/1909461698 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7848035040 con 0x5627dc3c0950 2024-04-25T00:37:26.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.833+0000 7f7842ffd640 1 -- 172.21.15.32:0/1909461698 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7848061070 con 0x5627dc3c0950 2024-04-25T00:37:26.835 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.834+0000 7f7862446740 1 -- 172.21.15.32:0/1909461698 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5627dc3c0950 msgr2=0x5627dc42c230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:26.836 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.834+0000 7f7862446740 1 --2- 172.21.15.32:0/1909461698 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5627dc3c0950 0x5627dc42c230 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f7848009930 tx=0x7f784802a420 comp rx=0 tx=0).stop 2024-04-25T00:37:26.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.840+0000 7f7862446740 1 -- 172.21.15.32:0/1909461698 shutdown_connections 2024-04-25T00:37:26.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.840+0000 7f7862446740 1 --2- 172.21.15.32:0/1909461698 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5627dc3f9980 0x5627dc401b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.840+0000 7f7862446740 1 --2- 172.21.15.32:0/1909461698 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5627dc3f8290 0x5627dc42c770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.840+0000 7f7862446740 1 --2- 172.21.15.32:0/1909461698 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5627dc3c0950 0x5627dc42c230 unknown :-1 s=CLOSED pgs=123 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.841 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.840+0000 7f7862446740 1 -- 172.21.15.32:0/1909461698 >> 172.21.15.32:0/1909461698 conn(0x5627dc403950 msgr2=0x5627dc3f4320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:26.844 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.843+0000 7f7862446740 1 -- 172.21.15.32:0/1909461698 shutdown_connections 2024-04-25T00:37:26.845 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.843+0000 7f7862446740 1 -- 172.21.15.32:0/1909461698 wait complete. 2024-04-25T00:37:26.845 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.844+0000 7f7862446740 1 -- start start 2024-04-25T00:37:26.845 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.844+0000 7f7862446740 1 Processor -- start 2024-04-25T00:37:26.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.845+0000 7f7862446740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5627dc3c0950 0x5627dc41ddf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:26.846 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.845+0000 7f7862446740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5627dc3f8290 0x5627dc41e330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:26.847 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.845+0000 7f7862446740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5627dc3f9980 0x5627dc41e870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:26.847 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.845+0000 7f7862446740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f78581ab200 con 0x5627dc3f8290 2024-04-25T00:37:26.847 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.845+0000 7f7862446740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f78581ac670 con 0x5627dc3c0950 2024-04-25T00:37:26.847 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.845+0000 7f7862446740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f78581ab8d0 con 0x5627dc3f9980 2024-04-25T00:37:26.847 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.845+0000 7f7862446740 10 client.0 _ll_register_callbacks cb 0x7f78540097a0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:37:26.848 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.847+0000 7f785ccd6640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5627dc3f8290 0x5627dc41e330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:26.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.847+0000 7f785ccd6640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5627dc3f8290 0x5627dc41e330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46796/0 (socket says 172.21.15.32:46796) 2024-04-25T00:37:26.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.847+0000 7f785ccd6640 1 -- 172.21.15.32:0/4252687667 learned_addr learned my addr 172.21.15.32:0/4252687667 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:26.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.847+0000 7f785dcd8640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5627dc3f9980 0x5627dc41e870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:26.849 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.847+0000 7f785d4d7640 1 --2- 172.21.15.32:0/4252687667 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5627dc3c0950 0x5627dc41ddf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:26.852 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.850+0000 7f785ccd6640 1 -- 172.21.15.32:0/4252687667 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5627dc3f9980 msgr2=0x5627dc41e870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:26.852 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.850+0000 7f785ccd6640 1 --2- 172.21.15.32:0/4252687667 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5627dc3f9980 0x5627dc41e870 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.852 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.850+0000 7f785ccd6640 1 -- 172.21.15.32:0/4252687667 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5627dc3c0950 msgr2=0x5627dc41ddf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:26.852 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.850+0000 7f785ccd6640 1 --2- 172.21.15.32:0/4252687667 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5627dc3c0950 0x5627dc41ddf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.852 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.850+0000 7f785ccd6640 1 -- 172.21.15.32:0/4252687667 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5627dc49ad10 con 0x5627dc3f8290 2024-04-25T00:37:26.858 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.857+0000 7f785ccd6640 1 --2- 172.21.15.32:0/4252687667 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5627dc3f8290 0x5627dc41e330 secure :-1 s=READY pgs=633 cs=0 l=1 rev1=1 crypto rx=0x5627dc404880 tx=0x7f785400f7e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=22ebc4ad3dcc0820 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:26.860 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.858+0000 7f7840ff9640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:37:26.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.860+0000 7f7840ff9640 1 -- 172.21.15.32:0/4252687667 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7854022040 con 0x5627dc3f8290 2024-04-25T00:37:26.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.860+0000 7f7862446740 1 -- 172.21.15.32:0/4252687667 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f7848061070 con 0x5627dc3f8290 2024-04-25T00:37:26.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.861+0000 7f7840ff9640 1 -- 172.21.15.32:0/4252687667 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f78540054f0 con 0x5627dc3f8290 2024-04-25T00:37:26.870 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.868+0000 7f7840ff9640 1 -- 172.21.15.32:0/4252687667 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f785403a050 con 0x5627dc3f8290 2024-04-25T00:37:26.870 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.868+0000 7f7862446740 10 client.16266 fetch_fsmap learned FSMap version 27 2024-04-25T00:37:26.870 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.868+0000 7f7862446740 1 -- 172.21.15.32:0/4252687667 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x5627dc49c050 con 0x5627dc3f8290 2024-04-25T00:37:26.876 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.874+0000 7f7840ff9640 1 -- 172.21.15.32:0/4252687667 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7848061070 con 0x5627dc3f8290 2024-04-25T00:37:26.876 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.874+0000 7f7840ff9640 1 -- 172.21.15.32:0/4252687667 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 27) ==== 8354+0+0 (secure 0 0 0) 0x7f785403d020 con 0x5627dc3f8290 2024-04-25T00:37:26.876 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.875+0000 7f7862446740 10 client.16266 fetch_fsmap finished waiting for FSMap version 27 2024-04-25T00:37:26.876 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.875+0000 7f7862446740 10 client.16266 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:37:26.876 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.875+0000 7f7862446740 20 client.16266 populate_metadata read hostname 'smithi032' 2024-04-25T00:37:26.876 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.875+0000 7f7862446740 1 --2- 172.21.15.32:0/4252687667 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5627dc3fd010 0x5627dc49e870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:26.876 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.875+0000 7f7862446740 4 client.16266 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-25T00:37:26.876 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.875+0000 7f7862446740 1 -- 172.21.15.32:0/4252687667 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) -- 0x5627dc369800 con 0x5627dc3fd010 2024-04-25T00:37:26.879 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.877+0000 7f785d4d7640 1 --2- 172.21.15.32:0/4252687667 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5627dc3fd010 0x5627dc49e870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:26.883 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.881+0000 7f785d4d7640 1 --2- 172.21.15.32:0/4252687667 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5627dc3fd010 0x5627dc49e870 crc :-1 s=READY pgs=92 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=eb052305906bb34c server_cookie=bffff2b27930513b in_seq=0 out_seq=0 2024-04-25T00:37:26.883 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.882+0000 7f7840ff9640 10 client.16266 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:37:26.890 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.888+0000 7f7840ff9640 1 -- 172.21.15.32:0/4252687667 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+110 (crc 0 0 0) 0x5627dc369800 con 0x5627dc3fd010 2024-04-25T00:37:26.890 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.888+0000 7f7840ff9640 10 client.16266 handle_command_reply: tid=0 2024-04-25T00:37:26.890 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.888+0000 7f7862446740 1 client.16266 shutdown 2024-04-25T00:37:26.897 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.895+0000 7f7862446740 1 -- 172.21.15.32:0/4252687667 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5627dc3f8290 msgr2=0x5627dc41e330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:26.897 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.895+0000 7f7862446740 1 --2- 172.21.15.32:0/4252687667 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5627dc3f8290 0x5627dc41e330 secure :-1 s=READY pgs=633 cs=0 l=1 rev1=1 crypto rx=0x5627dc404880 tx=0x7f785400f7e0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.896+0000 7f7862446740 1 -- 172.21.15.32:0/4252687667 shutdown_connections 2024-04-25T00:37:26.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.896+0000 7f7862446740 1 --2- 172.21.15.32:0/4252687667 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x5627dc3fd010 0x5627dc49e870 crc :-1 s=READY pgs=92 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.896+0000 7f7862446740 1 --2- 172.21.15.32:0/4252687667 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5627dc3f9980 0x5627dc41e870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.896+0000 7f7862446740 1 --2- 172.21.15.32:0/4252687667 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x5627dc3f8290 0x5627dc41e330 secure :-1 s=CLOSED pgs=633 cs=0 l=1 rev1=1 crypto rx=0x5627dc404880 tx=0x7f785400f7e0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.896+0000 7f7862446740 1 --2- 172.21.15.32:0/4252687667 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x5627dc3c0950 0x5627dc41ddf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.896+0000 7f7862446740 1 -- 172.21.15.32:0/4252687667 >> 172.21.15.32:0/4252687667 conn(0x5627dc403950 msgr2=0x5627dc401ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:26.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.897+0000 7f7840ff9640 0 client.16266 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:37:26.903 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.901+0000 7f7862446740 1 -- 172.21.15.32:0/4252687667 shutdown_connections 2024-04-25T00:37:26.903 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.901+0000 7f7862446740 1 -- 172.21.15.32:0/4252687667 wait complete. 2024-04-25T00:37:26.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.903+0000 7f7862446740 20 client.16266 trim_cache size 0 max 16384 2024-04-25T00:37:26.910 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:37:26.911 INFO:teuthology.orchestra.run.smithi032.stdout: "return_code": 0, 2024-04-25T00:37:26.911 INFO:teuthology.orchestra.run.smithi032.stdout: "scrub_tag": "24b1d566-1d7a-42ab-b621-3aa0f83549c2", 2024-04-25T00:37:26.911 INFO:teuthology.orchestra.run.smithi032.stdout: "mode": "asynchronous" 2024-04-25T00:37:26.911 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:37:26.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.911+0000 7f785f73b640 1 -- 172.21.15.32:0/2669849982 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f78440a2ed0 msgr2=0x7f78440a5390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:26.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.911+0000 7f785f73b640 1 --2- 172.21.15.32:0/2669849982 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f78440a2ed0 0x7f78440a5390 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f7848006010 tx=0x7f7848005e60 comp rx=0 tx=0).stop 2024-04-25T00:37:26.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.911+0000 7f785f73b640 1 -- 172.21.15.32:0/2669849982 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7858109900 msgr2=0x7f785819eb50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:26.913 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.911+0000 7f785f73b640 1 --2- 172.21.15.32:0/2669849982 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7858109900 0x7f785819eb50 secure :-1 s=READY pgs=631 cs=0 l=1 rev1=1 crypto rx=0x7f785000eb10 tx=0x7f785000c2d0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.913+0000 7f785f73b640 1 -- 172.21.15.32:0/2669849982 shutdown_connections 2024-04-25T00:37:26.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.913+0000 7f785f73b640 1 --2- 172.21.15.32:0/2669849982 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f78440a2ed0 0x7f78440a5390 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.913+0000 7f785f73b640 1 --2- 172.21.15.32:0/2669849982 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7858109900 0x7f785819eb50 unknown :-1 s=CLOSED pgs=631 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.913+0000 7f785f73b640 1 --2- 172.21.15.32:0/2669849982 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f78581042b0 0x7f785819a7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.913+0000 7f785f73b640 1 --2- 172.21.15.32:0/2669849982 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f78581038f0 0x7f785819a290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:26.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.913+0000 7f785f73b640 1 -- 172.21.15.32:0/2669849982 >> 172.21.15.32:0/2669849982 conn(0x7f78580ff3c0 msgr2=0x7f78580ffcd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:26.918 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.915+0000 7f785f73b640 1 -- 172.21.15.32:0/2669849982 shutdown_connections 2024-04-25T00:37:26.918 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:26.916+0000 7f785f73b640 1 -- 172.21.15.32:0/2669849982 wait complete. 2024-04-25T00:37:26.955 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-25T00:37:26.956 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-25T00:37:27.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.214+0000 7fe7ad7cf640 1 -- 172.21.15.32:0/2740209167 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe7a8108740 msgr2=0x7fe7a8108ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:27.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.214+0000 7fe7ad7cf640 1 --2- 172.21.15.32:0/2740209167 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe7a8108740 0x7fe7a8108ba0 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7fe79c009900 tx=0x7fe79c02a4d0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.222 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.220+0000 7fe7ad7cf640 1 -- 172.21.15.32:0/2740209167 shutdown_connections 2024-04-25T00:37:27.222 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.220+0000 7fe7ad7cf640 1 --2- 172.21.15.32:0/2740209167 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe7a81091c0 0x7fe7a811cf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.222 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.220+0000 7fe7ad7cf640 1 --2- 172.21.15.32:0/2740209167 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe7a8108740 0x7fe7a8108ba0 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.222 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.220+0000 7fe7ad7cf640 1 --2- 172.21.15.32:0/2740209167 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe7a810fde0 0x7fe7a81101c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.223 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.220+0000 7fe7ad7cf640 1 -- 172.21.15.32:0/2740209167 >> 172.21.15.32:0/2740209167 conn(0x7fe7a806c6c0 msgr2=0x7fe7a806cad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:27.223 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.221+0000 7fe7ad7cf640 1 -- 172.21.15.32:0/2740209167 shutdown_connections 2024-04-25T00:37:27.223 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.221+0000 7fe7ad7cf640 1 -- 172.21.15.32:0/2740209167 wait complete. 2024-04-25T00:37:27.223 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.222+0000 7fe7ad7cf640 1 Processor -- start 2024-04-25T00:37:27.224 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.222+0000 7fe7ad7cf640 1 -- start start 2024-04-25T00:37:27.224 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.223+0000 7fe7ad7cf640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe7a8108740 0x7fe7a81a6ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:27.224 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.223+0000 7fe7ad7cf640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe7a81091c0 0x7fe7a81a73e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:27.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.223+0000 7fe7ad7cf640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe7a810fde0 0x7fe7a81ab760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:27.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.223+0000 7fe7ad7cf640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fe7a8120900 con 0x7fe7a810fde0 2024-04-25T00:37:27.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.223+0000 7fe7ad7cf640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fe7a8120780 con 0x7fe7a81091c0 2024-04-25T00:37:27.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.223+0000 7fe7ad7cf640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fe7a8120a80 con 0x7fe7a8108740 2024-04-25T00:37:27.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.223+0000 7fe7a77fe640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe7a81091c0 0x7fe7a81a73e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:27.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.223+0000 7fe7a77fe640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe7a81091c0 0x7fe7a81a73e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:34274/0 (socket says 172.21.15.32:34274) 2024-04-25T00:37:27.225 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.223+0000 7fe7a77fe640 1 -- 172.21.15.32:0/4153972338 learned_addr learned my addr 172.21.15.32:0/4153972338 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:27.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.224+0000 7fe7a7fff640 1 --2- 172.21.15.32:0/4153972338 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe7a8108740 0x7fe7a81a6ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:27.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.224+0000 7fe7a77fe640 1 -- 172.21.15.32:0/4153972338 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe7a8108740 msgr2=0x7fe7a81a6ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:27.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.224+0000 7fe7a77fe640 1 --2- 172.21.15.32:0/4153972338 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe7a8108740 0x7fe7a81a6ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.224+0000 7fe7a77fe640 1 -- 172.21.15.32:0/4153972338 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe7a810fde0 msgr2=0x7fe7a81ab760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:27.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.224+0000 7fe7a77fe640 1 --2- 172.21.15.32:0/4153972338 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe7a810fde0 0x7fe7a81ab760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.226 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.224+0000 7fe7a77fe640 1 -- 172.21.15.32:0/4153972338 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe7a81abe90 con 0x7fe7a81091c0 2024-04-25T00:37:27.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.225+0000 7fe7a77fe640 1 --2- 172.21.15.32:0/4153972338 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe7a81091c0 0x7fe7a81a73e0 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7fe79c009a20 tx=0x7fe79c003cd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:27.227 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.226+0000 7fe7a57fa640 1 -- 172.21.15.32:0/4153972338 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe79c03f070 con 0x7fe7a81091c0 2024-04-25T00:37:27.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.226+0000 7fe7a57fa640 1 -- 172.21.15.32:0/4153972338 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe79c003e40 con 0x7fe7a81091c0 2024-04-25T00:37:27.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.226+0000 7fe7a57fa640 1 -- 172.21.15.32:0/4153972338 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe79c035030 con 0x7fe7a81091c0 2024-04-25T00:37:27.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.226+0000 7fe7ad7cf640 1 -- 172.21.15.32:0/4153972338 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe7a81ac120 con 0x7fe7a81091c0 2024-04-25T00:37:27.228 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.226+0000 7fe7ad7cf640 1 -- 172.21.15.32:0/4153972338 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe7a81b3a10 con 0x7fe7a81091c0 2024-04-25T00:37:27.232 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.231+0000 7fe7b1819740 1 Processor -- start 2024-04-25T00:37:27.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.233+0000 7fe7b1819740 1 -- start start 2024-04-25T00:37:27.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.233+0000 7fe7b1819740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560affd7e820 0x560affd7ec00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:27.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.233+0000 7fe7b1819740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560affd7f140 0x560affd892d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:27.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.233+0000 7fe7b1819740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560affd89810 0x560affd8bc00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:27.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.233+0000 7fe7b1819740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x560affce36b0 con 0x560affd7f140 2024-04-25T00:37:27.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.233+0000 7fe7b1819740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x560affc8b880 con 0x560affd89810 2024-04-25T00:37:27.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.233+0000 7fe7b1819740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x560affcdf670 con 0x560affd7e820 2024-04-25T00:37:27.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.234+0000 7fe7a57fa640 1 -- 172.21.15.32:0/4153972338 <== mon.2 v2:172.21.15.121:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7fe79c042070 con 0x7fe7a81091c0 2024-04-25T00:37:27.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.235+0000 7fe7a57fa640 1 --2- 172.21.15.32:0/4153972338 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe7900a2f00 0x7fe7900a53c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:27.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.235+0000 7fe7a7fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560affd7f140 0x560affd892d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:27.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.235+0000 7fe7a7fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560affd7f140 0x560affd892d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46810/0 (socket says 172.21.15.32:46810) 2024-04-25T00:37:27.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.235+0000 7fe7a7fff640 1 -- 172.21.15.32:0/483513504 learned_addr learned my addr 172.21.15.32:0/483513504 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:27.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.235+0000 7fe7acfce640 1 --2- 172.21.15.32:0/4153972338 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe7900a2f00 0x7fe7900a53c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:27.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.236+0000 7fe7a77fe640 1 --2- 172.21.15.32:0/483513504 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560affd89810 0x560affd8bc00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:27.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.236+0000 7fe7a57fa640 1 -- 172.21.15.32:0/4153972338 <== mon.2 v2:172.21.15.121:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fe79c0eff80 con 0x7fe7a81091c0 2024-04-25T00:37:27.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.237+0000 7fe7a7fff640 1 -- 172.21.15.32:0/483513504 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560affd7e820 msgr2=0x560affd7ec00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:27.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.237+0000 7fe7a7fff640 1 --2- 172.21.15.32:0/483513504 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560affd7e820 0x560affd7ec00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.237+0000 7fe7a7fff640 1 -- 172.21.15.32:0/483513504 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560affd89810 msgr2=0x560affd8bc00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:27.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.237+0000 7fe7a7fff640 1 --2- 172.21.15.32:0/483513504 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560affd89810 0x560affd8bc00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.237+0000 7fe7a7fff640 1 -- 172.21.15.32:0/483513504 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x560affac17d0 con 0x560affd7f140 2024-04-25T00:37:27.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.237+0000 7fe7acfce640 1 --2- 172.21.15.32:0/4153972338 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe7900a2f00 0x7fe7900a53c0 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7fe79800cb10 tx=0x7fe79800b250 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:27.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.238+0000 7fe7a7fff640 1 --2- 172.21.15.32:0/483513504 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560affd7f140 0x560affd892d0 secure :-1 s=READY pgs=634 cs=0 l=1 rev1=1 crypto rx=0x7fe7a000a5b0 tx=0x7fe7a00096d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=98819813e64a99c2 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:27.241 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.238+0000 7fe78e7fc640 1 -- 172.21.15.32:0/483513504 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe7a0018020 con 0x560affd7f140 2024-04-25T00:37:27.241 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.238+0000 7fe78e7fc640 1 -- 172.21.15.32:0/483513504 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe7a0014cf0 con 0x560affd7f140 2024-04-25T00:37:27.241 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.238+0000 7fe78e7fc640 1 -- 172.21.15.32:0/483513504 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe7a0013e10 con 0x560affd7f140 2024-04-25T00:37:27.241 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.240+0000 7fe7b1819740 1 -- 172.21.15.32:0/483513504 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560affd7f140 msgr2=0x560affd892d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:27.242 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.240+0000 7fe7b1819740 1 --2- 172.21.15.32:0/483513504 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560affd7f140 0x560affd892d0 secure :-1 s=READY pgs=634 cs=0 l=1 rev1=1 crypto rx=0x7fe7a000a5b0 tx=0x7fe7a00096d0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.242 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.240+0000 7fe7b1819740 1 -- 172.21.15.32:0/483513504 shutdown_connections 2024-04-25T00:37:27.242 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.240+0000 7fe7b1819740 1 --2- 172.21.15.32:0/483513504 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560affd89810 0x560affd8bc00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.242 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.240+0000 7fe7b1819740 1 --2- 172.21.15.32:0/483513504 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560affd7f140 0x560affd892d0 unknown :-1 s=CLOSED pgs=634 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.242 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.240+0000 7fe7b1819740 1 --2- 172.21.15.32:0/483513504 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560affd7e820 0x560affd7ec00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.242 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.240+0000 7fe7b1819740 1 -- 172.21.15.32:0/483513504 >> 172.21.15.32:0/483513504 conn(0x560affd7ab40 msgr2=0x560affd7af20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:27.242 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.240+0000 7fe7b1819740 1 -- 172.21.15.32:0/483513504 shutdown_connections 2024-04-25T00:37:27.243 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.240+0000 7fe7b1819740 1 -- 172.21.15.32:0/483513504 wait complete. 2024-04-25T00:37:27.243 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.241+0000 7fe7b1819740 1 -- start start 2024-04-25T00:37:27.243 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.241+0000 7fe7b1819740 1 Processor -- start 2024-04-25T00:37:27.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.242+0000 7fe7b1819740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560affd7e820 0x560affd867c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:27.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.242+0000 7fe7b1819740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560affd7f140 0x560affd86d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:27.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.242+0000 7fe7b1819740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560affd89810 0x560affd87240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:27.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.242+0000 7fe7b1819740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x560affd8c600 con 0x560affd7e820 2024-04-25T00:37:27.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.242+0000 7fe7b1819740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x560affc9ef50 con 0x560affd7f140 2024-04-25T00:37:27.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.242+0000 7fe7b1819740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x560affd8c780 con 0x560affd89810 2024-04-25T00:37:27.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.242+0000 7fe7b1819740 10 client.0 _ll_register_callbacks cb 0x560affcda000 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:37:27.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.244+0000 7fe7a77fe640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560affd7f140 0x560affd86d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:27.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.244+0000 7fe7a7fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560affd7e820 0x560affd867c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:27.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.244+0000 7fe7a7fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560affd7e820 0x560affd867c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:46814/0 (socket says 172.21.15.32:46814) 2024-04-25T00:37:27.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.244+0000 7fe7a7fff640 1 -- 172.21.15.32:0/2832136887 learned_addr learned my addr 172.21.15.32:0/2832136887 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:27.246 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.244+0000 7fe7acfce640 1 --2- 172.21.15.32:0/2832136887 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560affd89810 0x560affd87240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:27.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.245+0000 7fe7a7fff640 1 -- 172.21.15.32:0/2832136887 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560affd89810 msgr2=0x560affd87240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:27.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.245+0000 7fe7a7fff640 1 --2- 172.21.15.32:0/2832136887 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560affd89810 0x560affd87240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.245+0000 7fe7a7fff640 1 -- 172.21.15.32:0/2832136887 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560affd7f140 msgr2=0x560affd86d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:27.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.245+0000 7fe7a7fff640 1 --2- 172.21.15.32:0/2832136887 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560affd7f140 0x560affd86d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.245+0000 7fe7a7fff640 1 -- 172.21.15.32:0/2832136887 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x560aff9f73f0 con 0x560affd7e820 2024-04-25T00:37:27.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.246+0000 7fe7a7fff640 1 --2- 172.21.15.32:0/2832136887 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560affd7e820 0x560affd867c0 secure :-1 s=READY pgs=635 cs=0 l=1 rev1=1 crypto rx=0x7fe7a0013900 tx=0x7fe7a0004ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4b4595f25ddb8f0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:27.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.246+0000 7fe7877fe640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:37:27.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.247+0000 7fe7877fe640 1 -- 172.21.15.32:0/2832136887 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe7a0023070 con 0x560affd7e820 2024-04-25T00:37:27.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.247+0000 7fe7b1819740 1 -- 172.21.15.32:0/2832136887 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe7a0014cf0 con 0x560affd7e820 2024-04-25T00:37:27.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.247+0000 7fe7877fe640 1 -- 172.21.15.32:0/2832136887 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fe7a0014610 con 0x560affd7e820 2024-04-25T00:37:27.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.248+0000 7fe7877fe640 1 -- 172.21.15.32:0/2832136887 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fe7a0014830 con 0x560affd7e820 2024-04-25T00:37:27.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.248+0000 7fe7b1819740 10 client.16272 fetch_fsmap learned FSMap version 27 2024-04-25T00:37:27.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.248+0000 7fe7b1819740 1 -- 172.21.15.32:0/2832136887 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x560aff9b70f0 con 0x560affd7e820 2024-04-25T00:37:27.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.249+0000 7fe7877fe640 1 -- 172.21.15.32:0/2832136887 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe7a0014cf0 con 0x560affd7e820 2024-04-25T00:37:27.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.249+0000 7fe7877fe640 1 -- 172.21.15.32:0/2832136887 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 27) ==== 8354+0+0 (secure 0 0 0) 0x7fe7a0027400 con 0x560affd7e820 2024-04-25T00:37:27.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.249+0000 7fe7b1819740 10 client.16272 fetch_fsmap finished waiting for FSMap version 27 2024-04-25T00:37:27.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.249+0000 7fe7b1819740 10 client.16272 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:37:27.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.249+0000 7fe7b1819740 20 client.16272 populate_metadata read hostname 'smithi032' 2024-04-25T00:37:27.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.249+0000 7fe7b1819740 1 --2- 172.21.15.32:0/2832136887 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x560affdaaf40 0x560affdad3c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:27.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.249+0000 7fe7b1819740 4 client.16272 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:37:27.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.249+0000 7fe7b1819740 1 -- 172.21.15.32:0/2832136887 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x560affcf0bf0 con 0x560affdaaf40 2024-04-25T00:37:27.252 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.251+0000 7fe7a77fe640 1 --2- 172.21.15.32:0/2832136887 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x560affdaaf40 0x560affdad3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:27.255 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.253+0000 7fe7a77fe640 1 --2- 172.21.15.32:0/2832136887 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x560affdaaf40 0x560affdad3c0 crc :-1 s=READY pgs=93 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=af7871a5dafb8d2a server_cookie=b89053e934ab8881 in_seq=0 out_seq=0 2024-04-25T00:37:27.255 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.254+0000 7fe7877fe640 10 client.16272 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:37:27.258 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.256+0000 7fe7877fe640 1 -- 172.21.15.32:0/2832136887 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x560affcf0bf0 con 0x560affdaaf40 2024-04-25T00:37:27.258 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.256+0000 7fe7877fe640 10 client.16272 handle_command_reply: tid=0 2024-04-25T00:37:27.258 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.256+0000 7fe7b1819740 1 client.16272 shutdown 2024-04-25T00:37:27.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.263+0000 7fe7b1819740 1 -- 172.21.15.32:0/2832136887 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560affd7e820 msgr2=0x560affd867c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:27.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.263+0000 7fe7b1819740 1 --2- 172.21.15.32:0/2832136887 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560affd7e820 0x560affd867c0 secure :-1 s=READY pgs=635 cs=0 l=1 rev1=1 crypto rx=0x7fe7a0013900 tx=0x7fe7a0004ad0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.263+0000 7fe7b1819740 1 -- 172.21.15.32:0/2832136887 shutdown_connections 2024-04-25T00:37:27.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.263+0000 7fe7b1819740 1 --2- 172.21.15.32:0/2832136887 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x560affdaaf40 0x560affdad3c0 crc :-1 s=READY pgs=93 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.263+0000 7fe7b1819740 1 --2- 172.21.15.32:0/2832136887 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560affd89810 0x560affd87240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.263+0000 7fe7b1819740 1 --2- 172.21.15.32:0/2832136887 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560affd7f140 0x560affd86d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.263+0000 7fe7b1819740 1 --2- 172.21.15.32:0/2832136887 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560affd7e820 0x560affd867c0 unknown :-1 s=CLOSED pgs=635 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.263+0000 7fe7b1819740 1 -- 172.21.15.32:0/2832136887 >> 172.21.15.32:0/2832136887 conn(0x560affd7ab40 msgr2=0x560affd80840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:27.265 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.263+0000 7fe7877fe640 0 client.16272 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:37:27.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.264+0000 7fe7b1819740 1 -- 172.21.15.32:0/2832136887 shutdown_connections 2024-04-25T00:37:27.267 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.265+0000 7fe7b1819740 1 -- 172.21.15.32:0/2832136887 wait complete. 2024-04-25T00:37:27.279 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.278+0000 7fe7b1819740 20 client.16272 trim_cache size 0 max 16384 2024-04-25T00:37:27.339 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.337+0000 7fe7b1819740 1 Processor -- start 2024-04-25T00:37:27.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.342+0000 7fe7b1819740 1 -- start start 2024-04-25T00:37:27.347 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.343+0000 7fe7b1819740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560affd7e820 0x560affd95520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:27.347 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.343+0000 7fe7b1819740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560affd7f140 0x560affd85090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:27.347 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.343+0000 7fe7b1819740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560affd89810 0x560affd80f80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:27.347 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.343+0000 7fe7b1819740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fe794012950 con 0x560affd7e820 2024-04-25T00:37:27.347 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.343+0000 7fe7b1819740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fe794010220 con 0x560affd89810 2024-04-25T00:37:27.347 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.343+0000 7fe7b1819740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fe79400e070 con 0x560affd7f140 2024-04-25T00:37:27.347 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.344+0000 7fe7a77fe640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560affd7f140 0x560affd85090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:27.347 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.344+0000 7fe7a77fe640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560affd7f140 0x560affd85090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:60920/0 (socket says 172.21.15.32:60920) 2024-04-25T00:37:27.347 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.344+0000 7fe7a77fe640 1 -- 172.21.15.32:0/3201603425 learned_addr learned my addr 172.21.15.32:0/3201603425 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:27.347 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.345+0000 7fe7a77fe640 1 -- 172.21.15.32:0/3201603425 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560affd89810 msgr2=0x560affd80f80 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-25T00:37:27.347 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.345+0000 7fe7a77fe640 1 --2- 172.21.15.32:0/3201603425 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560affd89810 0x560affd80f80 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.347 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.345+0000 7fe7a77fe640 1 -- 172.21.15.32:0/3201603425 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560affd7e820 msgr2=0x560affd95520 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-25T00:37:27.348 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.345+0000 7fe7a77fe640 1 --2- 172.21.15.32:0/3201603425 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560affd7e820 0x560affd95520 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.348 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.345+0000 7fe7a77fe640 1 -- 172.21.15.32:0/3201603425 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x560affd814c0 con 0x560affd7f140 2024-04-25T00:37:27.351 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.349+0000 7fe7a77fe640 1 --2- 172.21.15.32:0/3201603425 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560affd7f140 0x560affd85090 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7fe79c0bc020 tx=0x7fe79c0c0040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=55b0a8362b25e1c0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:27.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.354+0000 7fe78dffb640 1 -- 172.21.15.32:0/3201603425 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe79c0bc090 con 0x560affd7f140 2024-04-25T00:37:27.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.354+0000 7fe78dffb640 1 -- 172.21.15.32:0/3201603425 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe79c0c0440 con 0x560affd7f140 2024-04-25T00:37:27.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.354+0000 7fe78dffb640 1 -- 172.21.15.32:0/3201603425 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe79c0b8700 con 0x560affd7f140 2024-04-25T00:37:27.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.356+0000 7fe7b1819740 1 -- 172.21.15.32:0/3201603425 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560affd7f140 msgr2=0x560affd85090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:27.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.356+0000 7fe7b1819740 1 --2- 172.21.15.32:0/3201603425 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560affd7f140 0x560affd85090 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7fe79c0bc020 tx=0x7fe79c0c0040 comp rx=0 tx=0).stop 2024-04-25T00:37:27.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.359+0000 7fe7b1819740 1 -- 172.21.15.32:0/3201603425 shutdown_connections 2024-04-25T00:37:27.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.359+0000 7fe7b1819740 1 --2- 172.21.15.32:0/3201603425 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560affd89810 0x560affd80f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.359+0000 7fe7b1819740 1 --2- 172.21.15.32:0/3201603425 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560affd7f140 0x560affd85090 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.359+0000 7fe7b1819740 1 --2- 172.21.15.32:0/3201603425 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560affd7e820 0x560affd95520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.359+0000 7fe7b1819740 1 -- 172.21.15.32:0/3201603425 >> 172.21.15.32:0/3201603425 conn(0x560affd7ab40 msgr2=0x560affd79d00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:27.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.364+0000 7fe7b1819740 1 -- 172.21.15.32:0/3201603425 shutdown_connections 2024-04-25T00:37:27.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.364+0000 7fe7b1819740 1 -- 172.21.15.32:0/3201603425 wait complete. 2024-04-25T00:37:27.366 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.365+0000 7fe7b1819740 1 -- start start 2024-04-25T00:37:27.367 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.365+0000 7fe7b1819740 1 Processor -- start 2024-04-25T00:37:27.368 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.366+0000 7fe7b1819740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560affd7e820 0x560affd83570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:27.369 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.366+0000 7fe7b1819740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560affd7f140 0x560affd83ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:27.369 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.367+0000 7fe7b1819740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560affd89810 0x560affd83ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:27.369 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.367+0000 7fe7b1819740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fe79400e6e0 con 0x560affd89810 2024-04-25T00:37:27.369 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.367+0000 7fe7a7fff640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560affd7e820 0x560affd83570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:27.369 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.367+0000 7fe7a7fff640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560affd7e820 0x560affd83570 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:34302/0 (socket says 172.21.15.32:34302) 2024-04-25T00:37:27.369 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.367+0000 7fe7a7fff640 1 -- 172.21.15.32:0/1952862333 learned_addr learned my addr 172.21.15.32:0/1952862333 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:27.369 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.367+0000 7fe7b1819740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fe79400fb50 con 0x560affd7e820 2024-04-25T00:37:27.369 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.367+0000 7fe7b1819740 1 -- 172.21.15.32:0/1952862333 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fe79400edb0 con 0x560affd7f140 2024-04-25T00:37:27.369 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.367+0000 7fe7b1819740 10 client.0 _ll_register_callbacks cb 0x7fe79c009560 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:37:27.374 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.368+0000 7fe7a77fe640 1 --2- 172.21.15.32:0/1952862333 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560affd7f140 0x560affd83ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:27.374 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.369+0000 7fe7a77fe640 1 -- 172.21.15.32:0/1952862333 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560affd7e820 msgr2=0x560affd83570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:27.374 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.369+0000 7fe7a77fe640 1 --2- 172.21.15.32:0/1952862333 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560affd7e820 0x560affd83570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.374 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.369+0000 7fe7a77fe640 1 -- 172.21.15.32:0/1952862333 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560affd89810 msgr2=0x560affd83ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:27.374 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.369+0000 7fe7a77fe640 1 --2- 172.21.15.32:0/1952862333 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560affd89810 0x560affd83ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.375 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.369+0000 7fe7a77fe640 1 -- 172.21.15.32:0/1952862333 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x560affe105d0 con 0x560affd7f140 2024-04-25T00:37:27.375 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.370+0000 7fe7a7fff640 1 --2- 172.21.15.32:0/1952862333 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560affd7e820 0x560affd83570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:37:27.375 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.370+0000 7fe7a77fe640 1 --2- 172.21.15.32:0/1952862333 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560affd7f140 0x560affd83ab0 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7fe79c0dc300 tx=0x7fe79c02a4d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d359c4a887987b7d server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:27.377 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.373+0000 7fe7877fe640 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:37:27.377 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.373+0000 7fe7877fe640 1 -- 172.21.15.32:0/1952862333 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe79c0ba390 con 0x560affd7f140 2024-04-25T00:37:27.378 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.373+0000 7fe7877fe640 1 -- 172.21.15.32:0/1952862333 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fe79c0b8700 con 0x560affd7f140 2024-04-25T00:37:27.378 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.376+0000 7fe7b1819740 1 -- 172.21.15.32:0/1952862333 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe79c0c0440 con 0x560affd7f140 2024-04-25T00:37:27.381 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.378+0000 7fe7877fe640 1 -- 172.21.15.32:0/1952862333 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fe79c0ed540 con 0x560affd7f140 2024-04-25T00:37:27.381 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.379+0000 7fe7b1819740 10 client.25795 fetch_fsmap learned FSMap version 27 2024-04-25T00:37:27.381 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.379+0000 7fe7b1819740 1 -- 172.21.15.32:0/1952862333 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x560affe118e0 con 0x560affd7f140 2024-04-25T00:37:27.388 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.383+0000 7fe7877fe640 1 -- 172.21.15.32:0/1952862333 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe79c0c0440 con 0x560affd7f140 2024-04-25T00:37:27.388 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.383+0000 7fe7877fe640 1 -- 172.21.15.32:0/1952862333 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 27) ==== 8354+0+0 (secure 0 0 0) 0x7fe79c0d7070 con 0x560affd7f140 2024-04-25T00:37:27.388 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.385+0000 7fe7b1819740 10 client.25795 fetch_fsmap finished waiting for FSMap version 27 2024-04-25T00:37:27.388 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.386+0000 7fe7b1819740 10 client.25795 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:37:27.388 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.386+0000 7fe7b1819740 20 client.25795 populate_metadata read hostname 'smithi032' 2024-04-25T00:37:27.389 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.386+0000 7fe7b1819740 1 --2- 172.21.15.32:0/1952862333 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x560affe11d90 0x560affe14180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:27.389 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.386+0000 7fe7b1819740 4 client.25795 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-25T00:37:27.389 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.386+0000 7fe7b1819740 1 -- 172.21.15.32:0/1952862333 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "scrub status"}) -- 0x560affceaf40 con 0x560affe11d90 2024-04-25T00:37:27.397 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.390+0000 7fe7a7fff640 1 --2- 172.21.15.32:0/1952862333 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x560affe11d90 0x560affe14180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:27.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.408+0000 7fe7a7fff640 1 --2- 172.21.15.32:0/1952862333 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x560affe11d90 0x560affe14180 crc :-1 s=READY pgs=94 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4b41ca3feb848a4b server_cookie=4f64cd6f88481e4 in_seq=0 out_seq=0 2024-04-25T00:37:27.410 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.409+0000 7fe7877fe640 10 client.25795 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:37:27.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.411+0000 7fe7877fe640 1 -- 172.21.15.32:0/1952862333 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+265 (crc 0 0 0) 0x560affceaf40 con 0x560affe11d90 2024-04-25T00:37:27.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.411+0000 7fe7877fe640 10 client.25795 handle_command_reply: tid=0 2024-04-25T00:37:27.413 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.412+0000 7fe7b1819740 1 client.25795 shutdown 2024-04-25T00:37:27.420 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.418+0000 7fe7b1819740 1 -- 172.21.15.32:0/1952862333 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560affd7f140 msgr2=0x560affd83ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:27.420 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.418+0000 7fe7b1819740 1 --2- 172.21.15.32:0/1952862333 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560affd7f140 0x560affd83ab0 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7fe79c0dc300 tx=0x7fe79c02a4d0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.424 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.420+0000 7fe7b1819740 1 -- 172.21.15.32:0/1952862333 shutdown_connections 2024-04-25T00:37:27.424 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.420+0000 7fe7b1819740 1 --2- 172.21.15.32:0/1952862333 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x560affe11d90 0x560affe14180 crc :-1 s=READY pgs=94 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.424 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.420+0000 7fe7b1819740 1 --2- 172.21.15.32:0/1952862333 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x560affd89810 0x560affd83ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.425 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.420+0000 7fe7b1819740 1 --2- 172.21.15.32:0/1952862333 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560affd7f140 0x560affd83ab0 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.425 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.420+0000 7fe7b1819740 1 --2- 172.21.15.32:0/1952862333 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x560affd7e820 0x560affd83570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.425 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.420+0000 7fe7b1819740 1 -- 172.21.15.32:0/1952862333 >> 172.21.15.32:0/1952862333 conn(0x560affd7ab40 msgr2=0x560affd86760 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:27.425 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.423+0000 7fe7877fe640 0 client.25795 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:37:27.425 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.423+0000 7fe7b1819740 1 -- 172.21.15.32:0/1952862333 shutdown_connections 2024-04-25T00:37:27.429 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:37:27.429 INFO:teuthology.orchestra.run.smithi032.stdout: "status": "scrub active (1 inodes in the stack)", 2024-04-25T00:37:27.429 INFO:teuthology.orchestra.run.smithi032.stdout: "scrubs": { 2024-04-25T00:37:27.429 INFO:teuthology.orchestra.run.smithi032.stdout: "24b1d566-1d7a-42ab-b621-3aa0f83549c2": { 2024-04-25T00:37:27.429 INFO:teuthology.orchestra.run.smithi032.stdout: "path": "/", 2024-04-25T00:37:27.429 INFO:teuthology.orchestra.run.smithi032.stdout: "tag": "24b1d566-1d7a-42ab-b621-3aa0f83549c2", 2024-04-25T00:37:27.429 INFO:teuthology.orchestra.run.smithi032.stdout: "options": "recursive,force" 2024-04-25T00:37:27.429 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:37:27.429 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:37:27.429 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:37:27.430 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.425+0000 7fe7b1819740 1 -- 172.21.15.32:0/1952862333 wait complete. 2024-04-25T00:37:27.430 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.425+0000 7fe7b1819740 20 client.25795 trim_cache size 0 max 16384 2024-04-25T00:37:27.434 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.429+0000 7fe7ad7cf640 1 -- 172.21.15.32:0/4153972338 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe7900a2f00 msgr2=0x7fe7900a53c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:27.434 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.429+0000 7fe7ad7cf640 1 --2- 172.21.15.32:0/4153972338 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe7900a2f00 0x7fe7900a53c0 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7fe79800cb10 tx=0x7fe79800b250 comp rx=0 tx=0).stop 2024-04-25T00:37:27.434 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.429+0000 7fe7ad7cf640 1 -- 172.21.15.32:0/4153972338 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe7a81091c0 msgr2=0x7fe7a81a73e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:27.434 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.429+0000 7fe7ad7cf640 1 --2- 172.21.15.32:0/4153972338 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe7a81091c0 0x7fe7a81a73e0 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7fe79c009a20 tx=0x7fe79c003cd0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.434 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.432+0000 7fe7ad7cf640 1 -- 172.21.15.32:0/4153972338 shutdown_connections 2024-04-25T00:37:27.434 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.432+0000 7fe7ad7cf640 1 --2- 172.21.15.32:0/4153972338 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe7900a2f00 0x7fe7900a53c0 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.434 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.432+0000 7fe7ad7cf640 1 --2- 172.21.15.32:0/4153972338 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe7a810fde0 0x7fe7a81ab760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.434 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.432+0000 7fe7ad7cf640 1 --2- 172.21.15.32:0/4153972338 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe7a81091c0 0x7fe7a81a73e0 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.434 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.432+0000 7fe7ad7cf640 1 --2- 172.21.15.32:0/4153972338 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe7a8108740 0x7fe7a81a6ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:27.434 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.432+0000 7fe7ad7cf640 1 -- 172.21.15.32:0/4153972338 >> 172.21.15.32:0/4153972338 conn(0x7fe7a806c6c0 msgr2=0x7fe7a80703c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:27.438 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.435+0000 7fe7ad7cf640 1 -- 172.21.15.32:0/4153972338 shutdown_connections 2024-04-25T00:37:27.441 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:27.440+0000 7fe7ad7cf640 1 -- 172.21.15.32:0/4153972338 wait complete. 2024-04-25T00:37:27.481 INFO:tasks.cephfs.filesystem:scrub status for tag:24b1d566-1d7a-42ab-b621-3aa0f83549c2 - {'path': '/', 'tag': '24b1d566-1d7a-42ab-b621-3aa0f83549c2', 'options': 'recursive,force'} 2024-04-25T00:37:27.752 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:27 smithi032 ceph-mon[29357]: pgmap v127: 129 pgs: 129 active+clean; 37 GiB data, 78 GiB used, 995 GiB / 1.0 TiB avail; 781 KiB/s rd, 651 MiB/s wr, 1.80k op/s 2024-04-25T00:37:27.752 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:27 smithi032 ceph-mon[29357]: scrub queued for path: / 2024-04-25T00:37:27.752 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:27 smithi032 ceph-mon[29357]: scrub summary: idle+waiting paths [/] 2024-04-25T00:37:27.753 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:27 smithi032 ceph-mon[29357]: scrub summary: active paths [/] 2024-04-25T00:37:27.753 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:27 smithi032 ceph-mon[29357]: scrub summary: active paths [/] 2024-04-25T00:37:27.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:27 smithi130 ceph-mon[32525]: pgmap v127: 129 pgs: 129 active+clean; 37 GiB data, 78 GiB used, 995 GiB / 1.0 TiB avail; 781 KiB/s rd, 651 MiB/s wr, 1.80k op/s 2024-04-25T00:37:27.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:27 smithi130 ceph-mon[32525]: scrub queued for path: / 2024-04-25T00:37:27.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:27 smithi130 ceph-mon[32525]: scrub summary: idle+waiting paths [/] 2024-04-25T00:37:27.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:27 smithi130 ceph-mon[32525]: scrub summary: active paths [/] 2024-04-25T00:37:27.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:27 smithi130 ceph-mon[32525]: scrub summary: active paths [/] 2024-04-25T00:37:28.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:27 smithi121 ceph-mon[32231]: pgmap v127: 129 pgs: 129 active+clean; 37 GiB data, 78 GiB used, 995 GiB / 1.0 TiB avail; 781 KiB/s rd, 651 MiB/s wr, 1.80k op/s 2024-04-25T00:37:28.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:27 smithi121 ceph-mon[32231]: scrub queued for path: / 2024-04-25T00:37:28.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:27 smithi121 ceph-mon[32231]: scrub summary: idle+waiting paths [/] 2024-04-25T00:37:28.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:27 smithi121 ceph-mon[32231]: scrub summary: active paths [/] 2024-04-25T00:37:28.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:27 smithi121 ceph-mon[32231]: scrub summary: active paths [/] 2024-04-25T00:37:28.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:28 smithi130 ceph-mon[32525]: scrub summary: active paths [/] 2024-04-25T00:37:28.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:28 smithi130 ceph-mon[32525]: scrub summary: idle+waiting paths [/] 2024-04-25T00:37:28.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:28 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:37:28.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:28 smithi032 ceph-mon[29357]: scrub summary: active paths [/] 2024-04-25T00:37:28.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:28 smithi032 ceph-mon[29357]: scrub summary: idle+waiting paths [/] 2024-04-25T00:37:28.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:28 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:37:29.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:28 smithi121 ceph-mon[32231]: scrub summary: active paths [/] 2024-04-25T00:37:29.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:28 smithi121 ceph-mon[32231]: scrub summary: idle+waiting paths [/] 2024-04-25T00:37:29.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:28 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:37:29.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:29 smithi130 ceph-mon[32525]: pgmap v128: 129 pgs: 129 active+clean; 37 GiB data, 79 GiB used, 994 GiB / 1.0 TiB avail; 381 KiB/s rd, 577 MiB/s wr, 912 op/s 2024-04-25T00:37:29.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:29 smithi032 ceph-mon[29357]: pgmap v128: 129 pgs: 129 active+clean; 37 GiB data, 79 GiB used, 994 GiB / 1.0 TiB avail; 381 KiB/s rd, 577 MiB/s wr, 912 op/s 2024-04-25T00:37:30.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:29 smithi121 ceph-mon[32231]: pgmap v128: 129 pgs: 129 active+clean; 37 GiB data, 79 GiB used, 994 GiB / 1.0 TiB avail; 381 KiB/s rd, 577 MiB/s wr, 912 op/s 2024-04-25T00:37:31.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:31 smithi130 ceph-mon[32525]: pgmap v129: 129 pgs: 129 active+clean; 38 GiB data, 81 GiB used, 992 GiB / 1.0 TiB avail; 533 KiB/s rd, 764 MiB/s wr, 1.43k op/s 2024-04-25T00:37:31.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:31 smithi032 ceph-mon[29357]: pgmap v129: 129 pgs: 129 active+clean; 38 GiB data, 81 GiB used, 992 GiB / 1.0 TiB avail; 533 KiB/s rd, 764 MiB/s wr, 1.43k op/s 2024-04-25T00:37:32.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:31 smithi121 ceph-mon[32231]: pgmap v129: 129 pgs: 129 active+clean; 38 GiB data, 81 GiB used, 992 GiB / 1.0 TiB avail; 533 KiB/s rd, 764 MiB/s wr, 1.43k op/s 2024-04-25T00:37:33.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:33 smithi130 ceph-mon[32525]: pgmap v130: 129 pgs: 129 active+clean; 38 GiB data, 81 GiB used, 992 GiB / 1.0 TiB avail; 315 KiB/s rd, 546 MiB/s wr, 1.06k op/s 2024-04-25T00:37:33.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:33 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:37:33.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:33 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:37:33.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:33 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:37:33.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:33 smithi130 ceph-mon[32525]: scrub summary: idle+waiting paths [/] 2024-04-25T00:37:33.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:33 smithi032 ceph-mon[29357]: pgmap v130: 129 pgs: 129 active+clean; 38 GiB data, 81 GiB used, 992 GiB / 1.0 TiB avail; 315 KiB/s rd, 546 MiB/s wr, 1.06k op/s 2024-04-25T00:37:33.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:33 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:37:33.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:33 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:37:33.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:33 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:37:33.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:33 smithi032 ceph-mon[29357]: scrub summary: idle+waiting paths [/] 2024-04-25T00:37:34.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:33 smithi121 ceph-mon[32231]: pgmap v130: 129 pgs: 129 active+clean; 38 GiB data, 81 GiB used, 992 GiB / 1.0 TiB avail; 315 KiB/s rd, 546 MiB/s wr, 1.06k op/s 2024-04-25T00:37:34.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:33 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:37:34.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:33 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:37:34.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:33 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:37:34.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:33 smithi121 ceph-mon[32231]: scrub summary: idle+waiting paths [/] 2024-04-25T00:37:35.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:35 smithi032 ceph-mon[29357]: pgmap v131: 129 pgs: 129 active+clean; 39 GiB data, 83 GiB used, 990 GiB / 1.0 TiB avail; 315 KiB/s rd, 638 MiB/s wr, 1.20k op/s 2024-04-25T00:37:36.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:35 smithi121 ceph-mon[32231]: pgmap v131: 129 pgs: 129 active+clean; 39 GiB data, 83 GiB used, 990 GiB / 1.0 TiB avail; 315 KiB/s rd, 638 MiB/s wr, 1.20k op/s 2024-04-25T00:37:36.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:35 smithi130 ceph-mon[32525]: pgmap v131: 129 pgs: 129 active+clean; 39 GiB data, 83 GiB used, 990 GiB / 1.0 TiB avail; 315 KiB/s rd, 638 MiB/s wr, 1.20k op/s 2024-04-25T00:37:36.642 INFO:journalctl@ceph.mds.i.smithi130.stdout:Apr 25 00:37:36 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-i[63369]: 2024-04-25T00:37:36.297+0000 7fcd81aab700 -1 mds.1.scrubstack _validate_inode_done scrub error on inode [inode 0x10000000003 [...2,head] /volumes/qa/sv_0/d23367ad-0ac2-4422-ae7b-65f85c49244e/ auth v4 ap=1 f() n(v0 rc2024-04-25T00:31:15.691664+0000 1=0+1) (inest lock) | dirfrag=0 replicated=0 authpin=1 scrubqueue=0 0x5621f541f180]: {"performed_validation":true,"passed_validation":false,"backtrace":{"checked":true,"passed":false,"read_ret_val":0,"ondisk_value":"(2)0x10000000003:[<0x10000000002/d23367ad-0ac2-4422-ae7b-65f85c49244e v4>,<0x10000000001/sv_0 v34>,<0x10000000000/qa v203>,<0x1/volumes v206>]//[]","memoryvalue":"(2)0x10000000003:[<0x10000000002/d23367ad-0ac2-4422-ae7b-65f85c49244e v4>,<0x10000000001/sv_0 v34>,<0x10000000000/qa v199>,<0x1/volumes v121>]//[]","error_str":"On-disk backtrace is divergent or newer"},"raw_stats":{"checked":true,"passed":true,"read_ret_val":0,"ondisk_value.dirstat":"f()","ondisk_value.rstat":"n(v0 1=0+1)","memory_value.dirstat":"f()","memory_value.rstat":"n(v0 rc2024-04-25T00:31:15.691664+0000 1=0+1)","error_str":""},"return_code":0} 2024-04-25T00:37:36.642 INFO:journalctl@ceph.mds.i.smithi130.stdout:Apr 25 00:37:36 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-i[63369]: 2024-04-25T00:37:36.297+0000 7fcd81aab700 -1 mds.1.scrubstack _validate_inode_done scrub error on inode [inode 0x10000000004 [2,head] /volumes/qa/sv_0/.meta auth v14 ap=1 s=117 n(v0 rc2024-04-25T00:31:15.700584+0000 b117 1=1+0) | replicated=0 authpin=1 scrubqueue=0 0x5621f541f700]: {"performed_validation":true,"passed_validation":false,"backtrace":{"checked":true,"passed":false,"read_ret_val":0,"ondisk_value":"(4)0x10000000004:[<0x10000000002/.meta v14>,<0x10000000001/sv_0 v34>,<0x10000000000/qa v203>,<0x1/volumes v206>]//[3]","memoryvalue":"(4)0x10000000004:[<0x10000000002/.meta v14>,<0x10000000001/sv_0 v34>,<0x10000000000/qa v199>,<0x1/volumes v121>]//[3]","error_str":"On-disk backtrace is divergent or newer"},"raw_stats":{"checked":false,"passed":false,"read_ret_val":0,"ondisk_value.dirstat":"f()","ondisk_value.rstat":"n()","memory_value.dirstat":"f()","memory_value.rstat":"n()","error_str":""},"return_code":0} 2024-04-25T00:37:36.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:36 smithi032 ceph-mon[29357]: scrub summary: active paths [/] 2024-04-25T00:37:36.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:36 smithi032 ceph-mon[29357]: Scrub error on inode 0x10000000003 (/volumes/qa/sv_0/d23367ad-0ac2-4422-ae7b-65f85c49244e) see mds.i log and `damage ls` output for details 2024-04-25T00:37:36.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:36 smithi032 ceph-mon[29357]: Scrub error on inode 0x10000000004 (/volumes/qa/sv_0/.meta) see mds.i log and `damage ls` output for details 2024-04-25T00:37:36.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:36 smithi032 ceph-mon[29357]: scrub summary: idle+waiting paths [/] 2024-04-25T00:37:36.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:36 smithi032 ceph-mon[29357]: scrub summary: idle+waiting paths [/] 2024-04-25T00:37:37.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:36 smithi121 ceph-mon[32231]: scrub summary: active paths [/] 2024-04-25T00:37:37.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:36 smithi121 ceph-mon[32231]: Scrub error on inode 0x10000000003 (/volumes/qa/sv_0/d23367ad-0ac2-4422-ae7b-65f85c49244e) see mds.i log and `damage ls` output for details 2024-04-25T00:37:37.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:36 smithi121 ceph-mon[32231]: Scrub error on inode 0x10000000004 (/volumes/qa/sv_0/.meta) see mds.i log and `damage ls` output for details 2024-04-25T00:37:37.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:36 smithi121 ceph-mon[32231]: scrub summary: idle+waiting paths [/] 2024-04-25T00:37:37.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:36 smithi121 ceph-mon[32231]: scrub summary: idle+waiting paths [/] 2024-04-25T00:37:37.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:36 smithi130 ceph-mon[32525]: scrub summary: active paths [/] 2024-04-25T00:37:37.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:36 smithi130 ceph-mon[32525]: Scrub error on inode 0x10000000003 (/volumes/qa/sv_0/d23367ad-0ac2-4422-ae7b-65f85c49244e) see mds.i log and `damage ls` output for details 2024-04-25T00:37:37.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:36 smithi130 ceph-mon[32525]: Scrub error on inode 0x10000000004 (/volumes/qa/sv_0/.meta) see mds.i log and `damage ls` output for details 2024-04-25T00:37:37.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:36 smithi130 ceph-mon[32525]: scrub summary: idle+waiting paths [/] 2024-04-25T00:37:37.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:36 smithi130 ceph-mon[32525]: scrub summary: idle+waiting paths [/] 2024-04-25T00:37:38.000 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:37 smithi032 ceph-mon[29357]: pgmap v132: 129 pgs: 129 active+clean; 40 GiB data, 85 GiB used, 988 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 698 MiB/s wr, 1.34k op/s 2024-04-25T00:37:38.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:37 smithi121 ceph-mon[32231]: pgmap v132: 129 pgs: 129 active+clean; 40 GiB data, 85 GiB used, 988 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 698 MiB/s wr, 1.34k op/s 2024-04-25T00:37:38.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:37 smithi130 ceph-mon[32525]: pgmap v132: 129 pgs: 129 active+clean; 40 GiB data, 85 GiB used, 988 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 698 MiB/s wr, 1.34k op/s 2024-04-25T00:37:39.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:39 smithi032 ceph-mon[29357]: pgmap v133: 129 pgs: 129 active+clean; 40 GiB data, 85 GiB used, 988 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 513 MiB/s wr, 994 op/s 2024-04-25T00:37:40.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:39 smithi121 ceph-mon[32231]: pgmap v133: 129 pgs: 129 active+clean; 40 GiB data, 85 GiB used, 988 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 513 MiB/s wr, 994 op/s 2024-04-25T00:37:40.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:39 smithi130 ceph-mon[32525]: pgmap v133: 129 pgs: 129 active+clean; 40 GiB data, 85 GiB used, 988 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 513 MiB/s wr, 994 op/s 2024-04-25T00:37:41.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:41 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-a[29353]: 2024-04-25T00:37:41.696+0000 7fe95959b700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 MDSs report damaged metadata (MDS_DAMAGE) 2024-04-25T00:37:41.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:41 smithi032 ceph-mon[29357]: pgmap v134: 129 pgs: 129 active+clean; 40 GiB data, 85 GiB used, 988 GiB / 1.0 TiB avail; 1.5 MiB/s rd, 745 MiB/s wr, 1.33k op/s 2024-04-25T00:37:42.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:41 smithi121 ceph-mon[32231]: pgmap v134: 129 pgs: 129 active+clean; 40 GiB data, 85 GiB used, 988 GiB / 1.0 TiB avail; 1.5 MiB/s rd, 745 MiB/s wr, 1.33k op/s 2024-04-25T00:37:42.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:41 smithi130 ceph-mon[32525]: pgmap v134: 129 pgs: 129 active+clean; 40 GiB data, 85 GiB used, 988 GiB / 1.0 TiB avail; 1.5 MiB/s rd, 745 MiB/s wr, 1.33k op/s 2024-04-25T00:37:42.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:42 smithi032 ceph-mon[29357]: Health check failed: 1 MDSs report damaged metadata (MDS_DAMAGE) 2024-04-25T00:37:42.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:42 smithi032 ceph-mon[29357]: mds.? [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] up:active 2024-04-25T00:37:42.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:42 smithi032 ceph-mon[29357]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:37:42.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:42 smithi032 ceph-mon[29357]: scrub summary: idle 2024-04-25T00:37:42.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:42 smithi032 ceph-mon[29357]: scrub summary: idle 2024-04-25T00:37:43.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:42 smithi121 ceph-mon[32231]: Health check failed: 1 MDSs report damaged metadata (MDS_DAMAGE) 2024-04-25T00:37:43.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:42 smithi121 ceph-mon[32231]: mds.? [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] up:active 2024-04-25T00:37:43.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:42 smithi121 ceph-mon[32231]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:37:43.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:42 smithi121 ceph-mon[32231]: scrub summary: idle 2024-04-25T00:37:43.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:42 smithi121 ceph-mon[32231]: scrub summary: idle 2024-04-25T00:37:43.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:42 smithi130 ceph-mon[32525]: Health check failed: 1 MDSs report damaged metadata (MDS_DAMAGE) 2024-04-25T00:37:43.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:42 smithi130 ceph-mon[32525]: mds.? [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] up:active 2024-04-25T00:37:43.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:42 smithi130 ceph-mon[32525]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:37:43.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:42 smithi130 ceph-mon[32525]: scrub summary: idle 2024-04-25T00:37:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:42 smithi130 ceph-mon[32525]: scrub summary: idle 2024-04-25T00:37:43.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:43 smithi032 ceph-mon[29357]: pgmap v135: 129 pgs: 129 active+clean; 40 GiB data, 85 GiB used, 988 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 517 MiB/s wr, 752 op/s 2024-04-25T00:37:43.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:43 smithi032 ceph-mon[29357]: scrub summary: idle 2024-04-25T00:37:44.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:43 smithi121 ceph-mon[32231]: pgmap v135: 129 pgs: 129 active+clean; 40 GiB data, 85 GiB used, 988 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 517 MiB/s wr, 752 op/s 2024-04-25T00:37:44.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:43 smithi121 ceph-mon[32231]: scrub summary: idle 2024-04-25T00:37:44.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:43 smithi130 ceph-mon[32525]: pgmap v135: 129 pgs: 129 active+clean; 40 GiB data, 85 GiB used, 988 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 517 MiB/s wr, 752 op/s 2024-04-25T00:37:44.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:43 smithi130 ceph-mon[32525]: scrub summary: idle 2024-04-25T00:37:45.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:45 smithi032 ceph-mon[29357]: pgmap v136: 129 pgs: 129 active+clean; 40 GiB data, 85 GiB used, 987 GiB / 1.0 TiB avail; 1.9 MiB/s rd, 603 MiB/s wr, 899 op/s 2024-04-25T00:37:46.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:45 smithi121 ceph-mon[32231]: pgmap v136: 129 pgs: 129 active+clean; 40 GiB data, 85 GiB used, 987 GiB / 1.0 TiB avail; 1.9 MiB/s rd, 603 MiB/s wr, 899 op/s 2024-04-25T00:37:46.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:45 smithi130 ceph-mon[32525]: pgmap v136: 129 pgs: 129 active+clean; 40 GiB data, 85 GiB used, 987 GiB / 1.0 TiB avail; 1.9 MiB/s rd, 603 MiB/s wr, 899 op/s 2024-04-25T00:37:47.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:47 smithi032 ceph-mon[29357]: pgmap v137: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 2.4 MiB/s rd, 753 MiB/s wr, 1.05k op/s 2024-04-25T00:37:48.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:47 smithi121 ceph-mon[32231]: pgmap v137: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 2.4 MiB/s rd, 753 MiB/s wr, 1.05k op/s 2024-04-25T00:37:48.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:47 smithi130 ceph-mon[32525]: pgmap v137: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 2.4 MiB/s rd, 753 MiB/s wr, 1.05k op/s 2024-04-25T00:37:49.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:49 smithi032 ceph-mon[29357]: pgmap v138: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 591 MiB/s wr, 839 op/s 2024-04-25T00:37:50.090 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:49 smithi121 ceph-mon[32231]: pgmap v138: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 591 MiB/s wr, 839 op/s 2024-04-25T00:37:50.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:49 smithi130 ceph-mon[32525]: pgmap v138: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 591 MiB/s wr, 839 op/s 2024-04-25T00:37:52.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:51 smithi121 ceph-mon[32231]: pgmap v139: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 2.0 MiB/s rd, 840 MiB/s wr, 1.56k op/s 2024-04-25T00:37:52.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:51 smithi130 ceph-mon[32525]: pgmap v139: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 2.0 MiB/s rd, 840 MiB/s wr, 1.56k op/s 2024-04-25T00:37:52.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:51 smithi032 ceph-mon[29357]: pgmap v139: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 2.0 MiB/s rd, 840 MiB/s wr, 1.56k op/s 2024-04-25T00:37:54.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:53 smithi121 ceph-mon[32231]: pgmap v140: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.6 MiB/s rd, 632 MiB/s wr, 1.29k op/s 2024-04-25T00:37:54.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:53 smithi130 ceph-mon[32525]: pgmap v140: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.6 MiB/s rd, 632 MiB/s wr, 1.29k op/s 2024-04-25T00:37:54.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:53 smithi032 ceph-mon[29357]: pgmap v140: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.6 MiB/s rd, 632 MiB/s wr, 1.29k op/s 2024-04-25T00:37:56.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:55 smithi121 ceph-mon[32231]: pgmap v141: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.9 MiB/s rd, 714 MiB/s wr, 1.47k op/s 2024-04-25T00:37:56.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:55 smithi130 ceph-mon[32525]: pgmap v141: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.9 MiB/s rd, 714 MiB/s wr, 1.47k op/s 2024-04-25T00:37:56.251 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:55 smithi032 ceph-mon[29357]: pgmap v141: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.9 MiB/s rd, 714 MiB/s wr, 1.47k op/s 2024-04-25T00:37:57.483 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-25T00:37:57.483 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-25T00:37:57.757 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.755+0000 7f5df3ab3640 1 -- 172.21.15.32:0/3853070229 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5dec10d470 msgr2=0x7f5dec115040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:57.757 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.755+0000 7f5df3ab3640 1 --2- 172.21.15.32:0/3853070229 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5dec10d470 0x7f5dec115040 secure :-1 s=READY pgs=636 cs=0 l=1 rev1=1 crypto rx=0x7f5de8008570 tx=0x7f5de802a1d0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.757 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.756+0000 7f5df3ab3640 1 -- 172.21.15.32:0/3853070229 shutdown_connections 2024-04-25T00:37:57.757 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.756+0000 7f5df3ab3640 1 --2- 172.21.15.32:0/3853070229 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5dec10d470 0x7f5dec115040 unknown :-1 s=CLOSED pgs=636 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.757 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.756+0000 7f5df3ab3640 1 --2- 172.21.15.32:0/3853070229 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5dec109360 0x7f5dec10cd90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.757 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.756+0000 7f5df3ab3640 1 --2- 172.21.15.32:0/3853070229 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5dec1089b0 0x7f5dec108d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.757 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.756+0000 7f5df3ab3640 1 -- 172.21.15.32:0/3853070229 >> 172.21.15.32:0/3853070229 conn(0x7f5dec06c850 msgr2=0x7f5dec06cc60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:57.757 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.756+0000 7f5df3ab3640 1 -- 172.21.15.32:0/3853070229 shutdown_connections 2024-04-25T00:37:57.757 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.756+0000 7f5df3ab3640 1 -- 172.21.15.32:0/3853070229 wait complete. 2024-04-25T00:37:57.758 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.757+0000 7f5df3ab3640 1 Processor -- start 2024-04-25T00:37:57.758 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.758+0000 7f5df3ab3640 1 -- start start 2024-04-25T00:37:57.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.758+0000 7f5df3ab3640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5dec1089b0 0x7f5dec19e5b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:57.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.758+0000 7f5df3ab3640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5dec10d470 0x7f5dec19eaf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:57.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.758+0000 7f5df3ab3640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5dec1a2e70 0x7f5dec1a3320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:57.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.758+0000 7f5df3ab3640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f5dec1180b0 con 0x7f5dec1a2e70 2024-04-25T00:37:57.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.758+0000 7f5df3ab3640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f5dec117f30 con 0x7f5dec10d470 2024-04-25T00:37:57.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.758+0000 7f5df184f640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5dec1089b0 0x7f5dec19e5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:57.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.758+0000 7f5df104e640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5dec10d470 0x7f5dec19eaf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:57.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.758+0000 7f5df184f640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5dec1089b0 0x7f5dec19e5b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:39846/0 (socket says 172.21.15.32:39846) 2024-04-25T00:37:57.760 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.758+0000 7f5df184f640 1 -- 172.21.15.32:0/1624295641 learned_addr learned my addr 172.21.15.32:0/1624295641 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:57.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.759+0000 7f5df3ab3640 1 -- 172.21.15.32:0/1624295641 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f5dec118230 con 0x7f5dec1089b0 2024-04-25T00:37:57.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.759+0000 7f5df2050640 1 --2- 172.21.15.32:0/1624295641 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5dec1a2e70 0x7f5dec1a3320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:57.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.759+0000 7f5df184f640 1 -- 172.21.15.32:0/1624295641 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5dec10d470 msgr2=0x7f5dec19eaf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:57.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.759+0000 7f5df184f640 1 --2- 172.21.15.32:0/1624295641 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5dec10d470 0x7f5dec19eaf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.759+0000 7f5df184f640 1 -- 172.21.15.32:0/1624295641 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5dec1a2e70 msgr2=0x7f5dec1a3320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:57.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.759+0000 7f5df184f640 1 --2- 172.21.15.32:0/1624295641 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5dec1a2e70 0x7f5dec1a3320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.759+0000 7f5df184f640 1 -- 172.21.15.32:0/1624295641 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5dec1a3960 con 0x7f5dec1089b0 2024-04-25T00:37:57.761 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.759+0000 7f5df2050640 1 --2- 172.21.15.32:0/1624295641 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5dec1a2e70 0x7f5dec1a3320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:37:57.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.760+0000 7f5df184f640 1 --2- 172.21.15.32:0/1624295641 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5dec1089b0 0x7f5dec19e5b0 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f5ddc011b00 tx=0x7f5ddc00f050 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:57.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.760+0000 7f5de2ffd640 1 -- 172.21.15.32:0/1624295641 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ddc00fc90 con 0x7f5dec1089b0 2024-04-25T00:37:57.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.760+0000 7f5de2ffd640 1 -- 172.21.15.32:0/1624295641 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5ddc00fe30 con 0x7f5dec1089b0 2024-04-25T00:37:57.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.760+0000 7f5de2ffd640 1 -- 172.21.15.32:0/1624295641 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ddc004ba0 con 0x7f5dec1089b0 2024-04-25T00:37:57.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.761+0000 7f5df3ab3640 1 -- 172.21.15.32:0/1624295641 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5dec1a3c50 con 0x7f5dec1089b0 2024-04-25T00:37:57.762 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.761+0000 7f5df3ab3640 1 -- 172.21.15.32:0/1624295641 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5dec1ab3c0 con 0x7f5dec1089b0 2024-04-25T00:37:57.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.766+0000 7f5df67c3740 1 Processor -- start 2024-04-25T00:37:57.767 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.766+0000 7f5df67c3740 1 -- start start 2024-04-25T00:37:57.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.767+0000 7f5df67c3740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d930605360 0x55d930605740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:57.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.767+0000 7f5df67c3740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d930605d10 0x55d9306188f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:57.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.767+0000 7f5df67c3740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d930618e30 0x55d93061b220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:57.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.767+0000 7f5df67c3740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55d93056a6b0 con 0x55d930618e30 2024-04-25T00:37:57.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.767+0000 7f5df67c3740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55d930512880 con 0x55d930605d10 2024-04-25T00:37:57.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.767+0000 7f5df67c3740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55d930566670 con 0x55d930605360 2024-04-25T00:37:57.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.767+0000 7f5df2050640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d930605360 0x55d930605740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:57.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.767+0000 7f5df2050640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d930605360 0x55d930605740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:39848/0 (socket says 172.21.15.32:39848) 2024-04-25T00:37:57.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.767+0000 7f5df2050640 1 -- 172.21.15.32:0/1007682844 learned_addr learned my addr 172.21.15.32:0/1007682844 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:57.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.768+0000 7f5df2050640 1 -- 172.21.15.32:0/1007682844 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d930605d10 msgr2=0x55d9306188f0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-25T00:37:57.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.768+0000 7f5df2050640 1 --2- 172.21.15.32:0/1007682844 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d930605d10 0x55d9306188f0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.768+0000 7f5df2050640 1 -- 172.21.15.32:0/1007682844 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d930618e30 msgr2=0x55d93061b220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:57.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.768+0000 7f5df2050640 1 --2- 172.21.15.32:0/1007682844 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d930618e30 0x55d93061b220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.768+0000 7f5df2050640 1 -- 172.21.15.32:0/1007682844 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d9303487d0 con 0x55d930605360 2024-04-25T00:37:57.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.768+0000 7f5de2ffd640 1 -- 172.21.15.32:0/1624295641 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f5ddc020020 con 0x7f5dec1089b0 2024-04-25T00:37:57.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.768+0000 7f5df2050640 1 --2- 172.21.15.32:0/1007682844 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d930605360 0x55d930605740 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f5de8032000 tx=0x7f5de80082c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=359839f7e3ab63f6 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:57.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.769+0000 7f5de2ffd640 1 --2- 172.21.15.32:0/1624295641 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f5dd80a2e40 0x7f5dd80a5300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:57.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.770+0000 7f5de2ffd640 1 -- 172.21.15.32:0/1624295641 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f5ddc0ceac0 con 0x7f5dec1089b0 2024-04-25T00:37:57.772 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.772+0000 7f5df184f640 1 --2- 172.21.15.32:0/1624295641 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f5dd80a2e40 0x7f5dd80a5300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:57.773 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.772+0000 7f5df184f640 1 --2- 172.21.15.32:0/1624295641 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f5dd80a2e40 0x7f5dd80a5300 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x55d930607f60 tx=0x7f5ddc096f00 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:57.774 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.773+0000 7f5dd7fff640 1 -- 172.21.15.32:0/1007682844 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5de8051070 con 0x55d930605360 2024-04-25T00:37:57.774 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.773+0000 7f5dd7fff640 1 -- 172.21.15.32:0/1007682844 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5de80048f0 con 0x55d930605360 2024-04-25T00:37:57.774 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.773+0000 7f5dd7fff640 1 -- 172.21.15.32:0/1007682844 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5de8003be0 con 0x55d930605360 2024-04-25T00:37:57.775 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.774+0000 7f5df67c3740 1 -- 172.21.15.32:0/1007682844 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d930605360 msgr2=0x55d930605740 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:57.775 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.774+0000 7f5df67c3740 1 --2- 172.21.15.32:0/1007682844 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d930605360 0x55d930605740 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f5de8032000 tx=0x7f5de80082c0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.782+0000 7f5df67c3740 1 -- 172.21.15.32:0/1007682844 shutdown_connections 2024-04-25T00:37:57.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.782+0000 7f5df67c3740 1 --2- 172.21.15.32:0/1007682844 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d930618e30 0x55d93061b220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.782+0000 7f5df67c3740 1 --2- 172.21.15.32:0/1007682844 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d930605d10 0x55d9306188f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.782+0000 7f5df67c3740 1 --2- 172.21.15.32:0/1007682844 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d930605360 0x55d930605740 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.782+0000 7f5df67c3740 1 -- 172.21.15.32:0/1007682844 >> 172.21.15.32:0/1007682844 conn(0x55d930601950 msgr2=0x55d930601d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:57.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.783+0000 7f5df67c3740 1 -- 172.21.15.32:0/1007682844 shutdown_connections 2024-04-25T00:37:57.784 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.784+0000 7f5df67c3740 1 -- 172.21.15.32:0/1007682844 wait complete. 2024-04-25T00:37:57.784 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.784+0000 7f5df67c3740 1 -- start start 2024-04-25T00:37:57.785 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.785+0000 7f5df67c3740 1 Processor -- start 2024-04-25T00:37:57.791 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.791+0000 7f5df67c3740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d930605360 0x55d930612960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:57.792 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.791+0000 7f5df67c3740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d930605d10 0x55d930612ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:57.792 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.791+0000 7f5df67c3740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d930618e30 0x55d930618680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:57.792 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.791+0000 7f5df67c3740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55d930608260 con 0x55d930605d10 2024-04-25T00:37:57.792 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.791+0000 7f5df2050640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d930605360 0x55d930612960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:57.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.791+0000 7f5df2050640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d930605360 0x55d930612960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:38054/0 (socket says 172.21.15.32:38054) 2024-04-25T00:37:57.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.791+0000 7f5df2050640 1 -- 172.21.15.32:0/3697516201 learned_addr learned my addr 172.21.15.32:0/3697516201 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:57.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.792+0000 7f5df67c3740 1 -- 172.21.15.32:0/3697516201 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55d930525f50 con 0x55d930605360 2024-04-25T00:37:57.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.792+0000 7f5df67c3740 1 -- 172.21.15.32:0/3697516201 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55d9306083e0 con 0x55d930618e30 2024-04-25T00:37:57.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.792+0000 7f5df2050640 1 --2- 172.21.15.32:0/3697516201 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d930618e30 0x55d930618680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:57.793 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.792+0000 7f5df67c3740 10 client.0 _ll_register_callbacks cb 0x55d930561000 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:37:57.794 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.792+0000 7f5df2050640 1 -- 172.21.15.32:0/3697516201 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d930618e30 msgr2=0x55d930618680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:57.794 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.792+0000 7f5df2050640 1 --2- 172.21.15.32:0/3697516201 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d930618e30 0x55d930618680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.794 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.792+0000 7f5df2050640 1 -- 172.21.15.32:0/3697516201 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d930605d10 msgr2=0x55d930612ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:57.794 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.792+0000 7f5df2050640 1 --2- 172.21.15.32:0/3697516201 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d930605d10 0x55d930612ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.794 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.792+0000 7f5df2050640 1 -- 172.21.15.32:0/3697516201 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d93027e3f0 con 0x55d930605360 2024-04-25T00:37:57.794 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.793+0000 7f5df2050640 1 --2- 172.21.15.32:0/3697516201 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d930605360 0x55d930612960 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f5de802a7c0 tx=0x7f5de80037c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ad327382e1b42bb7 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:57.794 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.793+0000 7f5dd57fa640 10 client.0 ms_handle_connect on v2:172.21.15.121:3300/0 2024-04-25T00:37:57.795 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.793+0000 7f5dd57fa640 1 -- 172.21.15.32:0/3697516201 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5de8051070 con 0x55d930605360 2024-04-25T00:37:57.795 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.793+0000 7f5dd57fa640 1 -- 172.21.15.32:0/3697516201 <== mon.2 v2:172.21.15.121:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f5de8003b70 con 0x55d930605360 2024-04-25T00:37:57.795 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.794+0000 7f5df67c3740 1 -- 172.21.15.32:0/3697516201 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5de80048f0 con 0x55d930605360 2024-04-25T00:37:57.800 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.797+0000 7f5dd57fa640 1 -- 172.21.15.32:0/3697516201 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7f5de8004400 con 0x55d930605360 2024-04-25T00:37:57.803 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.803+0000 7f5df67c3740 10 client.25835 fetch_fsmap learned FSMap version 28 2024-04-25T00:37:57.804 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.803+0000 7f5df67c3740 1 -- 172.21.15.32:0/3697516201 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x55d93023e0f0 con 0x55d930605360 2024-04-25T00:37:57.804 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.804+0000 7f5dd57fa640 1 -- 172.21.15.32:0/3697516201 <== mon.2 v2:172.21.15.121:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5de8003d70 con 0x55d930605360 2024-04-25T00:37:57.804 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.804+0000 7f5dd57fa640 1 -- 172.21.15.32:0/3697516201 <== mon.2 v2:172.21.15.121:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7f5de800a0d0 con 0x55d930605360 2024-04-25T00:37:57.804 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.804+0000 7f5df67c3740 10 client.25835 fetch_fsmap finished waiting for FSMap version 28 2024-04-25T00:37:57.805 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.804+0000 7f5df67c3740 10 client.25835 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:37:57.805 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.804+0000 7f5df67c3740 20 client.25835 populate_metadata read hostname 'smithi032' 2024-04-25T00:37:57.806 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.804+0000 7f5df67c3740 1 --2- 172.21.15.32:0/3697516201 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55d9306208b0 0x55d930622ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:57.806 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.804+0000 7f5df67c3740 4 client.25835 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:37:57.806 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.804+0000 7f5df67c3740 1 -- 172.21.15.32:0/3697516201 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x7f5de80048f0 con 0x55d9306208b0 2024-04-25T00:37:57.807 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.807+0000 7f5df104e640 1 --2- 172.21.15.32:0/3697516201 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55d9306208b0 0x55d930622ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:57.811 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.810+0000 7f5dd57fa640 10 client.25835 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:37:57.811 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.811+0000 7f5df104e640 1 --2- 172.21.15.32:0/3697516201 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55d9306208b0 0x55d930622ca0 crc :-1 s=READY pgs=95 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a63deaa4bcfab8c3 server_cookie=f234451ac27c9562 in_seq=0 out_seq=0 2024-04-25T00:37:57.816 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.815+0000 7f5dd57fa640 1 -- 172.21.15.32:0/3697516201 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x7f5de80048f0 con 0x55d9306208b0 2024-04-25T00:37:57.816 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.815+0000 7f5dd57fa640 10 client.25835 handle_command_reply: tid=0 2024-04-25T00:37:57.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.823+0000 7f5df67c3740 1 client.25835 shutdown 2024-04-25T00:37:57.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.824+0000 7f5df67c3740 1 -- 172.21.15.32:0/3697516201 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d930605360 msgr2=0x55d930612960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:57.824 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.824+0000 7f5df67c3740 1 --2- 172.21.15.32:0/3697516201 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d930605360 0x55d930612960 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f5de802a7c0 tx=0x7f5de80037c0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.830+0000 7f5df67c3740 1 -- 172.21.15.32:0/3697516201 shutdown_connections 2024-04-25T00:37:57.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.830+0000 7f5df67c3740 1 --2- 172.21.15.32:0/3697516201 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55d9306208b0 0x55d930622ca0 crc :-1 s=READY pgs=95 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.831+0000 7f5df67c3740 1 --2- 172.21.15.32:0/3697516201 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d930618e30 0x55d930618680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.831+0000 7f5df67c3740 1 --2- 172.21.15.32:0/3697516201 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d930605d10 0x55d930612ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.831+0000 7f5df67c3740 1 --2- 172.21.15.32:0/3697516201 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d930605360 0x55d930612960 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.831+0000 7f5df67c3740 1 -- 172.21.15.32:0/3697516201 >> 172.21.15.32:0/3697516201 conn(0x55d930601950 msgr2=0x55d9306077c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:57.832 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.831+0000 7f5dd57fa640 0 client.25835 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:37:57.833 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.832+0000 7f5df67c3740 1 -- 172.21.15.32:0/3697516201 shutdown_connections 2024-04-25T00:37:57.834 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.833+0000 7f5df67c3740 1 -- 172.21.15.32:0/3697516201 wait complete. 2024-04-25T00:37:57.839 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.837+0000 7f5df67c3740 20 client.25835 trim_cache size 0 max 16384 2024-04-25T00:37:57.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.892+0000 7f5df67c3740 1 Processor -- start 2024-04-25T00:37:57.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.892+0000 7f5df67c3740 1 -- start start 2024-04-25T00:37:57.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.893+0000 7f5df67c3740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d930601950 0x55d930607640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:57.897 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.895+0000 7f5df2050640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d930601950 0x55d930607640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:57.897 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.895+0000 7f5df67c3740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d930605360 0x55d930625cd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:57.897 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.896+0000 7f5df67c3740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d930605d10 0x55d930626210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:57.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.896+0000 7f5df67c3740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f5dec1b7cb0 con 0x55d930601950 2024-04-25T00:37:57.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.896+0000 7f5df67c3740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f5dec1b5580 con 0x55d930605360 2024-04-25T00:37:57.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.896+0000 7f5df67c3740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f5dec1b33d0 con 0x55d930605d10 2024-04-25T00:37:57.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.896+0000 7f5df2050640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d930601950 0x55d930607640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44976/0 (socket says 172.21.15.32:44976) 2024-04-25T00:37:57.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.896+0000 7f5df2050640 1 -- 172.21.15.32:0/3936486563 learned_addr learned my addr 172.21.15.32:0/3936486563 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:57.898 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.896+0000 7f5df2050640 1 --2- 172.21.15.32:0/3936486563 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d930605d10 0x55d930626210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:57.899 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.898+0000 7f5df104e640 1 --2- 172.21.15.32:0/3936486563 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d930605360 0x55d930625cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:57.899 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.898+0000 7f5df2050640 1 -- 172.21.15.32:0/3936486563 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d930605360 msgr2=0x55d930625cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:57.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.898+0000 7f5df2050640 1 --2- 172.21.15.32:0/3936486563 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d930605360 0x55d930625cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.898+0000 7f5df2050640 1 -- 172.21.15.32:0/3936486563 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d930601950 msgr2=0x55d930607640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:57.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.898+0000 7f5df2050640 1 --2- 172.21.15.32:0/3936486563 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d930601950 0x55d930607640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.898+0000 7f5df2050640 1 -- 172.21.15.32:0/3936486563 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d93060db90 con 0x55d930605d10 2024-04-25T00:37:57.901 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.899+0000 7f5df2050640 1 --2- 172.21.15.32:0/3936486563 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d930605d10 0x55d930626210 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f5de800aae0 tx=0x7f5de8005fd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1035f8ae4ddb8039 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:57.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.900+0000 7f5dd77fe640 1 -- 172.21.15.32:0/3936486563 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5de8051030 con 0x55d930605d10 2024-04-25T00:37:57.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.900+0000 7f5dd77fe640 1 -- 172.21.15.32:0/3936486563 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5de806b070 con 0x55d930605d10 2024-04-25T00:37:57.902 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.900+0000 7f5dd77fe640 1 -- 172.21.15.32:0/3936486563 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5de800a2d0 con 0x55d930605d10 2024-04-25T00:37:57.903 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.902+0000 7f5df67c3740 1 -- 172.21.15.32:0/3936486563 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d930605d10 msgr2=0x55d930626210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:57.903 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.902+0000 7f5df67c3740 1 --2- 172.21.15.32:0/3936486563 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d930605d10 0x55d930626210 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f5de800aae0 tx=0x7f5de8005fd0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.905+0000 7f5df67c3740 1 -- 172.21.15.32:0/3936486563 shutdown_connections 2024-04-25T00:37:57.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.905+0000 7f5df67c3740 1 --2- 172.21.15.32:0/3936486563 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d930605d10 0x55d930626210 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.910+0000 7f5df67c3740 1 --2- 172.21.15.32:0/3936486563 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d930605360 0x55d930625cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.910+0000 7f5df67c3740 1 --2- 172.21.15.32:0/3936486563 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d930601950 0x55d930607640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.910+0000 7f5df67c3740 1 -- 172.21.15.32:0/3936486563 >> 172.21.15.32:0/3936486563 conn(0x55d9305fcd80 msgr2=0x55d930606f30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:57.917 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.917+0000 7f5df67c3740 1 -- 172.21.15.32:0/3936486563 shutdown_connections 2024-04-25T00:37:57.918 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.917+0000 7f5df67c3740 1 -- 172.21.15.32:0/3936486563 wait complete. 2024-04-25T00:37:57.918 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.917+0000 7f5df67c3740 1 -- start start 2024-04-25T00:37:57.918 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.918+0000 7f5df67c3740 1 Processor -- start 2024-04-25T00:37:57.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.919+0000 7f5df67c3740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d930601950 0x55d93067f0b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:57.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.919+0000 7f5df67c3740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d930605360 0x55d93067f5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:57.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.919+0000 7f5df67c3740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d930605d10 0x55d9306251f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:57.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.919+0000 7f5df67c3740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f5dec1b3a40 con 0x55d930601950 2024-04-25T00:37:57.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.919+0000 7f5df67c3740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f5dec1b4eb0 con 0x55d930605360 2024-04-25T00:37:57.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.919+0000 7f5df67c3740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f5dec1b4110 con 0x55d930605d10 2024-04-25T00:37:57.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.920+0000 7f5df67c3740 10 client.0 _ll_register_callbacks cb 0x7f5de804f070 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:37:57.921 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.921+0000 7f5df2050640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d930601950 0x55d93067f0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:57.921 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.921+0000 7f5df2050640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d930601950 0x55d93067f0b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:44992/0 (socket says 172.21.15.32:44992) 2024-04-25T00:37:57.921 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.921+0000 7f5df2050640 1 -- 172.21.15.32:0/900215314 learned_addr learned my addr 172.21.15.32:0/900215314 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:57.925 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.921+0000 7f5df104e640 1 --2- 172.21.15.32:0/900215314 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d930605360 0x55d93067f5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:57.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.925+0000 7f5df104e640 1 -- 172.21.15.32:0/900215314 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d930605d10 msgr2=0x55d9306251f0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-25T00:37:57.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.925+0000 7f5df104e640 1 --2- 172.21.15.32:0/900215314 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d930605d10 0x55d9306251f0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.925+0000 7f5df104e640 1 -- 172.21.15.32:0/900215314 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d930601950 msgr2=0x55d93067f0b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:57.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.925+0000 7f5df104e640 1 --2- 172.21.15.32:0/900215314 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d930601950 0x55d93067f0b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.925+0000 7f5df104e640 1 -- 172.21.15.32:0/900215314 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d9306258e0 con 0x55d930605360 2024-04-25T00:37:57.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.931+0000 7f5df104e640 1 --2- 172.21.15.32:0/900215314 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d930605360 0x55d93067f5f0 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x55d93060d5c0 tx=0x7f5de40196e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ec8c42fe26d71c32 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:57.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.931+0000 7f5dd57fa640 10 client.0 ms_handle_connect on v2:172.21.15.121:3300/0 2024-04-25T00:37:57.933 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.933+0000 7f5dd57fa640 1 -- 172.21.15.32:0/900215314 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5de4003970 con 0x55d930605360 2024-04-25T00:37:57.935 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.933+0000 7f5dd57fa640 1 -- 172.21.15.32:0/900215314 <== mon.2 v2:172.21.15.121:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f5de4003b10 con 0x55d930605360 2024-04-25T00:37:57.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.934+0000 7f5df67c3740 1 -- 172.21.15.32:0/900215314 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5de806b070 con 0x55d930605360 2024-04-25T00:37:57.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.936+0000 7f5dd57fa640 1 -- 172.21.15.32:0/900215314 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7f5de4027050 con 0x55d930605360 2024-04-25T00:37:57.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.936+0000 7f5df67c3740 10 client.25841 fetch_fsmap learned FSMap version 28 2024-04-25T00:37:57.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.936+0000 7f5df67c3740 1 -- 172.21.15.32:0/900215314 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x55d930686920 con 0x55d930605360 2024-04-25T00:37:57.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.938+0000 7f5dd57fa640 1 -- 172.21.15.32:0/900215314 <== mon.2 v2:172.21.15.121:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5de806b070 con 0x55d930605360 2024-04-25T00:37:57.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.938+0000 7f5dd57fa640 1 -- 172.21.15.32:0/900215314 <== mon.2 v2:172.21.15.121:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7f5de400f040 con 0x55d930605360 2024-04-25T00:37:57.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.940+0000 7f5df67c3740 10 client.25841 fetch_fsmap finished waiting for FSMap version 28 2024-04-25T00:37:57.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.940+0000 7f5df67c3740 10 client.25841 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:37:57.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.940+0000 7f5df67c3740 20 client.25841 populate_metadata read hostname 'smithi032' 2024-04-25T00:37:57.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.940+0000 7f5df67c3740 1 --2- 172.21.15.32:0/900215314 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55d930618e30 0x55d930689090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:57.941 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.941+0000 7f5df67c3740 4 client.25841 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-25T00:37:57.942 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.941+0000 7f5df67c3740 1 -- 172.21.15.32:0/900215314 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "scrub status"}) -- 0x55d930577bf0 con 0x55d930618e30 2024-04-25T00:37:57.943 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.942+0000 7f5df2050640 1 --2- 172.21.15.32:0/900215314 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55d930618e30 0x55d930689090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:57.958 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.953+0000 7f5df2050640 1 --2- 172.21.15.32:0/900215314 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55d930618e30 0x55d930689090 crc :-1 s=READY pgs=96 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=db95fb5438fa55a8 server_cookie=3dfeb0303fc88269 in_seq=0 out_seq=0 2024-04-25T00:37:57.958 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.956+0000 7f5dd57fa640 10 client.25841 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:37:57.958 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.957+0000 7f5dd57fa640 1 -- 172.21.15.32:0/900215314 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+63 (crc 0 0 0) 0x55d930577bf0 con 0x55d930618e30 2024-04-25T00:37:57.959 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.957+0000 7f5dd57fa640 10 client.25841 handle_command_reply: tid=0 2024-04-25T00:37:57.959 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.958+0000 7f5df67c3740 1 client.25841 shutdown 2024-04-25T00:37:57.966 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.965+0000 7f5df67c3740 1 -- 172.21.15.32:0/900215314 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d930605360 msgr2=0x55d93067f5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:57.966 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.965+0000 7f5df67c3740 1 --2- 172.21.15.32:0/900215314 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d930605360 0x55d93067f5f0 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x55d93060d5c0 tx=0x7f5de40196e0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.967+0000 7f5df67c3740 1 -- 172.21.15.32:0/900215314 shutdown_connections 2024-04-25T00:37:57.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.967+0000 7f5df67c3740 1 --2- 172.21.15.32:0/900215314 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55d930618e30 0x55d930689090 crc :-1 s=READY pgs=96 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.967+0000 7f5df67c3740 1 --2- 172.21.15.32:0/900215314 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d930605d10 0x55d9306251f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.967+0000 7f5df67c3740 1 --2- 172.21.15.32:0/900215314 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55d930605360 0x55d93067f5f0 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.968 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.967+0000 7f5df67c3740 1 --2- 172.21.15.32:0/900215314 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55d930601950 0x55d93067f0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.967+0000 7f5df67c3740 1 -- 172.21.15.32:0/900215314 >> 172.21.15.32:0/900215314 conn(0x55d9305fcd80 msgr2=0x55d9306077c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:57.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.967+0000 7f5dd57fa640 0 client.25841 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:37:57.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.969+0000 7f5df67c3740 1 -- 172.21.15.32:0/900215314 shutdown_connections 2024-04-25T00:37:57.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.969+0000 7f5df67c3740 1 -- 172.21.15.32:0/900215314 wait complete. 2024-04-25T00:37:57.972 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.971+0000 7f5df67c3740 20 client.25841 trim_cache size 0 max 16384 2024-04-25T00:37:57.973 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:37:57.973 INFO:teuthology.orchestra.run.smithi032.stdout: "status": "no active scrubs running", 2024-04-25T00:37:57.973 INFO:teuthology.orchestra.run.smithi032.stdout: "scrubs": {} 2024-04-25T00:37:57.973 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:37:57.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.979+0000 7f5df3ab3640 1 -- 172.21.15.32:0/1624295641 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f5dd80a2e40 msgr2=0x7f5dd80a5300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:57.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.979+0000 7f5df3ab3640 1 --2- 172.21.15.32:0/1624295641 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f5dd80a2e40 0x7f5dd80a5300 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x55d930607f60 tx=0x7f5ddc096f00 comp rx=0 tx=0).stop 2024-04-25T00:37:57.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.979+0000 7f5df3ab3640 1 -- 172.21.15.32:0/1624295641 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5dec1089b0 msgr2=0x7f5dec19e5b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:57.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.979+0000 7f5df3ab3640 1 --2- 172.21.15.32:0/1624295641 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5dec1089b0 0x7f5dec19e5b0 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f5ddc011b00 tx=0x7f5ddc00f050 comp rx=0 tx=0).stop 2024-04-25T00:37:57.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.983+0000 7f5df3ab3640 1 -- 172.21.15.32:0/1624295641 shutdown_connections 2024-04-25T00:37:57.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.983+0000 7f5df3ab3640 1 --2- 172.21.15.32:0/1624295641 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f5dd80a2e40 0x7f5dd80a5300 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.983+0000 7f5df3ab3640 1 --2- 172.21.15.32:0/1624295641 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f5dec1a2e70 0x7f5dec1a3320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.983+0000 7f5df3ab3640 1 --2- 172.21.15.32:0/1624295641 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f5dec10d470 0x7f5dec19eaf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.983+0000 7f5df3ab3640 1 --2- 172.21.15.32:0/1624295641 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5dec1089b0 0x7f5dec19e5b0 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:57.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.983+0000 7f5df3ab3640 1 -- 172.21.15.32:0/1624295641 >> 172.21.15.32:0/1624295641 conn(0x7f5dec06c850 msgr2=0x7f5dec0714b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:57.986 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.986+0000 7f5df3ab3640 1 -- 172.21.15.32:0/1624295641 shutdown_connections 2024-04-25T00:37:57.987 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:57.987+0000 7f5df3ab3640 1 -- 172.21.15.32:0/1624295641 wait complete. 2024-04-25T00:37:57.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:57 smithi032 ceph-mon[29357]: pgmap v142: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.9 MiB/s rd, 864 MiB/s wr, 1.68k op/s 2024-04-25T00:37:58.037 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-25T00:37:58.038 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:37:58.090 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:57 smithi121 ceph-mon[32231]: pgmap v142: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.9 MiB/s rd, 864 MiB/s wr, 1.68k op/s 2024-04-25T00:37:58.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:57 smithi130 ceph-mon[32525]: pgmap v142: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.9 MiB/s rd, 864 MiB/s wr, 1.68k op/s 2024-04-25T00:37:58.328 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.327+0000 7f1661443640 1 -- 172.21.15.32:0/2199293756 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1654009070 msgr2=0x7f1654009450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:58.328 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.327+0000 7f1661443640 1 --2- 172.21.15.32:0/2199293756 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1654009070 0x7f1654009450 secure :-1 s=READY pgs=637 cs=0 l=1 rev1=1 crypto rx=0x7f1650009860 tx=0x7f165002a360 comp rx=0 tx=0).stop 2024-04-25T00:37:58.337 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.335+0000 7f1661443640 1 -- 172.21.15.32:0/2199293756 shutdown_connections 2024-04-25T00:37:58.337 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.335+0000 7f1661443640 1 --2- 172.21.15.32:0/2199293756 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f16540a6b90 0x7f16540a8f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:58.337 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.335+0000 7f1661443640 1 --2- 172.21.15.32:0/2199293756 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1654009990 0x7f16540a6650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:58.337 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.335+0000 7f1661443640 1 --2- 172.21.15.32:0/2199293756 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1654009070 0x7f1654009450 unknown :-1 s=CLOSED pgs=637 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:58.337 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.335+0000 7f1661443640 1 -- 172.21.15.32:0/2199293756 >> 172.21.15.32:0/2199293756 conn(0x7f1654093a00 msgr2=0x7f1654095e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:58.342 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.340+0000 7f1661443640 1 -- 172.21.15.32:0/2199293756 shutdown_connections 2024-04-25T00:37:58.342 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.342+0000 7f1661443640 1 -- 172.21.15.32:0/2199293756 wait complete. 2024-04-25T00:37:58.343 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.343+0000 7f1661443640 1 Processor -- start 2024-04-25T00:37:58.347 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.346+0000 7f1661443640 1 -- start start 2024-04-25T00:37:58.347 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.346+0000 7f1661443640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1654009070 0x7f1654143d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:58.347 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.346+0000 7f1661443640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1654009990 0x7f16541442b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:58.347 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.346+0000 7f1661443640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16540a6b90 0x7f16541485e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:58.347 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.346+0000 7f1661443640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f16540bd7b0 con 0x7f1654009990 2024-04-25T00:37:58.347 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.346+0000 7f1661443640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f16540bd630 con 0x7f1654009070 2024-04-25T00:37:58.347 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.346+0000 7f1661443640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f16540bd930 con 0x7f16540a6b90 2024-04-25T00:37:58.349 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.347+0000 7f165b7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1654009990 0x7f16541442b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:58.349 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.347+0000 7f165bfff640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1654009070 0x7f1654143d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:58.349 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.347+0000 7f1660c42640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16540a6b90 0x7f16541485e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:58.349 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.347+0000 7f165bfff640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1654009070 0x7f1654143d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:38092/0 (socket says 172.21.15.32:38092) 2024-04-25T00:37:58.349 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.347+0000 7f165bfff640 1 -- 172.21.15.32:0/2540928517 learned_addr learned my addr 172.21.15.32:0/2540928517 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:58.349 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.348+0000 7f165bfff640 1 -- 172.21.15.32:0/2540928517 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16540a6b90 msgr2=0x7f16541485e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:58.349 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.348+0000 7f165bfff640 1 --2- 172.21.15.32:0/2540928517 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16540a6b90 0x7f16541485e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:58.349 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.348+0000 7f165bfff640 1 -- 172.21.15.32:0/2540928517 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1654009990 msgr2=0x7f16541442b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:58.349 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.348+0000 7f165bfff640 1 --2- 172.21.15.32:0/2540928517 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1654009990 0x7f16541442b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:58.349 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.348+0000 7f165bfff640 1 -- 172.21.15.32:0/2540928517 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1654148ce0 con 0x7f1654009070 2024-04-25T00:37:58.351 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.349+0000 7f165bfff640 1 --2- 172.21.15.32:0/2540928517 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1654009070 0x7f1654143d70 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f1650032670 tx=0x7f16500326a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:58.351 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.350+0000 7f165b7fe640 1 --2- 172.21.15.32:0/2540928517 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1654009990 0x7f16541442b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:37:58.351 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.351+0000 7f16597fa640 1 -- 172.21.15.32:0/2540928517 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1650037040 con 0x7f1654009070 2024-04-25T00:37:58.351 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.351+0000 7f1661443640 1 -- 172.21.15.32:0/2540928517 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1654148f70 con 0x7f1654009070 2024-04-25T00:37:58.352 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.351+0000 7f16597fa640 1 -- 172.21.15.32:0/2540928517 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1650032cd0 con 0x7f1654009070 2024-04-25T00:37:58.352 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.351+0000 7f16597fa640 1 -- 172.21.15.32:0/2540928517 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f165003b3f0 con 0x7f1654009070 2024-04-25T00:37:58.352 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.351+0000 7f1661443640 1 -- 172.21.15.32:0/2540928517 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1654097e00 con 0x7f1654009070 2024-04-25T00:37:58.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.359+0000 7f1661443640 1 -- 172.21.15.32:0/2540928517 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f165400a7b0 con 0x7f1654009070 2024-04-25T00:37:58.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.360+0000 7f16597fa640 1 -- 172.21.15.32:0/2540928517 <== mon.2 v2:172.21.15.121:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f1650004050 con 0x7f1654009070 2024-04-25T00:37:58.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.361+0000 7f16597fa640 1 --2- 172.21.15.32:0/2540928517 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f164c0a2f10 0x7f164c0a53d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:58.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.361+0000 7f16597fa640 1 -- 172.21.15.32:0/2540928517 <== mon.2 v2:172.21.15.121:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f16500ef1c0 con 0x7f1654009070 2024-04-25T00:37:58.366 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.364+0000 7f165b7fe640 1 --2- 172.21.15.32:0/2540928517 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f164c0a2f10 0x7f164c0a53d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:58.376 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.375+0000 7f165b7fe640 1 --2- 172.21.15.32:0/2540928517 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f164c0a2f10 0x7f164c0a53d0 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f16540a80a0 tx=0x7f164800b390 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:58.377 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.377+0000 7f16597fa640 1 -- 172.21.15.32:0/2540928517 <== mon.2 v2:172.21.15.121:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f16500bbae0 con 0x7f1654009070 2024-04-25T00:37:58.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.914+0000 7f1661443640 1 -- 172.21.15.32:0/2540928517 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f1654145070 con 0x7f1654009070 2024-04-25T00:37:58.918 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.918+0000 7f16597fa640 1 -- 172.21.15.32:0/2540928517 <== mon.2 v2:172.21.15.121:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 28 v28) ==== 94+0+11880 (secure 0 0 0) 0x7f16500bb3e0 con 0x7f1654009070 2024-04-25T00:37:58.923 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:37:58.924 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":28,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14643,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6839/187492674","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6838","nonce":187492674},{"type":"v1","addr":"172.21.15.32:6839","nonce":187492674}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14655,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6841/1948785545","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6840","nonce":1948785545},{"type":"v1","addr":"172.21.15.32:6841","nonce":1948785545}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":28,"flags":306,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":true},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-25T00:30:47.866875+0000","modified":"2024-04-25T00:37:40.998307+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24506,"mds_1":24481,"mds_2":24476},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24443":{"gid":24443,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24445":{"gid":24445,"name":"f","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24457":{"gid":24457,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24476":{"gid":24476,"name":"h","rank":2,"incarnation":22,"state":"up:active","state_seq":12,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24481":{"gid":24481,"name":"i","rank":1,"incarnation":20,"state":"up:active","state_seq":108,"addr":"172.21.15.130:6839/4267125809","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":4267125809},{"type":"v1","addr":"172.21.15.130:6839","nonce":4267125809}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24506":{"gid":24506,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.121:6839/258444853","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6838","nonce":258444853},{"type":"v1","addr":"172.21.15.121:6839","nonce":258444853}]},"join_fscid":-1,"export_targets":[1,2],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24506,"qdb_cluster":[24506,24476,24481]},"id":1}]} 2024-04-25T00:37:58.924 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 28 2024-04-25T00:37:58.935 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.934+0000 7f1661443640 1 -- 172.21.15.32:0/2540928517 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f164c0a2f10 msgr2=0x7f164c0a53d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:58.935 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.934+0000 7f1661443640 1 --2- 172.21.15.32:0/2540928517 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f164c0a2f10 0x7f164c0a53d0 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f16540a80a0 tx=0x7f164800b390 comp rx=0 tx=0).stop 2024-04-25T00:37:58.935 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.934+0000 7f1661443640 1 -- 172.21.15.32:0/2540928517 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1654009070 msgr2=0x7f1654143d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:58.935 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.934+0000 7f1661443640 1 --2- 172.21.15.32:0/2540928517 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1654009070 0x7f1654143d70 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f1650032670 tx=0x7f16500326a0 comp rx=0 tx=0).stop 2024-04-25T00:37:58.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.937+0000 7f1661443640 1 -- 172.21.15.32:0/2540928517 shutdown_connections 2024-04-25T00:37:58.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.937+0000 7f1661443640 1 --2- 172.21.15.32:0/2540928517 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f164c0a2f10 0x7f164c0a53d0 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:58.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.937+0000 7f1661443640 1 --2- 172.21.15.32:0/2540928517 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16540a6b90 0x7f16541485e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:58.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.937+0000 7f1661443640 1 --2- 172.21.15.32:0/2540928517 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1654009990 0x7f16541442b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:58.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.937+0000 7f1661443640 1 --2- 172.21.15.32:0/2540928517 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1654009070 0x7f1654143d70 unknown :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:58.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.937+0000 7f1661443640 1 -- 172.21.15.32:0/2540928517 >> 172.21.15.32:0/2540928517 conn(0x7f1654093a00 msgr2=0x7f16540a8250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:58.938 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.937+0000 7f1661443640 1 -- 172.21.15.32:0/2540928517 shutdown_connections 2024-04-25T00:37:58.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:58.938+0000 7f1661443640 1 -- 172.21.15.32:0/2540928517 wait complete. 2024-04-25T00:37:58.976 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-25T00:37:59.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.200+0000 7f7b2bc12640 1 -- 172.21.15.32:0/4028348091 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b24107720 msgr2=0x7f7b24107b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:59.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.200+0000 7f7b2bc12640 1 --2- 172.21.15.32:0/4028348091 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b24107720 0x7f7b24107b00 secure :-1 s=READY pgs=638 cs=0 l=1 rev1=1 crypto rx=0x7f7b20009930 tx=0x7f7b2002a420 comp rx=0 tx=0).stop 2024-04-25T00:37:59.203 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.200+0000 7f7b2bc12640 1 -- 172.21.15.32:0/4028348091 shutdown_connections 2024-04-25T00:37:59.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.200+0000 7f7b2bc12640 1 --2- 172.21.15.32:0/4028348091 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7b241020d0 0x7f7b2410e5e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.200+0000 7f7b2bc12640 1 --2- 172.21.15.32:0/4028348091 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7b24101710 0x7f7b24101b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.200+0000 7f7b2bc12640 1 --2- 172.21.15.32:0/4028348091 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b24107720 0x7f7b24107b00 unknown :-1 s=CLOSED pgs=638 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.200+0000 7f7b2bc12640 1 -- 172.21.15.32:0/4028348091 >> 172.21.15.32:0/4028348091 conn(0x7f7b24078b90 msgr2=0x7f7b24078fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:59.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.200+0000 7f7b2bc12640 1 -- 172.21.15.32:0/4028348091 shutdown_connections 2024-04-25T00:37:59.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.200+0000 7f7b2bc12640 1 -- 172.21.15.32:0/4028348091 wait complete. 2024-04-25T00:37:59.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.201+0000 7f7b2bc12640 1 Processor -- start 2024-04-25T00:37:59.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.201+0000 7f7b2bc12640 1 -- start start 2024-04-25T00:37:59.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.201+0000 7f7b2bc12640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7b24101710 0x7f7b241980a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.201+0000 7f7b2bc12640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7b241020d0 0x7f7b241985e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.201+0000 7f7b2bc12640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b24107720 0x7f7b2419c960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.201+0000 7f7b2bc12640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f7b24111b40 con 0x7f7b24107720 2024-04-25T00:37:59.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.201+0000 7f7b2bc12640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f7b241119c0 con 0x7f7b24101710 2024-04-25T00:37:59.204 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.201+0000 7f7b2bc12640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f7b24111cc0 con 0x7f7b241020d0 2024-04-25T00:37:59.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.201+0000 7f7b2a1af640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b24107720 0x7f7b2419c960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.201+0000 7f7b2a1af640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b24107720 0x7f7b2419c960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:45028/0 (socket says 172.21.15.32:45028) 2024-04-25T00:37:59.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.201+0000 7f7b2a1af640 1 -- 172.21.15.32:0/314688312 learned_addr learned my addr 172.21.15.32:0/314688312 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:59.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.201+0000 7f7b299ae640 1 --2- 172.21.15.32:0/314688312 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7b24101710 0x7f7b241980a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.202+0000 7f7b291ad640 1 --2- 172.21.15.32:0/314688312 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7b241020d0 0x7f7b241985e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.202+0000 7f7b2a1af640 1 -- 172.21.15.32:0/314688312 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7b241020d0 msgr2=0x7f7b241985e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:59.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.202+0000 7f7b2a1af640 1 --2- 172.21.15.32:0/314688312 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7b241020d0 0x7f7b241985e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.202+0000 7f7b2a1af640 1 -- 172.21.15.32:0/314688312 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7b24101710 msgr2=0x7f7b241980a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:59.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.202+0000 7f7b2a1af640 1 --2- 172.21.15.32:0/314688312 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7b24101710 0x7f7b241980a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.202+0000 7f7b2a1af640 1 -- 172.21.15.32:0/314688312 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7b2419d090 con 0x7f7b24107720 2024-04-25T00:37:59.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.202+0000 7f7b299ae640 1 --2- 172.21.15.32:0/314688312 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7b24101710 0x7f7b241980a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:37:59.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.202+0000 7f7b2a1af640 1 --2- 172.21.15.32:0/314688312 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b24107720 0x7f7b2419c960 secure :-1 s=READY pgs=639 cs=0 l=1 rev1=1 crypto rx=0x7f7b1c00eb10 tx=0x7f7b1c00c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:59.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.202+0000 7f7b1affd640 1 -- 172.21.15.32:0/314688312 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b1c004b20 con 0x7f7b24107720 2024-04-25T00:37:59.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.202+0000 7f7b1affd640 1 -- 172.21.15.32:0/314688312 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7b1c004cc0 con 0x7f7b24107720 2024-04-25T00:37:59.205 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.202+0000 7f7b2bc12640 1 -- 172.21.15.32:0/314688312 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7b2419d380 con 0x7f7b24107720 2024-04-25T00:37:59.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.202+0000 7f7b1affd640 1 -- 172.21.15.32:0/314688312 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b1c010760 con 0x7f7b24107720 2024-04-25T00:37:59.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.202+0000 7f7b2bc12640 1 -- 172.21.15.32:0/314688312 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7b241a4c10 con 0x7f7b24107720 2024-04-25T00:37:59.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.205+0000 7f7b2e9fb740 1 Processor -- start 2024-04-25T00:37:59.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.206+0000 7f7b1affd640 1 -- 172.21.15.32:0/314688312 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f7b1c005020 con 0x7f7b24107720 2024-04-25T00:37:59.207 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.206+0000 7f7b2e9fb740 1 -- start start 2024-04-25T00:37:59.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.206+0000 7f7b2e9fb740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564ed9323460 0x564ed9323840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.206+0000 7f7b2e9fb740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564ed9323d80 0x564ed9332440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.206+0000 7f7b2e9fb740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9332980 0x564ed9334d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.206+0000 7f7b2e9fb740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x564ed9288670 con 0x564ed9332980 2024-04-25T00:37:59.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.206+0000 7f7b2e9fb740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x564ed9230970 con 0x564ed9323460 2024-04-25T00:37:59.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.206+0000 7f7b2e9fb740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x564ed9284630 con 0x564ed9323d80 2024-04-25T00:37:59.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.206+0000 7f7b291ad640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564ed9323460 0x564ed9323840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.206+0000 7f7b291ad640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564ed9323460 0x564ed9323840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:38132/0 (socket says 172.21.15.32:38132) 2024-04-25T00:37:59.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.206+0000 7f7b291ad640 1 -- 172.21.15.32:0/1596140961 learned_addr learned my addr 172.21.15.32:0/1596140961 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:59.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.206+0000 7f7b299ae640 1 --2- 172.21.15.32:0/1596140961 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564ed9323d80 0x564ed9332440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.206+0000 7f7b291ad640 1 --2- 172.21.15.32:0/1596140961 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9332980 0x564ed9334d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.207+0000 7f7b299ae640 1 -- 172.21.15.32:0/1596140961 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564ed9323460 msgr2=0x564ed9323840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:59.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.207+0000 7f7b299ae640 1 --2- 172.21.15.32:0/1596140961 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564ed9323460 0x564ed9323840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.207+0000 7f7b299ae640 1 -- 172.21.15.32:0/1596140961 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9332980 msgr2=0x564ed9334d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:59.208 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.207+0000 7f7b299ae640 1 --2- 172.21.15.32:0/1596140961 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9332980 0x564ed9334d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.209 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.207+0000 7f7b299ae640 1 -- 172.21.15.32:0/1596140961 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564ed9254400 con 0x564ed9323d80 2024-04-25T00:37:59.209 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.207+0000 7f7b291ad640 1 --2- 172.21.15.32:0/1596140961 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564ed9323460 0x564ed9323840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:37:59.209 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.207+0000 7f7b1affd640 1 --2- 172.21.15.32:0/314688312 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7b100a2d80 0x7f7b100a5240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.209 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.207+0000 7f7b299ae640 1 --2- 172.21.15.32:0/1596140961 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564ed9323d80 0x564ed9332440 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f7b2002a940 tx=0x7f7b20000bc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=de303f21ea441a3a server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:59.209 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.207+0000 7f7b291ad640 1 --2- 172.21.15.32:0/314688312 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7b100a2d80 0x7f7b100a5240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.214 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.207+0000 7f7b0ffff640 1 -- 172.21.15.32:0/1596140961 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b20035040 con 0x564ed9323d80 2024-04-25T00:37:59.214 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.207+0000 7f7b1affd640 1 -- 172.21.15.32:0/314688312 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f7b1c0cfc90 con 0x7f7b24107720 2024-04-25T00:37:59.214 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.207+0000 7f7b0ffff640 1 -- 172.21.15.32:0/1596140961 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7b20004760 con 0x564ed9323d80 2024-04-25T00:37:59.214 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.207+0000 7f7b0ffff640 1 -- 172.21.15.32:0/1596140961 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b20003830 con 0x564ed9323d80 2024-04-25T00:37:59.214 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.208+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/1596140961 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564ed9323d80 msgr2=0x564ed9332440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:59.214 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.208+0000 7f7b291ad640 1 --2- 172.21.15.32:0/314688312 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7b100a2d80 0x7f7b100a5240 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x564ed9324a10 tx=0x7f7b14005f90 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:59.214 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.208+0000 7f7b2e9fb740 1 --2- 172.21.15.32:0/1596140961 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564ed9323d80 0x564ed9332440 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f7b2002a940 tx=0x7f7b20000bc0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.214 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.208+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/1596140961 shutdown_connections 2024-04-25T00:37:59.214 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.208+0000 7f7b2e9fb740 1 --2- 172.21.15.32:0/1596140961 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9332980 0x564ed9334d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.214 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.208+0000 7f7b2e9fb740 1 --2- 172.21.15.32:0/1596140961 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564ed9323d80 0x564ed9332440 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.214 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.208+0000 7f7b2e9fb740 1 --2- 172.21.15.32:0/1596140961 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564ed9323460 0x564ed9323840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.215 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.208+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/1596140961 >> 172.21.15.32:0/1596140961 conn(0x564ed931f8a0 msgr2=0x564ed931fc80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:59.215 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.208+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/1596140961 shutdown_connections 2024-04-25T00:37:59.215 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.208+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/1596140961 wait complete. 2024-04-25T00:37:59.215 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.208+0000 7f7b2e9fb740 1 -- start start 2024-04-25T00:37:59.215 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.209+0000 7f7b2e9fb740 1 Processor -- start 2024-04-25T00:37:59.215 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.209+0000 7f7b2e9fb740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564ed9323460 0x564ed9332290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.215 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.209+0000 7f7b2e9fb740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564ed9323d80 0x564ed932f580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.215 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.209+0000 7f7b2e9fb740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9332980 0x564ed932fac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.215 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.209+0000 7f7b2e9fb740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x564ed9335760 con 0x564ed9332980 2024-04-25T00:37:59.215 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.209+0000 7f7b2e9fb740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x564ed9244050 con 0x564ed9323d80 2024-04-25T00:37:59.215 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.209+0000 7f7b2e9fb740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x564ed93358e0 con 0x564ed9323460 2024-04-25T00:37:59.215 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.209+0000 7f7b2e9fb740 10 client.0 _ll_register_callbacks cb 0x564ed8f9bbe0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:37:59.215 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.209+0000 7f7b2a1af640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9332980 0x564ed932fac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.215 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.209+0000 7f7b291ad640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564ed9323d80 0x564ed932f580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.209+0000 7f7b2a1af640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9332980 0x564ed932fac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:45046/0 (socket says 172.21.15.32:45046) 2024-04-25T00:37:59.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.209+0000 7f7b2a1af640 1 -- 172.21.15.32:0/3851375393 learned_addr learned my addr 172.21.15.32:0/3851375393 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:59.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.209+0000 7f7b299ae640 1 --2- 172.21.15.32:0/3851375393 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564ed9323460 0x564ed9332290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.210+0000 7f7b299ae640 1 -- 172.21.15.32:0/3851375393 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564ed9323d80 msgr2=0x564ed932f580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:59.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.210+0000 7f7b299ae640 1 --2- 172.21.15.32:0/3851375393 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564ed9323d80 0x564ed932f580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.210+0000 7f7b299ae640 1 -- 172.21.15.32:0/3851375393 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9332980 msgr2=0x564ed932fac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:59.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.210+0000 7f7b299ae640 1 --2- 172.21.15.32:0/3851375393 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9332980 0x564ed932fac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.210+0000 7f7b299ae640 1 -- 172.21.15.32:0/3851375393 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564ed9215dc0 con 0x564ed9323460 2024-04-25T00:37:59.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.210+0000 7f7b291ad640 1 --2- 172.21.15.32:0/3851375393 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564ed9323d80 0x564ed932f580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:37:59.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.210+0000 7f7b299ae640 1 --2- 172.21.15.32:0/3851375393 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564ed9323460 0x564ed9332290 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f7b20009900 tx=0x7f7b20003820 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d4d005c3f7186170 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:59.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.210+0000 7f7b0d7fa640 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:37:59.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.210+0000 7f7b0d7fa640 1 -- 172.21.15.32:0/3851375393 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b20035040 con 0x564ed9323460 2024-04-25T00:37:59.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.210+0000 7f7b0d7fa640 1 -- 172.21.15.32:0/3851375393 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f7b200315b0 con 0x564ed9323460 2024-04-25T00:37:59.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.210+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/3851375393 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f7b20004760 con 0x564ed9323460 2024-04-25T00:37:59.216 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.210+0000 7f7b0d7fa640 1 -- 172.21.15.32:0/3851375393 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7f7b2005a050 con 0x564ed9323460 2024-04-25T00:37:59.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.210+0000 7f7b2e9fb740 10 client.25828 fetch_fsmap learned FSMap version 28 2024-04-25T00:37:59.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.210+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/3851375393 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x564ed8f9c3f0 con 0x564ed9323460 2024-04-25T00:37:59.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.211+0000 7f7b0d7fa640 1 -- 172.21.15.32:0/3851375393 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7b20004760 con 0x564ed9323460 2024-04-25T00:37:59.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.211+0000 7f7b0d7fa640 1 -- 172.21.15.32:0/3851375393 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7f7b20055030 con 0x564ed9323460 2024-04-25T00:37:59.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.211+0000 7f7b2e9fb740 10 client.25828 fetch_fsmap finished waiting for FSMap version 28 2024-04-25T00:37:59.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.211+0000 7f7b2e9fb740 10 client.25828 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-25T00:37:59.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.211+0000 7f7b2e9fb740 20 client.25828 populate_metadata read hostname 'smithi032' 2024-04-25T00:37:59.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.211+0000 7f7b2e9fb740 1 --2- 172.21.15.32:0/3851375393 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x564ed9342870 0x564ed9344c60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.211+0000 7f7b2e9fb740 4 client.25828 mds_command: new command op to 24476 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:37:59.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.211+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/3851375393 --> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x564ed9295bb0 con 0x564ed9342870 2024-04-25T00:37:59.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.211+0000 7f7b291ad640 1 --2- 172.21.15.32:0/3851375393 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x564ed9342870 0x564ed9344c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.211+0000 7f7b0d7fa640 10 client.25828 ms_handle_connect on v2:172.21.15.121:6836/3456022860 2024-04-25T00:37:59.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.211+0000 7f7b291ad640 1 --2- 172.21.15.32:0/3851375393 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x564ed9342870 0x564ed9344c60 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=e394aaed5c746b15 server_cookie=84cc25c75a5a1448 in_seq=0 out_seq=0 2024-04-25T00:37:59.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.212+0000 7f7b0d7fa640 1 -- 172.21.15.32:0/3851375393 <== mds.2 v2:172.21.15.121:6836/3456022860 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x564ed9295bb0 con 0x564ed9342870 2024-04-25T00:37:59.217 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.212+0000 7f7b0d7fa640 10 client.25828 handle_command_reply: tid=0 2024-04-25T00:37:59.218 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.212+0000 7f7b2e9fb740 1 client.25828 shutdown 2024-04-25T00:37:59.218 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.212+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/3851375393 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564ed9323460 msgr2=0x564ed9332290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:59.218 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.212+0000 7f7b2e9fb740 1 --2- 172.21.15.32:0/3851375393 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564ed9323460 0x564ed9332290 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f7b20009900 tx=0x7f7b20003820 comp rx=0 tx=0).stop 2024-04-25T00:37:59.218 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.212+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/3851375393 shutdown_connections 2024-04-25T00:37:59.218 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.212+0000 7f7b2e9fb740 1 --2- 172.21.15.32:0/3851375393 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x564ed9342870 0x564ed9344c60 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.218 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.212+0000 7f7b2e9fb740 1 --2- 172.21.15.32:0/3851375393 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9332980 0x564ed932fac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.218 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.212+0000 7f7b2e9fb740 1 --2- 172.21.15.32:0/3851375393 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564ed9323d80 0x564ed932f580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.218 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.212+0000 7f7b0d7fa640 0 client.25828 ms_handle_reset on v2:172.21.15.121:6836/3456022860 2024-04-25T00:37:59.218 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.212+0000 7f7b2e9fb740 1 --2- 172.21.15.32:0/3851375393 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564ed9323460 0x564ed9332290 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.218 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.212+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/3851375393 >> 172.21.15.32:0/3851375393 conn(0x564ed931f8a0 msgr2=0x564ed9325650 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:59.218 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.212+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/3851375393 shutdown_connections 2024-04-25T00:37:59.218 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.212+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/3851375393 wait complete. 2024-04-25T00:37:59.218 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.213+0000 7f7b2e9fb740 20 client.25828 trim_cache size 0 max 16384 2024-04-25T00:37:59.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.226+0000 7f7b2e9fb740 1 Processor -- start 2024-04-25T00:37:59.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.226+0000 7f7b2e9fb740 1 -- start start 2024-04-25T00:37:59.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.226+0000 7f7b2e9fb740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564ed9323460 0x564ed933e690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.226+0000 7f7b2e9fb740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9323d80 0x564ed932e6e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.226+0000 7f7b2e9fb740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564ed9332980 0x564ed9331dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.226+0000 7f7b2e9fb740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f7b241af340 con 0x564ed9323d80 2024-04-25T00:37:59.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.226+0000 7f7b2e9fb740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f7b241acc10 con 0x564ed9332980 2024-04-25T00:37:59.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.226+0000 7f7b2e9fb740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f7b241aaa60 con 0x564ed9323460 2024-04-25T00:37:59.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.227+0000 7f7b291ad640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9323d80 0x564ed932e6e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.227+0000 7f7b291ad640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9323d80 0x564ed932e6e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:45062/0 (socket says 172.21.15.32:45062) 2024-04-25T00:37:59.229 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.227+0000 7f7b291ad640 1 -- 172.21.15.32:0/3601212277 learned_addr learned my addr 172.21.15.32:0/3601212277 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:59.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.227+0000 7f7b299ae640 1 --2- 172.21.15.32:0/3601212277 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564ed9323460 0x564ed933e690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.227+0000 7f7b2a1af640 1 --2- 172.21.15.32:0/3601212277 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564ed9332980 0x564ed9331dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.227+0000 7f7b291ad640 1 -- 172.21.15.32:0/3601212277 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564ed9323460 msgr2=0x564ed933e690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:59.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.227+0000 7f7b291ad640 1 --2- 172.21.15.32:0/3601212277 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564ed9323460 0x564ed933e690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.227+0000 7f7b291ad640 1 -- 172.21.15.32:0/3601212277 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564ed9332980 msgr2=0x564ed9331dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:59.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.227+0000 7f7b291ad640 1 --2- 172.21.15.32:0/3601212277 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564ed9332980 0x564ed9331dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.227+0000 7f7b291ad640 1 -- 172.21.15.32:0/3601212277 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564ed90667d0 con 0x564ed9323d80 2024-04-25T00:37:59.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.227+0000 7f7b291ad640 1 --2- 172.21.15.32:0/3601212277 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9323d80 0x564ed932e6e0 secure :-1 s=READY pgs=640 cs=0 l=1 rev1=1 crypto rx=0x7f7b14009fd0 tx=0x7f7b140046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3efbfb4d2670d5d server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:59.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.227+0000 7f7b0f7fe640 1 -- 172.21.15.32:0/3601212277 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b1402d470 con 0x564ed9323d80 2024-04-25T00:37:59.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.227+0000 7f7b0f7fe640 1 -- 172.21.15.32:0/3601212277 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7b14032070 con 0x564ed9323d80 2024-04-25T00:37:59.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.227+0000 7f7b0f7fe640 1 -- 172.21.15.32:0/3601212277 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b14020410 con 0x564ed9323d80 2024-04-25T00:37:59.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.228+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/3601212277 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9323d80 msgr2=0x564ed932e6e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:59.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.228+0000 7f7b2e9fb740 1 --2- 172.21.15.32:0/3601212277 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9323d80 0x564ed932e6e0 secure :-1 s=READY pgs=640 cs=0 l=1 rev1=1 crypto rx=0x7f7b14009fd0 tx=0x7f7b140046c0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.228+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/3601212277 shutdown_connections 2024-04-25T00:37:59.230 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.228+0000 7f7b2e9fb740 1 --2- 172.21.15.32:0/3601212277 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564ed9332980 0x564ed9331dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.228+0000 7f7b2e9fb740 1 --2- 172.21.15.32:0/3601212277 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9323d80 0x564ed932e6e0 unknown :-1 s=CLOSED pgs=640 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.228+0000 7f7b2e9fb740 1 --2- 172.21.15.32:0/3601212277 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564ed9323460 0x564ed933e690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.228+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/3601212277 >> 172.21.15.32:0/3601212277 conn(0x564ed931f8a0 msgr2=0x564ed9324dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:59.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.228+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/3601212277 shutdown_connections 2024-04-25T00:37:59.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.228+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/3601212277 wait complete. 2024-04-25T00:37:59.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.228+0000 7f7b2e9fb740 1 -- start start 2024-04-25T00:37:59.231 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.229+0000 7f7b2e9fb740 1 Processor -- start 2024-04-25T00:37:59.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.229+0000 7f7b2e9fb740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564ed9323460 0x564ed933ad90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.234 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.229+0000 7f7b2e9fb740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564ed9323d80 0x564ed932bdf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.229+0000 7f7b2e9fb740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9332980 0x564ed932c330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.229+0000 7f7b2e9fb740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f7b241ab0d0 con 0x564ed9332980 2024-04-25T00:37:59.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.229+0000 7f7b2e9fb740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f7b241ac540 con 0x564ed9323d80 2024-04-25T00:37:59.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.229+0000 7f7b2e9fb740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f7b241ab7a0 con 0x564ed9323460 2024-04-25T00:37:59.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.229+0000 7f7b2e9fb740 10 client.0 _ll_register_callbacks cb 0x7f7b14009ea0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:37:59.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.229+0000 7f7b2a1af640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9332980 0x564ed932c330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.229+0000 7f7b2a1af640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9332980 0x564ed932c330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:45072/0 (socket says 172.21.15.32:45072) 2024-04-25T00:37:59.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.229+0000 7f7b2a1af640 1 -- 172.21.15.32:0/2773771271 learned_addr learned my addr 172.21.15.32:0/2773771271 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:59.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.229+0000 7f7b299ae640 1 --2- 172.21.15.32:0/2773771271 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564ed9323460 0x564ed933ad90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.229+0000 7f7b291ad640 1 --2- 172.21.15.32:0/2773771271 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564ed9323d80 0x564ed932bdf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.229+0000 7f7b2a1af640 1 -- 172.21.15.32:0/2773771271 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564ed9323460 msgr2=0x564ed933ad90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:59.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.229+0000 7f7b2a1af640 1 --2- 172.21.15.32:0/2773771271 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564ed9323460 0x564ed933ad90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.229+0000 7f7b2a1af640 1 -- 172.21.15.32:0/2773771271 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564ed9323d80 msgr2=0x564ed932bdf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:59.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.229+0000 7f7b2a1af640 1 --2- 172.21.15.32:0/2773771271 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564ed9323d80 0x564ed932bdf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.229+0000 7f7b2a1af640 1 -- 172.21.15.32:0/2773771271 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564ed932c870 con 0x564ed9332980 2024-04-25T00:37:59.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.229+0000 7f7b291ad640 1 --2- 172.21.15.32:0/2773771271 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564ed9323d80 0x564ed932bdf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:37:59.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.229+0000 7f7b2a1af640 1 --2- 172.21.15.32:0/2773771271 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9332980 0x564ed932c330 secure :-1 s=READY pgs=641 cs=0 l=1 rev1=1 crypto rx=0x7f7b1c0ccfa0 tx=0x7f7b1c09b820 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a7b4e1e189e268e6 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:59.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.229+0000 7f7b0d7fa640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:37:59.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.230+0000 7f7b0d7fa640 1 -- 172.21.15.32:0/2773771271 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b1c0a47e0 con 0x564ed9332980 2024-04-25T00:37:59.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.230+0000 7f7b0d7fa640 1 -- 172.21.15.32:0/2773771271 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7f7b1c005310 con 0x564ed9332980 2024-04-25T00:37:59.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.230+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/2773771271 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f7b14032070 con 0x564ed9332980 2024-04-25T00:37:59.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.230+0000 7f7b0d7fa640 1 -- 172.21.15.32:0/2773771271 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7f7b1c082050 con 0x564ed9332980 2024-04-25T00:37:59.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.230+0000 7f7b2e9fb740 10 client.16308 fetch_fsmap learned FSMap version 28 2024-04-25T00:37:59.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.230+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/2773771271 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x564ed93a87b0 con 0x564ed9332980 2024-04-25T00:37:59.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.230+0000 7f7b0d7fa640 1 -- 172.21.15.32:0/2773771271 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7b14032070 con 0x564ed9332980 2024-04-25T00:37:59.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.230+0000 7f7b0d7fa640 1 -- 172.21.15.32:0/2773771271 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7f7b1c09b3e0 con 0x564ed9332980 2024-04-25T00:37:59.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.230+0000 7f7b2e9fb740 10 client.16308 fetch_fsmap finished waiting for FSMap version 28 2024-04-25T00:37:59.236 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.230+0000 7f7b2e9fb740 10 client.16308 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-25T00:37:59.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.230+0000 7f7b2e9fb740 20 client.16308 populate_metadata read hostname 'smithi032' 2024-04-25T00:37:59.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.230+0000 7f7b2e9fb740 1 --2- 172.21.15.32:0/2773771271 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x564ed93a8f20 0x564ed93ab340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.230+0000 7f7b2e9fb740 4 client.16308 mds_command: new command op to 24476 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-25T00:37:59.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.230+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/2773771271 --> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] -- command(tid 0: {"prefix": "damage ls"}) -- 0x564ed928ff00 con 0x564ed93a8f20 2024-04-25T00:37:59.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.230+0000 7f7b299ae640 1 --2- 172.21.15.32:0/2773771271 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x564ed93a8f20 0x564ed93ab340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.231+0000 7f7b299ae640 1 --2- 172.21.15.32:0/2773771271 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x564ed93a8f20 0x564ed93ab340 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=e8d5fafd4d4dc519 server_cookie=5289422e188c068e in_seq=0 out_seq=0 2024-04-25T00:37:59.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.231+0000 7f7b0d7fa640 10 client.16308 ms_handle_connect on v2:172.21.15.121:6836/3456022860 2024-04-25T00:37:59.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.231+0000 7f7b0d7fa640 1 -- 172.21.15.32:0/2773771271 <== mds.2 v2:172.21.15.121:6836/3456022860 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x564ed928ff00 con 0x564ed93a8f20 2024-04-25T00:37:59.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.231+0000 7f7b0d7fa640 10 client.16308 handle_command_reply: tid=0 2024-04-25T00:37:59.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.231+0000 7f7b2e9fb740 1 client.16308 shutdown 2024-04-25T00:37:59.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.231+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/2773771271 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9332980 msgr2=0x564ed932c330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:59.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.231+0000 7f7b2e9fb740 1 --2- 172.21.15.32:0/2773771271 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9332980 0x564ed932c330 secure :-1 s=READY pgs=641 cs=0 l=1 rev1=1 crypto rx=0x7f7b1c0ccfa0 tx=0x7f7b1c09b820 comp rx=0 tx=0).stop 2024-04-25T00:37:59.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.231+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/2773771271 shutdown_connections 2024-04-25T00:37:59.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.231+0000 7f7b2e9fb740 1 --2- 172.21.15.32:0/2773771271 >> [v2:172.21.15.121:6836/3456022860,v1:172.21.15.121:6837/3456022860] conn(0x564ed93a8f20 0x564ed93ab340 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.231+0000 7f7b2e9fb740 1 --2- 172.21.15.32:0/2773771271 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x564ed9332980 0x564ed932c330 unknown :-1 s=CLOSED pgs=641 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.231+0000 7f7b0d7fa640 0 client.16308 ms_handle_reset on v2:172.21.15.121:6836/3456022860 2024-04-25T00:37:59.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.231+0000 7f7b2e9fb740 1 --2- 172.21.15.32:0/2773771271 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x564ed9323d80 0x564ed932bdf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.231+0000 7f7b2e9fb740 1 --2- 172.21.15.32:0/2773771271 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564ed9323460 0x564ed933ad90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.231+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/2773771271 >> 172.21.15.32:0/2773771271 conn(0x564ed931f8a0 msgr2=0x564ed9324dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:59.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.231+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/2773771271 shutdown_connections 2024-04-25T00:37:59.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.231+0000 7f7b2e9fb740 1 -- 172.21.15.32:0/2773771271 wait complete. 2024-04-25T00:37:59.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.231+0000 7f7b2e9fb740 20 client.16308 trim_cache size 0 max 16384 2024-04-25T00:37:59.238 INFO:teuthology.orchestra.run.smithi032.stdout:[] 2024-04-25T00:37:59.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.232+0000 7f7b2bc12640 1 -- 172.21.15.32:0/314688312 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7b100a2d80 msgr2=0x7f7b100a5240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:59.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.232+0000 7f7b2bc12640 1 --2- 172.21.15.32:0/314688312 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7b100a2d80 0x7f7b100a5240 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x564ed9324a10 tx=0x7f7b14005f90 comp rx=0 tx=0).stop 2024-04-25T00:37:59.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.232+0000 7f7b2bc12640 1 -- 172.21.15.32:0/314688312 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b24107720 msgr2=0x7f7b2419c960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:59.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.232+0000 7f7b2bc12640 1 --2- 172.21.15.32:0/314688312 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b24107720 0x7f7b2419c960 secure :-1 s=READY pgs=639 cs=0 l=1 rev1=1 crypto rx=0x7f7b1c00eb10 tx=0x7f7b1c00c2d0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.232+0000 7f7b2bc12640 1 -- 172.21.15.32:0/314688312 shutdown_connections 2024-04-25T00:37:59.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.232+0000 7f7b2bc12640 1 --2- 172.21.15.32:0/314688312 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f7b100a2d80 0x7f7b100a5240 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.239 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.232+0000 7f7b2bc12640 1 --2- 172.21.15.32:0/314688312 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7b24107720 0x7f7b2419c960 unknown :-1 s=CLOSED pgs=639 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.240 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.232+0000 7f7b2bc12640 1 --2- 172.21.15.32:0/314688312 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7b241020d0 0x7f7b241985e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.240 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.232+0000 7f7b2bc12640 1 --2- 172.21.15.32:0/314688312 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7b24101710 0x7f7b241980a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.240 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.232+0000 7f7b2bc12640 1 -- 172.21.15.32:0/314688312 >> 172.21.15.32:0/314688312 conn(0x7f7b24078b90 msgr2=0x7f7b240ff1c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:59.240 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.232+0000 7f7b2bc12640 1 -- 172.21.15.32:0/314688312 shutdown_connections 2024-04-25T00:37:59.240 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.232+0000 7f7b2bc12640 1 -- 172.21.15.32:0/314688312 wait complete. 2024-04-25T00:37:59.245 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-25T00:37:59.391 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.389+0000 7fd2af390640 1 -- 172.21.15.32:0/533695041 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd2a810ff90 msgr2=0x7fd2a810da80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:59.391 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.389+0000 7fd2af390640 1 --2- 172.21.15.32:0/533695041 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd2a810ff90 0x7fd2a810da80 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7fd298009900 tx=0x7fd29802a440 comp rx=0 tx=0).stop 2024-04-25T00:37:59.394 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.391+0000 7fd2af390640 1 -- 172.21.15.32:0/533695041 shutdown_connections 2024-04-25T00:37:59.394 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.391+0000 7fd2af390640 1 --2- 172.21.15.32:0/533695041 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd2a811af20 0x7fd2a811d310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.394 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.391+0000 7fd2af390640 1 --2- 172.21.15.32:0/533695041 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd2a810dfc0 0x7fd2a810e440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.394 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.391+0000 7fd2af390640 1 --2- 172.21.15.32:0/533695041 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd2a810ff90 0x7fd2a810da80 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.394 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.391+0000 7fd2af390640 1 -- 172.21.15.32:0/533695041 >> 172.21.15.32:0/533695041 conn(0x7fd2a806c850 msgr2=0x7fd2a806cc60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:59.403 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.401+0000 7fd2af390640 1 -- 172.21.15.32:0/533695041 shutdown_connections 2024-04-25T00:37:59.406 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.404+0000 7fd2af390640 1 -- 172.21.15.32:0/533695041 wait complete. 2024-04-25T00:37:59.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.407+0000 7fd2af390640 1 Processor -- start 2024-04-25T00:37:59.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.411+0000 7fd2af390640 1 -- start start 2024-04-25T00:37:59.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.411+0000 7fd2af390640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd2a810dfc0 0x7fd2a81a6e80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.411+0000 7fd2af390640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd2a810ff90 0x7fd2a81a73c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.411+0000 7fd2af390640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd2a811af20 0x7fd2a81ab6f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.413 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.411+0000 7fd2af390640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fd2a8120960 con 0x7fd2a811af20 2024-04-25T00:37:59.413 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.411+0000 7fd2af390640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fd2a81207e0 con 0x7fd2a810ff90 2024-04-25T00:37:59.413 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.411+0000 7fd2af390640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fd2a8120ae0 con 0x7fd2a810dfc0 2024-04-25T00:37:59.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.412+0000 7fd2ad12c640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd2a810dfc0 0x7fd2a81a6e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.412+0000 7fd2ad12c640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd2a810dfc0 0x7fd2a81a6e80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:39952/0 (socket says 172.21.15.32:39952) 2024-04-25T00:37:59.414 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.412+0000 7fd2ad12c640 1 -- 172.21.15.32:0/2307721654 learned_addr learned my addr 172.21.15.32:0/2307721654 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:59.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.412+0000 7fd2ac92b640 1 --2- 172.21.15.32:0/2307721654 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd2a810ff90 0x7fd2a81a73c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.413+0000 7fd2ad12c640 1 -- 172.21.15.32:0/2307721654 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd2a810ff90 msgr2=0x7fd2a81a73c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:59.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.413+0000 7fd2ad12c640 1 --2- 172.21.15.32:0/2307721654 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd2a810ff90 0x7fd2a81a73c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.413+0000 7fd2ad12c640 1 -- 172.21.15.32:0/2307721654 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd2a811af20 msgr2=0x7fd2a81ab6f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:59.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.413+0000 7fd2ad12c640 1 --2- 172.21.15.32:0/2307721654 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd2a811af20 0x7fd2a81ab6f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.413+0000 7fd2ad12c640 1 -- 172.21.15.32:0/2307721654 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd2a81abe20 con 0x7fd2a810dfc0 2024-04-25T00:37:59.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.413+0000 7fd2ad12c640 1 --2- 172.21.15.32:0/2307721654 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd2a810dfc0 0x7fd2a81a6e80 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7fd29802a9b0 tx=0x7fd298031540 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:59.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.417+0000 7fd29e7fc640 1 -- 172.21.15.32:0/2307721654 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd29803f030 con 0x7fd2a810dfc0 2024-04-25T00:37:59.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.417+0000 7fd2af390640 1 -- 172.21.15.32:0/2307721654 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd2a81ac0b0 con 0x7fd2a810dfc0 2024-04-25T00:37:59.419 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.417+0000 7fd2af390640 1 -- 172.21.15.32:0/2307721654 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd2a81b39a0 con 0x7fd2a810dfc0 2024-04-25T00:37:59.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.422+0000 7fd29e7fc640 1 -- 172.21.15.32:0/2307721654 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd29803b460 con 0x7fd2a810dfc0 2024-04-25T00:37:59.423 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.422+0000 7fd29e7fc640 1 -- 172.21.15.32:0/2307721654 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd298035070 con 0x7fd2a810dfc0 2024-04-25T00:37:59.427 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.425+0000 7fd2b2178740 1 Processor -- start 2024-04-25T00:37:59.427 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.425+0000 7fd2b2178740 1 -- start start 2024-04-25T00:37:59.427 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.425+0000 7fd2b2178740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559a55ce5580 0x559a55ce5960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.427 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.426+0000 7fd2b2178740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x559a55ce5ea0 0x559a55cf0030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.427 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.426+0000 7fd2b2178740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55cf0570 0x559a55cf2960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.427 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.426+0000 7fd2b2178740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x559a55c4a670 con 0x559a55cf0570 2024-04-25T00:37:59.427 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.426+0000 7fd2b2178740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x559a55bf2970 con 0x559a55ce5ea0 2024-04-25T00:37:59.428 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.426+0000 7fd2b2178740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x559a55c46630 con 0x559a55ce5580 2024-04-25T00:37:59.429 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.427+0000 7fd29e7fc640 1 -- 172.21.15.32:0/2307721654 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7fd298042030 con 0x7fd2a810dfc0 2024-04-25T00:37:59.429 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.427+0000 7fd2ad92d640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559a55ce5580 0x559a55ce5960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.429 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.428+0000 7fd29e7fc640 1 --2- 172.21.15.32:0/2307721654 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd2940a2f00 0x7fd2940a53c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.429 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.428+0000 7fd29e7fc640 1 -- 172.21.15.32:0/2307721654 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fd2980efb80 con 0x7fd2a810dfc0 2024-04-25T00:37:59.429 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.428+0000 7fd2ac92b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55cf0570 0x559a55cf2960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.429 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.428+0000 7fd2ac92b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55cf0570 0x559a55cf2960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:45104/0 (socket says 172.21.15.32:45104) 2024-04-25T00:37:59.429 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.428+0000 7fd2ac92b640 1 -- 172.21.15.32:0/4278304155 learned_addr learned my addr 172.21.15.32:0/4278304155 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:59.429 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.428+0000 7fd2ad12c640 1 --2- 172.21.15.32:0/4278304155 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x559a55ce5ea0 0x559a55cf0030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.430 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.429+0000 7fd2ad12c640 1 -- 172.21.15.32:0/4278304155 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559a55ce5580 msgr2=0x559a55ce5960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:59.430 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.429+0000 7fd2ad12c640 1 --2- 172.21.15.32:0/4278304155 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559a55ce5580 0x559a55ce5960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.430 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.429+0000 7fd2ad12c640 1 -- 172.21.15.32:0/4278304155 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55cf0570 msgr2=0x559a55cf2960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:59.430 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.429+0000 7fd2ad12c640 1 --2- 172.21.15.32:0/4278304155 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55cf0570 0x559a55cf2960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.431 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.429+0000 7fd2ad12c640 1 -- 172.21.15.32:0/4278304155 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x559a55c16400 con 0x559a55ce5ea0 2024-04-25T00:37:59.431 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.430+0000 7fd2ac92b640 1 --2- 172.21.15.32:0/4278304155 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55cf0570 0x559a55cf2960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:37:59.431 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.430+0000 7fd2ad92d640 1 --2- 172.21.15.32:0/2307721654 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd2940a2f00 0x7fd2940a53c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.431 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.430+0000 7fd2ad12c640 1 --2- 172.21.15.32:0/4278304155 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x559a55ce5ea0 0x559a55cf0030 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7fd2980ecf20 tx=0x7fd2980bc6e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=bed07de958b5b2ea server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:59.431 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.431+0000 7fd2937fe640 1 -- 172.21.15.32:0/4278304155 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd298049040 con 0x559a55ce5ea0 2024-04-25T00:37:59.432 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.431+0000 7fd2937fe640 1 -- 172.21.15.32:0/4278304155 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd2980bd3c0 con 0x559a55ce5ea0 2024-04-25T00:37:59.433 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.432+0000 7fd2b2178740 1 -- 172.21.15.32:0/4278304155 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x559a55ce5ea0 msgr2=0x559a55cf0030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:59.433 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.432+0000 7fd2b2178740 1 --2- 172.21.15.32:0/4278304155 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x559a55ce5ea0 0x559a55cf0030 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7fd2980ecf20 tx=0x7fd2980bc6e0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.435 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.433+0000 7fd2ad92d640 1 --2- 172.21.15.32:0/2307721654 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd2940a2f00 0x7fd2940a53c0 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7fd2a4009d30 tx=0x7fd2a40046c0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:59.435 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.433+0000 7fd2b2178740 1 -- 172.21.15.32:0/4278304155 shutdown_connections 2024-04-25T00:37:59.435 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.433+0000 7fd2b2178740 1 --2- 172.21.15.32:0/4278304155 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55cf0570 0x559a55cf2960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.435 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.434+0000 7fd2b2178740 1 --2- 172.21.15.32:0/4278304155 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x559a55ce5ea0 0x559a55cf0030 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.435 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.434+0000 7fd2b2178740 1 --2- 172.21.15.32:0/4278304155 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559a55ce5580 0x559a55ce5960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.435 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.434+0000 7fd2b2178740 1 -- 172.21.15.32:0/4278304155 >> 172.21.15.32:0/4278304155 conn(0x559a55ce18a0 msgr2=0x559a55ce1c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:59.439 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.439+0000 7fd2b2178740 1 -- 172.21.15.32:0/4278304155 shutdown_connections 2024-04-25T00:37:59.442 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.441+0000 7fd2b2178740 1 -- 172.21.15.32:0/4278304155 wait complete. 2024-04-25T00:37:59.443 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.442+0000 7fd2b2178740 1 -- start start 2024-04-25T00:37:59.443 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.443+0000 7fd2b2178740 1 Processor -- start 2024-04-25T00:37:59.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.449+0000 7fd2b2178740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55ce5580 0x559a55cefde0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.449+0000 7fd2b2178740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559a55ce5ea0 0x559a55ced0d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.450 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.450+0000 7fd2b2178740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x559a55cf0570 0x559a55ced610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.450+0000 7fd2b2178740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x559a55cf3120 con 0x559a55ce5580 2024-04-25T00:37:59.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.450+0000 7fd2b2178740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x559a55c06050 con 0x559a55cf0570 2024-04-25T00:37:59.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.450+0000 7fd2b2178740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x559a55cf32a0 con 0x559a55ce5ea0 2024-04-25T00:37:59.451 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.450+0000 7fd2b2178740 10 client.0 _ll_register_callbacks cb 0x559a5595dbe0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:37:59.456 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.455+0000 7fd2ac92b640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559a55ce5ea0 0x559a55ced0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.456 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.455+0000 7fd2ad12c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55ce5580 0x559a55cefde0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.456 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.455+0000 7fd2ad12c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55ce5580 0x559a55cefde0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:45114/0 (socket says 172.21.15.32:45114) 2024-04-25T00:37:59.456 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.455+0000 7fd2ad12c640 1 -- 172.21.15.32:0/997942287 learned_addr learned my addr 172.21.15.32:0/997942287 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:59.456 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.455+0000 7fd2ad92d640 1 --2- 172.21.15.32:0/997942287 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x559a55cf0570 0x559a55ced610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.457 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.456+0000 7fd2ac92b640 1 -- 172.21.15.32:0/997942287 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x559a55cf0570 msgr2=0x559a55ced610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:59.457 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.456+0000 7fd2ac92b640 1 --2- 172.21.15.32:0/997942287 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x559a55cf0570 0x559a55ced610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.458 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.456+0000 7fd2ac92b640 1 -- 172.21.15.32:0/997942287 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55ce5580 msgr2=0x559a55cefde0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:59.458 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.456+0000 7fd2ac92b640 1 --2- 172.21.15.32:0/997942287 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55ce5580 0x559a55cefde0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.458 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.456+0000 7fd2ac92b640 1 -- 172.21.15.32:0/997942287 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x559a55bd7dc0 con 0x559a55ce5ea0 2024-04-25T00:37:59.458 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.458+0000 7fd2ac92b640 1 --2- 172.21.15.32:0/997942287 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559a55ce5ea0 0x559a55ced0d0 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x559a55ce2a30 tx=0x7fd2a00111b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d920a61f787592d0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:59.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.459+0000 7fd290ff9640 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:37:59.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.459+0000 7fd290ff9640 1 -- 172.21.15.32:0/997942287 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd2a001a070 con 0x559a55ce5ea0 2024-04-25T00:37:59.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.460+0000 7fd2b2178740 1 -- 172.21.15.32:0/997942287 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd2980bd3c0 con 0x559a55ce5ea0 2024-04-25T00:37:59.463 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.460+0000 7fd290ff9640 1 -- 172.21.15.32:0/997942287 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fd2a0011dc0 con 0x559a55ce5ea0 2024-04-25T00:37:59.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.463+0000 7fd290ff9640 1 -- 172.21.15.32:0/997942287 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7fd2a0024050 con 0x559a55ce5ea0 2024-04-25T00:37:59.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.463+0000 7fd2b2178740 10 client.25846 fetch_fsmap learned FSMap version 28 2024-04-25T00:37:59.464 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.463+0000 7fd2b2178740 1 -- 172.21.15.32:0/997942287 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x559a5595e3f0 con 0x559a55ce5ea0 2024-04-25T00:37:59.466 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.465+0000 7fd290ff9640 1 -- 172.21.15.32:0/997942287 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd2980bd3c0 con 0x559a55ce5ea0 2024-04-25T00:37:59.468 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.467+0000 7fd290ff9640 1 -- 172.21.15.32:0/997942287 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7fd2a001f760 con 0x559a55ce5ea0 2024-04-25T00:37:59.468 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.467+0000 7fd2b2178740 10 client.25846 fetch_fsmap finished waiting for FSMap version 28 2024-04-25T00:37:59.468 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.467+0000 7fd2b2178740 10 client.25846 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-25T00:37:59.468 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.467+0000 7fd2b2178740 20 client.25846 populate_metadata read hostname 'smithi032' 2024-04-25T00:37:59.468 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.467+0000 7fd2b2178740 1 --2- 172.21.15.32:0/997942287 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x559a55d11620 0x559a55d13a40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.468 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.467+0000 7fd2b2178740 4 client.25846 mds_command: new command op to 24481 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:37:59.468 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.467+0000 7fd2b2178740 1 -- 172.21.15.32:0/997942287 --> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x559a55c57bb0 con 0x559a55d11620 2024-04-25T00:37:59.469 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.468+0000 7fd2ad12c640 1 --2- 172.21.15.32:0/997942287 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x559a55d11620 0x559a55d13a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.473 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.473+0000 7fd2ad12c640 1 --2- 172.21.15.32:0/997942287 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x559a55d11620 0x559a55d13a40 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=784afe80a5eb57ed server_cookie=e201b4a9b3f4af75 in_seq=0 out_seq=0 2024-04-25T00:37:59.473 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.473+0000 7fd290ff9640 10 client.25846 ms_handle_connect on v2:172.21.15.130:6838/4267125809 2024-04-25T00:37:59.477 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.475+0000 7fd290ff9640 1 -- 172.21.15.32:0/997942287 <== mds.1 v2:172.21.15.130:6838/4267125809 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x559a55c57bb0 con 0x559a55d11620 2024-04-25T00:37:59.477 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.476+0000 7fd290ff9640 10 client.25846 handle_command_reply: tid=0 2024-04-25T00:37:59.478 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.477+0000 7fd2b2178740 1 client.25846 shutdown 2024-04-25T00:37:59.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.480+0000 7fd2b2178740 1 -- 172.21.15.32:0/997942287 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559a55ce5ea0 msgr2=0x559a55ced0d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:59.481 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.480+0000 7fd2b2178740 1 --2- 172.21.15.32:0/997942287 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559a55ce5ea0 0x559a55ced0d0 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x559a55ce2a30 tx=0x7fd2a00111b0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.481+0000 7fd2b2178740 1 -- 172.21.15.32:0/997942287 shutdown_connections 2024-04-25T00:37:59.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.481+0000 7fd2b2178740 1 --2- 172.21.15.32:0/997942287 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x559a55cf0570 0x559a55ced610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.481+0000 7fd2b2178740 1 --2- 172.21.15.32:0/997942287 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559a55ce5ea0 0x559a55ced0d0 secure :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0x559a55ce2a30 tx=0x7fd2a00111b0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.481+0000 7fd2b2178740 1 --2- 172.21.15.32:0/997942287 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x559a55d11620 0x559a55d13a40 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.481+0000 7fd2b2178740 1 --2- 172.21.15.32:0/997942287 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55ce5580 0x559a55cefde0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.481+0000 7fd2b2178740 1 -- 172.21.15.32:0/997942287 >> 172.21.15.32:0/997942287 conn(0x559a55ce18a0 msgr2=0x559a55ce0b40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:59.482 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.481+0000 7fd290ff9640 0 client.25846 ms_handle_reset on v2:172.21.15.130:6838/4267125809 2024-04-25T00:37:59.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.482+0000 7fd2b2178740 1 -- 172.21.15.32:0/997942287 shutdown_connections 2024-04-25T00:37:59.483 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.482+0000 7fd2b2178740 1 -- 172.21.15.32:0/997942287 wait complete. 2024-04-25T00:37:59.484 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.484+0000 7fd2b2178740 20 client.25846 trim_cache size 0 max 16384 2024-04-25T00:37:59.510 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.509+0000 7fd2b2178740 1 Processor -- start 2024-04-25T00:37:59.510 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.509+0000 7fd2b2178740 1 -- start start 2024-04-25T00:37:59.511 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.509+0000 7fd2b2178740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x559a55ce18a0 0x559a55ce8860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.511 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.509+0000 7fd2b2178740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55ce5580 0x559a55ceb9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.511 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.509+0000 7fd2b2178740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559a55cf0570 0x559a55cef910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.511 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.510+0000 7fd2b2178740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fd28c012800 con 0x559a55ce5580 2024-04-25T00:37:59.511 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.510+0000 7fd2b2178740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fd28c0100d0 con 0x559a55ce18a0 2024-04-25T00:37:59.511 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.510+0000 7fd2b2178740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fd28c00dfd0 con 0x559a55cf0570 2024-04-25T00:37:59.511 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.510+0000 7fd2ac92b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55ce5580 0x559a55ceb9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.511 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.510+0000 7fd2ac92b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55ce5580 0x559a55ceb9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:45118/0 (socket says 172.21.15.32:45118) 2024-04-25T00:37:59.511 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.510+0000 7fd2ac92b640 1 -- 172.21.15.32:0/1647230314 learned_addr learned my addr 172.21.15.32:0/1647230314 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:59.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.510+0000 7fd2ad12c640 1 --2- 172.21.15.32:0/1647230314 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x559a55ce18a0 0x559a55ce8860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.511+0000 7fd2ad92d640 1 --2- 172.21.15.32:0/1647230314 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559a55cf0570 0x559a55cef910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.511+0000 7fd2ac92b640 1 -- 172.21.15.32:0/1647230314 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559a55cf0570 msgr2=0x559a55cef910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:59.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.511+0000 7fd2ac92b640 1 --2- 172.21.15.32:0/1647230314 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559a55cf0570 0x559a55cef910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.511+0000 7fd2ac92b640 1 -- 172.21.15.32:0/1647230314 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x559a55ce18a0 msgr2=0x559a55ce8860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:59.512 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.511+0000 7fd2ac92b640 1 --2- 172.21.15.32:0/1647230314 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x559a55ce18a0 0x559a55ce8860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.511+0000 7fd2ac92b640 1 -- 172.21.15.32:0/1647230314 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x559a55cec080 con 0x559a55ce5580 2024-04-25T00:37:59.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.512+0000 7fd2ac92b640 1 --2- 172.21.15.32:0/1647230314 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55ce5580 0x559a55ceb9a0 secure :-1 s=READY pgs=642 cs=0 l=1 rev1=1 crypto rx=0x7fd2a001fbb0 tx=0x7fd2a001ff30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=162a682e1b92c4b server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:59.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.512+0000 7fd292ffd640 1 -- 172.21.15.32:0/1647230314 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd2a002d020 con 0x559a55ce5580 2024-04-25T00:37:59.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.512+0000 7fd292ffd640 1 -- 172.21.15.32:0/1647230314 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd2a001f920 con 0x559a55ce5580 2024-04-25T00:37:59.513 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.512+0000 7fd292ffd640 1 -- 172.21.15.32:0/1647230314 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd2a001a040 con 0x559a55ce5580 2024-04-25T00:37:59.514 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.513+0000 7fd2b2178740 1 -- 172.21.15.32:0/1647230314 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55ce5580 msgr2=0x559a55ceb9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:59.514 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.513+0000 7fd2b2178740 1 --2- 172.21.15.32:0/1647230314 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55ce5580 0x559a55ceb9a0 secure :-1 s=READY pgs=642 cs=0 l=1 rev1=1 crypto rx=0x7fd2a001fbb0 tx=0x7fd2a001ff30 comp rx=0 tx=0).stop 2024-04-25T00:37:59.516 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.514+0000 7fd2b2178740 1 -- 172.21.15.32:0/1647230314 shutdown_connections 2024-04-25T00:37:59.516 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.515+0000 7fd2b2178740 1 --2- 172.21.15.32:0/1647230314 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559a55cf0570 0x559a55cef910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.516 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.515+0000 7fd2b2178740 1 --2- 172.21.15.32:0/1647230314 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55ce5580 0x559a55ceb9a0 unknown :-1 s=CLOSED pgs=642 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.516 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.515+0000 7fd2b2178740 1 --2- 172.21.15.32:0/1647230314 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x559a55ce18a0 0x559a55ce8860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.516 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.515+0000 7fd2b2178740 1 -- 172.21.15.32:0/1647230314 >> 172.21.15.32:0/1647230314 conn(0x559a55ced770 msgr2=0x559a55ce02b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:59.516 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.515+0000 7fd2b2178740 1 -- 172.21.15.32:0/1647230314 shutdown_connections 2024-04-25T00:37:59.516 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.515+0000 7fd2b2178740 1 -- 172.21.15.32:0/1647230314 wait complete. 2024-04-25T00:37:59.516 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.516+0000 7fd2b2178740 1 -- start start 2024-04-25T00:37:59.517 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.516+0000 7fd2b2178740 1 Processor -- start 2024-04-25T00:37:59.518 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.517+0000 7fd2b2178740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x559a55ce18a0 0x559a55cf86c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.518 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.517+0000 7fd2b2178740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559a55ce5580 0x559a55cf3b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.518 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.517+0000 7fd2b2178740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55cf0570 0x559a55cf4060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.518 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.517+0000 7fd2b2178740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fd28c00e590 con 0x559a55cf0570 2024-04-25T00:37:59.518 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.517+0000 7fd2b2178740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fd28c00fa00 con 0x559a55ce18a0 2024-04-25T00:37:59.518 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.517+0000 7fd2b2178740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fd28c00ec60 con 0x559a55ce5580 2024-04-25T00:37:59.518 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.517+0000 7fd2b2178740 10 client.0 _ll_register_callbacks cb 0x7fd2a000a490 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:37:59.518 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.517+0000 7fd2ad92d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55cf0570 0x559a55cf4060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.518 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.517+0000 7fd2ad92d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55cf0570 0x559a55cf4060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:45126/0 (socket says 172.21.15.32:45126) 2024-04-25T00:37:59.518 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.517+0000 7fd2ad92d640 1 -- 172.21.15.32:0/3973185124 learned_addr learned my addr 172.21.15.32:0/3973185124 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:59.523 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.518+0000 7fd2ad92d640 1 -- 172.21.15.32:0/3973185124 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559a55ce5580 msgr2=0x559a55cf3b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:59.523 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.518+0000 7fd2ad92d640 1 --2- 172.21.15.32:0/3973185124 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559a55ce5580 0x559a55cf3b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.523 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.518+0000 7fd2ad92d640 1 -- 172.21.15.32:0/3973185124 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x559a55ce18a0 msgr2=0x559a55cf86c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:59.523 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.518+0000 7fd2ad92d640 1 --2- 172.21.15.32:0/3973185124 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x559a55ce18a0 0x559a55cf86c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.523 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.518+0000 7fd2ad92d640 1 -- 172.21.15.32:0/3973185124 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x559a55a287d0 con 0x559a55cf0570 2024-04-25T00:37:59.523 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.520+0000 7fd2ad92d640 1 --2- 172.21.15.32:0/3973185124 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55cf0570 0x559a55cf4060 secure :-1 s=READY pgs=643 cs=0 l=1 rev1=1 crypto rx=0x7fd2a401ec10 tx=0x7fd2a40095c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=79a7f8249e8a6ebb server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:59.523 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.520+0000 7fd290ff9640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:37:59.523 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.520+0000 7fd290ff9640 1 -- 172.21.15.32:0/3973185124 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd2a402d060 con 0x559a55cf0570 2024-04-25T00:37:59.523 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.520+0000 7fd290ff9640 1 -- 172.21.15.32:0/3973185124 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fd2a400fb40 con 0x559a55cf0570 2024-04-25T00:37:59.523 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.521+0000 7fd2b2178740 1 -- 172.21.15.32:0/3973185124 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x559a55c51f00 con 0x559a55cf0570 2024-04-25T00:37:59.523 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.521+0000 7fd290ff9640 1 -- 172.21.15.32:0/3973185124 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7fd2a400fd40 con 0x559a55cf0570 2024-04-25T00:37:59.523 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.522+0000 7fd2b2178740 10 client.16320 fetch_fsmap learned FSMap version 28 2024-04-25T00:37:59.523 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.522+0000 7fd2b2178740 1 -- 172.21.15.32:0/3973185124 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x559a55d77800 con 0x559a55cf0570 2024-04-25T00:37:59.524 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.523+0000 7fd290ff9640 1 -- 172.21.15.32:0/3973185124 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x559a55c51f00 con 0x559a55cf0570 2024-04-25T00:37:59.524 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.523+0000 7fd290ff9640 1 -- 172.21.15.32:0/3973185124 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7fd2a4027030 con 0x559a55cf0570 2024-04-25T00:37:59.524 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.523+0000 7fd2b2178740 10 client.16320 fetch_fsmap finished waiting for FSMap version 28 2024-04-25T00:37:59.524 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.523+0000 7fd2b2178740 10 client.16320 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-25T00:37:59.524 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.523+0000 7fd2b2178740 20 client.16320 populate_metadata read hostname 'smithi032' 2024-04-25T00:37:59.524 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.523+0000 7fd2b2178740 1 --2- 172.21.15.32:0/3973185124 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x559a55d77f40 0x559a55d7a350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.524 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.523+0000 7fd2b2178740 4 client.16320 mds_command: new command op to 24481 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-25T00:37:59.524 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.523+0000 7fd2b2178740 1 -- 172.21.15.32:0/3973185124 --> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] -- command(tid 0: {"prefix": "damage ls"}) -- 0x559a55bcf820 con 0x559a55d77f40 2024-04-25T00:37:59.525 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.524+0000 7fd2ad12c640 1 --2- 172.21.15.32:0/3973185124 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x559a55d77f40 0x559a55d7a350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.534 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.534+0000 7fd2ad12c640 1 --2- 172.21.15.32:0/3973185124 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x559a55d77f40 0x559a55d7a350 crc :-1 s=READY pgs=44 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=4abeaa862c613000 server_cookie=4fb66a65e32e4bd9 in_seq=0 out_seq=0 2024-04-25T00:37:59.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.535+0000 7fd290ff9640 10 client.16320 ms_handle_connect on v2:172.21.15.130:6838/4267125809 2024-04-25T00:37:59.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.535+0000 7fd290ff9640 1 -- 172.21.15.32:0/3973185124 <== mds.1 v2:172.21.15.130:6838/4267125809 1 ==== command_reply(tid 0: 0 ) ==== 8+0+326 (crc 0 0 0) 0x559a55bcf820 con 0x559a55d77f40 2024-04-25T00:37:59.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.535+0000 7fd290ff9640 10 client.16320 handle_command_reply: tid=0 2024-04-25T00:37:59.536 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.536+0000 7fd2b2178740 1 client.16320 shutdown 2024-04-25T00:37:59.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.539+0000 7fd2b2178740 1 -- 172.21.15.32:0/3973185124 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55cf0570 msgr2=0x559a55cf4060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:59.540 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.539+0000 7fd2b2178740 1 --2- 172.21.15.32:0/3973185124 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55cf0570 0x559a55cf4060 secure :-1 s=READY pgs=643 cs=0 l=1 rev1=1 crypto rx=0x7fd2a401ec10 tx=0x7fd2a40095c0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.541 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.541+0000 7fd2b2178740 1 -- 172.21.15.32:0/3973185124 shutdown_connections 2024-04-25T00:37:59.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.541+0000 7fd2b2178740 1 --2- 172.21.15.32:0/3973185124 >> [v2:172.21.15.130:6838/4267125809,v1:172.21.15.130:6839/4267125809] conn(0x559a55d77f40 0x559a55d7a350 crc :-1 s=READY pgs=44 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.541+0000 7fd2b2178740 1 --2- 172.21.15.32:0/3973185124 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x559a55cf0570 0x559a55cf4060 unknown :-1 s=CLOSED pgs=643 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.541+0000 7fd2b2178740 1 --2- 172.21.15.32:0/3973185124 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559a55ce5580 0x559a55cf3b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.541+0000 7fd2b2178740 1 --2- 172.21.15.32:0/3973185124 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x559a55ce18a0 0x559a55cf86c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.541+0000 7fd2b2178740 1 -- 172.21.15.32:0/3973185124 >> 172.21.15.32:0/3973185124 conn(0x559a55ced770 msgr2=0x559a55ce0b40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:59.542 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.541+0000 7fd290ff9640 0 client.16320 ms_handle_reset on v2:172.21.15.130:6838/4267125809 2024-04-25T00:37:59.544 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.544+0000 7fd2b2178740 1 -- 172.21.15.32:0/3973185124 shutdown_connections 2024-04-25T00:37:59.545 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.544+0000 7fd2b2178740 1 -- 172.21.15.32:0/3973185124 wait complete. 2024-04-25T00:37:59.547 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.546+0000 7fd2b2178740 20 client.16320 trim_cache size 0 max 16384 2024-04-25T00:37:59.551 INFO:teuthology.orchestra.run.smithi032.stdout:[ 2024-04-25T00:37:59.551 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:37:59.551 INFO:teuthology.orchestra.run.smithi032.stdout: "damage_type": "backtrace", 2024-04-25T00:37:59.551 INFO:teuthology.orchestra.run.smithi032.stdout: "id": 3699566877, 2024-04-25T00:37:59.551 INFO:teuthology.orchestra.run.smithi032.stdout: "ino": 1099511627780, 2024-04-25T00:37:59.551 INFO:teuthology.orchestra.run.smithi032.stdout: "path": "/volumes/qa/sv_0/.meta" 2024-04-25T00:37:59.551 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:37:59.551 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:37:59.551 INFO:teuthology.orchestra.run.smithi032.stdout: "damage_type": "backtrace", 2024-04-25T00:37:59.551 INFO:teuthology.orchestra.run.smithi032.stdout: "id": 4177872315, 2024-04-25T00:37:59.551 INFO:teuthology.orchestra.run.smithi032.stdout: "ino": 1099511627779, 2024-04-25T00:37:59.551 INFO:teuthology.orchestra.run.smithi032.stdout: "path": "/volumes/qa/sv_0/d23367ad-0ac2-4422-ae7b-65f85c49244e" 2024-04-25T00:37:59.551 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:37:59.552 INFO:teuthology.orchestra.run.smithi032.stdout:] 2024-04-25T00:37:59.562 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.561+0000 7fd2af390640 1 -- 172.21.15.32:0/2307721654 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd2940a2f00 msgr2=0x7fd2940a53c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:59.562 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.561+0000 7fd2af390640 1 --2- 172.21.15.32:0/2307721654 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd2940a2f00 0x7fd2940a53c0 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7fd2a4009d30 tx=0x7fd2a40046c0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.562 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.561+0000 7fd2af390640 1 -- 172.21.15.32:0/2307721654 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd2a810dfc0 msgr2=0x7fd2a81a6e80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:59.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.561+0000 7fd2af390640 1 --2- 172.21.15.32:0/2307721654 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd2a810dfc0 0x7fd2a81a6e80 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7fd29802a9b0 tx=0x7fd298031540 comp rx=0 tx=0).stop 2024-04-25T00:37:59.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.561+0000 7fd2af390640 1 -- 172.21.15.32:0/2307721654 shutdown_connections 2024-04-25T00:37:59.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.561+0000 7fd2af390640 1 --2- 172.21.15.32:0/2307721654 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fd2940a2f00 0x7fd2940a53c0 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.561+0000 7fd2af390640 1 --2- 172.21.15.32:0/2307721654 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fd2a811af20 0x7fd2a81ab6f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.562+0000 7fd2af390640 1 --2- 172.21.15.32:0/2307721654 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fd2a810ff90 0x7fd2a81a73c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.562+0000 7fd2af390640 1 --2- 172.21.15.32:0/2307721654 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd2a810dfc0 0x7fd2a81a6e80 secure :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7fd29802a9b0 tx=0x7fd298031540 comp rx=0 tx=0).stop 2024-04-25T00:37:59.563 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.562+0000 7fd2af390640 1 -- 172.21.15.32:0/2307721654 >> 172.21.15.32:0/2307721654 conn(0x7fd2a806c850 msgr2=0x7fd2a811c9c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:59.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.566+0000 7fd2af390640 1 -- 172.21.15.32:0/2307721654 shutdown_connections 2024-04-25T00:37:59.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.568+0000 7fd2af390640 1 -- 172.21.15.32:0/2307721654 wait complete. 2024-04-25T00:37:59.607 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-25T00:37:59.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.867+0000 7fa667790640 1 -- 172.21.15.32:0/945564374 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa660100fc0 msgr2=0x7fa66010d4d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:59.868 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.867+0000 7fa667790640 1 --2- 172.21.15.32:0/945564374 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa660100fc0 0x7fa66010d4d0 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7fa658009ca0 tx=0x7fa65802a8c0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.868+0000 7fa667790640 1 -- 172.21.15.32:0/945564374 shutdown_connections 2024-04-25T00:37:59.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.868+0000 7fa667790640 1 --2- 172.21.15.32:0/945564374 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa660100fc0 0x7fa66010d4d0 unknown :-1 s=CLOSED pgs=135 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.868+0000 7fa667790640 1 --2- 172.21.15.32:0/945564374 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa660100600 0x7fa660100a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.868+0000 7fa667790640 1 --2- 172.21.15.32:0/945564374 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa6601066a0 0x7fa660106a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.868+0000 7fa667790640 1 -- 172.21.15.32:0/945564374 >> 172.21.15.32:0/945564374 conn(0x7fa6600776e0 msgr2=0x7fa660077af0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:59.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.869+0000 7fa667790640 1 -- 172.21.15.32:0/945564374 shutdown_connections 2024-04-25T00:37:59.869 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.869+0000 7fa667790640 1 -- 172.21.15.32:0/945564374 wait complete. 2024-04-25T00:37:59.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.870+0000 7fa667790640 1 Processor -- start 2024-04-25T00:37:59.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.871+0000 7fa667790640 1 -- start start 2024-04-25T00:37:59.871 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.871+0000 7fa667790640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa660100600 0x7fa660196dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.873 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.872+0000 7fa667790640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa660100fc0 0x7fa660197300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.873 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.872+0000 7fa667790640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa6601066a0 0x7fa66019b680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.873 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.872+0000 7fa66552c640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa660100600 0x7fa660196dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.873 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.872+0000 7fa66552c640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa660100600 0x7fa660196dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:40016/0 (socket says 172.21.15.32:40016) 2024-04-25T00:37:59.873 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.872+0000 7fa664d2b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa660100fc0 0x7fa660197300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.873 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.872+0000 7fa664d2b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa660100fc0 0x7fa660197300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:38220/0 (socket says 172.21.15.32:38220) 2024-04-25T00:37:59.873 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.872+0000 7fa66552c640 1 -- 172.21.15.32:0/4115355371 learned_addr learned my addr 172.21.15.32:0/4115355371 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:59.873 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.872+0000 7fa667790640 1 -- 172.21.15.32:0/4115355371 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fa6601109e0 con 0x7fa6601066a0 2024-04-25T00:37:59.873 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.872+0000 7fa667790640 1 -- 172.21.15.32:0/4115355371 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fa660110860 con 0x7fa660100fc0 2024-04-25T00:37:59.873 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.872+0000 7fa667790640 1 -- 172.21.15.32:0/4115355371 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fa660110b60 con 0x7fa660100600 2024-04-25T00:37:59.874 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.873+0000 7fa66552c640 1 -- 172.21.15.32:0/4115355371 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa660100fc0 msgr2=0x7fa660197300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:59.874 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.873+0000 7fa66552c640 1 --2- 172.21.15.32:0/4115355371 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa660100fc0 0x7fa660197300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.874 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.873+0000 7fa66552c640 1 -- 172.21.15.32:0/4115355371 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa6601066a0 msgr2=0x7fa66019b680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:59.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.873+0000 7fa66552c640 1 --2- 172.21.15.32:0/4115355371 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa6601066a0 0x7fa66019b680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.873+0000 7fa66552c640 1 -- 172.21.15.32:0/4115355371 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa66019bdb0 con 0x7fa660100600 2024-04-25T00:37:59.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.873+0000 7fa66552c640 1 --2- 172.21.15.32:0/4115355371 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa660100600 0x7fa660196dc0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7fa65c00d770 tx=0x7fa65c00dc30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:59.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.874+0000 7fa664d2b640 1 --2- 172.21.15.32:0/4115355371 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa660100fc0 0x7fa660197300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:37:59.875 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.875+0000 7fa6567fc640 1 -- 172.21.15.32:0/4115355371 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa65c01b070 con 0x7fa660100600 2024-04-25T00:37:59.876 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.875+0000 7fa6567fc640 1 -- 172.21.15.32:0/4115355371 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa65c014e50 con 0x7fa660100600 2024-04-25T00:37:59.876 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.875+0000 7fa6567fc640 1 -- 172.21.15.32:0/4115355371 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa65c0035f0 con 0x7fa660100600 2024-04-25T00:37:59.876 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.875+0000 7fa667790640 1 -- 172.21.15.32:0/4115355371 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa66019c0a0 con 0x7fa660100600 2024-04-25T00:37:59.876 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.875+0000 7fa667790640 1 -- 172.21.15.32:0/4115355371 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa660069ab0 con 0x7fa660100600 2024-04-25T00:37:59.879 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.878+0000 7fa6567fc640 1 -- 172.21.15.32:0/4115355371 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7fa65c003790 con 0x7fa660100600 2024-04-25T00:37:59.880 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.880+0000 7fa6567fc640 1 --2- 172.21.15.32:0/4115355371 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa64c0a2ed0 0x7fa64c0a5390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.885 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.881+0000 7fa664d2b640 1 --2- 172.21.15.32:0/4115355371 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa64c0a2ed0 0x7fa64c0a5390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.885 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.883+0000 7fa6567fc640 1 -- 172.21.15.32:0/4115355371 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fa65c0d45e0 con 0x7fa660100600 2024-04-25T00:37:59.885 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.883+0000 7fa66a578740 1 Processor -- start 2024-04-25T00:37:59.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.885+0000 7fa66a578740 1 -- start start 2024-04-25T00:37:59.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.885+0000 7fa66a578740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b602d0 0x55efa9b606b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.885+0000 7fa66a578740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55efa9b60c80 0x55efa9b73840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.885+0000 7fa66a578740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55efa9b73d80 0x55efa9b76170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.885+0000 7fa66a578740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55efa9ac5670 con 0x55efa9b73d80 2024-04-25T00:37:59.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.885+0000 7fa66a578740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55efa9a6d970 con 0x55efa9b602d0 2024-04-25T00:37:59.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.885+0000 7fa66a578740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55efa9ac1630 con 0x55efa9b60c80 2024-04-25T00:37:59.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.886+0000 7fa664d2b640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55efa9b60c80 0x55efa9b73840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.886+0000 7fa664d2b640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55efa9b60c80 0x55efa9b73840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:40032/0 (socket says 172.21.15.32:40032) 2024-04-25T00:37:59.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.886+0000 7fa664d2b640 1 -- 172.21.15.32:0/520983041 learned_addr learned my addr 172.21.15.32:0/520983041 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:59.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.886+0000 7fa664d2b640 1 -- 172.21.15.32:0/520983041 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b602d0 msgr2=0x55efa9b606b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:59.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.886+0000 7fa66552c640 1 --2- 172.21.15.32:0/520983041 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b602d0 0x55efa9b606b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.887+0000 7fa665d2d640 1 --2- 172.21.15.32:0/520983041 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55efa9b73d80 0x55efa9b76170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.887+0000 7fa664d2b640 1 --2- 172.21.15.32:0/520983041 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b602d0 0x55efa9b606b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.887+0000 7fa664d2b640 1 -- 172.21.15.32:0/520983041 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55efa9b73d80 msgr2=0x55efa9b76170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:59.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.887+0000 7fa664d2b640 1 --2- 172.21.15.32:0/520983041 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55efa9b73d80 0x55efa9b76170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.887+0000 7fa664d2b640 1 -- 172.21.15.32:0/520983041 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55efa9a91400 con 0x55efa9b60c80 2024-04-25T00:37:59.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.887+0000 7fa664d2b640 1 --2- 172.21.15.32:0/4115355371 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa64c0a2ed0 0x7fa64c0a5390 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7fa660077c00 tx=0x7fa650005e50 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:59.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.888+0000 7fa66552c640 1 --2- 172.21.15.32:0/520983041 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b602d0 0x55efa9b606b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:37:59.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.888+0000 7fa665d2d640 1 --2- 172.21.15.32:0/520983041 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55efa9b73d80 0x55efa9b76170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:37:59.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.888+0000 7fa664d2b640 1 --2- 172.21.15.32:0/520983041 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55efa9b60c80 0x55efa9b73840 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7fa65000ee20 tx=0x7fa650011000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e1e91ae416282bc5 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:59.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.889+0000 7fa64b7fe640 1 -- 172.21.15.32:0/520983041 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa650011e00 con 0x55efa9b60c80 2024-04-25T00:37:59.890 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.889+0000 7fa64b7fe640 1 -- 172.21.15.32:0/520983041 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa650017030 con 0x55efa9b60c80 2024-04-25T00:37:59.890 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.889+0000 7fa64b7fe640 1 -- 172.21.15.32:0/520983041 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa65001e6a0 con 0x55efa9b60c80 2024-04-25T00:37:59.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.890+0000 7fa66a578740 1 -- 172.21.15.32:0/520983041 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55efa9b60c80 msgr2=0x55efa9b73840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:59.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.890+0000 7fa66a578740 1 --2- 172.21.15.32:0/520983041 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55efa9b60c80 0x55efa9b73840 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7fa65000ee20 tx=0x7fa650011000 comp rx=0 tx=0).stop 2024-04-25T00:37:59.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.891+0000 7fa66a578740 1 -- 172.21.15.32:0/520983041 shutdown_connections 2024-04-25T00:37:59.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.891+0000 7fa66a578740 1 --2- 172.21.15.32:0/520983041 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55efa9b73d80 0x55efa9b76170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.891+0000 7fa66a578740 1 --2- 172.21.15.32:0/520983041 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55efa9b60c80 0x55efa9b73840 unknown :-1 s=CLOSED pgs=135 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.891+0000 7fa66a578740 1 --2- 172.21.15.32:0/520983041 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b602d0 0x55efa9b606b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.891+0000 7fa66a578740 1 -- 172.21.15.32:0/520983041 >> 172.21.15.32:0/520983041 conn(0x55efa9b5c8a0 msgr2=0x55efa9b5cc80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:59.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.891+0000 7fa66a578740 1 -- 172.21.15.32:0/520983041 shutdown_connections 2024-04-25T00:37:59.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.891+0000 7fa66a578740 1 -- 172.21.15.32:0/520983041 wait complete. 2024-04-25T00:37:59.892 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.892+0000 7fa66a578740 1 -- start start 2024-04-25T00:37:59.893 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.892+0000 7fa66a578740 1 Processor -- start 2024-04-25T00:37:59.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.893+0000 7fa66a578740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55efa9b602d0 0x55efa9b6d670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.893+0000 7fa66a578740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b60c80 0x55efa9b6dbb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.893+0000 7fa66a578740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55efa9b73d80 0x55efa9b73390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.893+0000 7fa664d2b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b60c80 0x55efa9b6dbb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.894 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.893+0000 7fa664d2b640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b60c80 0x55efa9b6dbb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:38246/0 (socket says 172.21.15.32:38246) 2024-04-25T00:37:59.904 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.901+0000 7fa66a578740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x55efa9b63370 con 0x55efa9b602d0 2024-04-25T00:37:59.904 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.901+0000 7fa66a578740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x55efa9a81050 con 0x55efa9b60c80 2024-04-25T00:37:59.904 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.901+0000 7fa66a578740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x55efa9b634f0 con 0x55efa9b73d80 2024-04-25T00:37:59.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.901+0000 7fa66a578740 10 client.0 _ll_register_callbacks cb 0x55efa97d8be0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:37:59.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.902+0000 7fa66552c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55efa9b602d0 0x55efa9b6d670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.902+0000 7fa66552c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55efa9b602d0 0x55efa9b6d670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:45140/0 (socket says 172.21.15.32:45140) 2024-04-25T00:37:59.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.902+0000 7fa66552c640 1 -- 172.21.15.32:0/2237125121 learned_addr learned my addr 172.21.15.32:0/2237125121 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:59.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.902+0000 7fa66552c640 1 -- 172.21.15.32:0/2237125121 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55efa9b73d80 msgr2=0x55efa9b73390 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-25T00:37:59.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.902+0000 7fa66552c640 1 --2- 172.21.15.32:0/2237125121 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55efa9b73d80 0x55efa9b73390 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.902+0000 7fa66552c640 1 -- 172.21.15.32:0/2237125121 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b60c80 msgr2=0x55efa9b6dbb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:59.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.902+0000 7fa66552c640 1 --2- 172.21.15.32:0/2237125121 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b60c80 0x55efa9b6dbb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.902+0000 7fa66552c640 1 -- 172.21.15.32:0/2237125121 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55efa9a52dc0 con 0x55efa9b602d0 2024-04-25T00:37:59.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.903+0000 7fa66552c640 1 --2- 172.21.15.32:0/2237125121 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55efa9b602d0 0x55efa9b6d670 secure :-1 s=READY pgs=644 cs=0 l=1 rev1=1 crypto rx=0x7fa65c09d0c0 tx=0x7fa65c0a0830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=67a0e181c7e82679 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:59.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.903+0000 7fa648ff9640 10 client.0 ms_handle_connect on v2:172.21.15.32:3300/0 2024-04-25T00:37:59.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.903+0000 7fa648ff9640 1 -- 172.21.15.32:0/2237125121 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa65c024070 con 0x55efa9b602d0 2024-04-25T00:37:59.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.903+0000 7fa648ff9640 1 -- 172.21.15.32:0/2237125121 <== mon.0 v2:172.21.15.32:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fa65c09cd60 con 0x55efa9b602d0 2024-04-25T00:37:59.905 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.903+0000 7fa66a578740 1 -- 172.21.15.32:0/2237125121 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa650017030 con 0x55efa9b602d0 2024-04-25T00:37:59.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.904+0000 7fa664d2b640 1 --2- 172.21.15.32:0/2237125121 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b60c80 0x55efa9b6dbb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello state changed while learned_addr, mark_down or replacing must be happened just now 2024-04-25T00:37:59.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.904+0000 7fa648ff9640 1 -- 172.21.15.32:0/2237125121 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7fa65c0d1ba0 con 0x55efa9b602d0 2024-04-25T00:37:59.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.905+0000 7fa66a578740 10 client.16326 fetch_fsmap learned FSMap version 28 2024-04-25T00:37:59.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.905+0000 7fa66a578740 1 -- 172.21.15.32:0/2237125121 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x55efa97d93f0 con 0x55efa9b602d0 2024-04-25T00:37:59.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.906+0000 7fa648ff9640 1 -- 172.21.15.32:0/2237125121 <== mon.0 v2:172.21.15.32:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa65c0d1d40 con 0x55efa9b602d0 2024-04-25T00:37:59.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.906+0000 7fa648ff9640 1 -- 172.21.15.32:0/2237125121 <== mon.0 v2:172.21.15.32:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7fa65c09f5b0 con 0x55efa9b602d0 2024-04-25T00:37:59.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.906+0000 7fa66a578740 10 client.16326 fetch_fsmap finished waiting for FSMap version 28 2024-04-25T00:37:59.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.906+0000 7fa66a578740 10 client.16326 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:37:59.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.906+0000 7fa66a578740 20 client.16326 populate_metadata read hostname 'smithi032' 2024-04-25T00:37:59.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.906+0000 7fa66a578740 1 --2- 172.21.15.32:0/2237125121 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55efa9b8c610 0x55efa9b8eab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.906+0000 7fa66a578740 4 client.16326 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-25T00:37:59.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.906+0000 7fa66a578740 1 -- 172.21.15.32:0/2237125121 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x7fa650017030 con 0x55efa9b8c610 2024-04-25T00:37:59.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.911+0000 7fa664d2b640 1 --2- 172.21.15.32:0/2237125121 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55efa9b8c610 0x55efa9b8eab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.914+0000 7fa664d2b640 1 --2- 172.21.15.32:0/2237125121 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55efa9b8c610 0x55efa9b8eab0 crc :-1 s=READY pgs=97 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d44a326bd5434177 server_cookie=bc9bd9da458b5c5 in_seq=0 out_seq=0 2024-04-25T00:37:59.915 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.914+0000 7fa648ff9640 10 client.16326 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:37:59.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.919+0000 7fa648ff9640 1 -- 172.21.15.32:0/2237125121 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+23246 (crc 0 0 0) 0x7fa650017030 con 0x55efa9b8c610 2024-04-25T00:37:59.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.919+0000 7fa648ff9640 10 client.16326 handle_command_reply: tid=0 2024-04-25T00:37:59.919 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.919+0000 7fa66a578740 1 client.16326 shutdown 2024-04-25T00:37:59.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.919+0000 7fa66a578740 1 -- 172.21.15.32:0/2237125121 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55efa9b602d0 msgr2=0x55efa9b6d670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:59.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.919+0000 7fa66a578740 1 --2- 172.21.15.32:0/2237125121 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55efa9b602d0 0x55efa9b6d670 secure :-1 s=READY pgs=644 cs=0 l=1 rev1=1 crypto rx=0x7fa65c09d0c0 tx=0x7fa65c0a0830 comp rx=0 tx=0).stop 2024-04-25T00:37:59.920 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.920+0000 7fa66a578740 1 -- 172.21.15.32:0/2237125121 shutdown_connections 2024-04-25T00:37:59.921 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.920+0000 7fa66a578740 1 --2- 172.21.15.32:0/2237125121 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55efa9b8c610 0x55efa9b8eab0 crc :-1 s=READY pgs=97 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.921 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.920+0000 7fa66a578740 1 --2- 172.21.15.32:0/2237125121 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55efa9b73d80 0x55efa9b73390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.921 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.920+0000 7fa66a578740 1 --2- 172.21.15.32:0/2237125121 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b60c80 0x55efa9b6dbb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.921 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.920+0000 7fa66a578740 1 --2- 172.21.15.32:0/2237125121 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55efa9b602d0 0x55efa9b6d670 unknown :-1 s=CLOSED pgs=644 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.921 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.920+0000 7fa66a578740 1 -- 172.21.15.32:0/2237125121 >> 172.21.15.32:0/2237125121 conn(0x55efa9b5c8a0 msgr2=0x55efa9b62710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:59.921 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.920+0000 7fa648ff9640 0 client.16326 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:37:59.921 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.920+0000 7fa66a578740 1 -- 172.21.15.32:0/2237125121 shutdown_connections 2024-04-25T00:37:59.921 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.920+0000 7fa66a578740 1 -- 172.21.15.32:0/2237125121 wait complete. 2024-04-25T00:37:59.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.922+0000 7fa66a578740 20 client.16326 trim_cache size 0 max 16384 2024-04-25T00:37:59.951 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.951+0000 7fa66a578740 1 Processor -- start 2024-04-25T00:37:59.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.951+0000 7fa66a578740 1 -- start start 2024-04-25T00:37:59.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.952+0000 7fa66a578740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55efa9b602d0 0x55efa9b61e60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.952+0000 7fa66a578740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55efa9b60c80 0x55efa9b94380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.953 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.952+0000 7fa66a578740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b73d80 0x55efa9b948c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.953 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.952+0000 7fa66a578740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fa644012950 con 0x55efa9b60c80 2024-04-25T00:37:59.953 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.952+0000 7fa66a578740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fa644010220 con 0x55efa9b73d80 2024-04-25T00:37:59.953 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.952+0000 7fa66a578740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fa64400e070 con 0x55efa9b602d0 2024-04-25T00:37:59.953 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.953+0000 7fa665d2d640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b73d80 0x55efa9b948c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.954 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.953+0000 7fa665d2d640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b73d80 0x55efa9b948c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:38262/0 (socket says 172.21.15.32:38262) 2024-04-25T00:37:59.954 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.953+0000 7fa665d2d640 1 -- 172.21.15.32:0/666097376 learned_addr learned my addr 172.21.15.32:0/666097376 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:59.954 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.953+0000 7fa664d2b640 1 --2- 172.21.15.32:0/666097376 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55efa9b60c80 0x55efa9b94380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.955 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.954+0000 7fa665d2d640 1 -- 172.21.15.32:0/666097376 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55efa9b602d0 msgr2=0x55efa9b61e60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:59.955 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.954+0000 7fa665d2d640 1 --2- 172.21.15.32:0/666097376 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55efa9b602d0 0x55efa9b61e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.956 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.954+0000 7fa665d2d640 1 -- 172.21.15.32:0/666097376 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55efa9b60c80 msgr2=0x55efa9b94380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:59.956 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.954+0000 7fa665d2d640 1 --2- 172.21.15.32:0/666097376 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55efa9b60c80 0x55efa9b94380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.956 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.954+0000 7fa665d2d640 1 -- 172.21.15.32:0/666097376 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55efa98a37d0 con 0x55efa9b73d80 2024-04-25T00:37:59.956 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.954+0000 7fa664d2b640 1 --2- 172.21.15.32:0/666097376 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55efa9b60c80 0x55efa9b94380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:37:59.959 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.957+0000 7fa665d2d640 1 --2- 172.21.15.32:0/666097376 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b73d80 0x55efa9b948c0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7fa65802ade0 tx=0x7fa658005fa0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6b7823cb8b0d31ce server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:59.960 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.959+0000 7fa64affd640 1 -- 172.21.15.32:0/666097376 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa658031da0 con 0x55efa9b73d80 2024-04-25T00:37:59.960 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.959+0000 7fa64affd640 1 -- 172.21.15.32:0/666097376 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa658036070 con 0x55efa9b73d80 2024-04-25T00:37:59.961 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.960+0000 7fa66a578740 1 -- 172.21.15.32:0/666097376 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b73d80 msgr2=0x55efa9b948c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:37:59.961 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.960+0000 7fa66a578740 1 --2- 172.21.15.32:0/666097376 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b73d80 0x55efa9b948c0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7fa65802ade0 tx=0x7fa658005fa0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.961 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.961+0000 7fa66a578740 1 -- 172.21.15.32:0/666097376 shutdown_connections 2024-04-25T00:37:59.961 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.961+0000 7fa66a578740 1 --2- 172.21.15.32:0/666097376 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b73d80 0x55efa9b948c0 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.961 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.961+0000 7fa66a578740 1 --2- 172.21.15.32:0/666097376 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55efa9b60c80 0x55efa9b94380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.961 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.961+0000 7fa66a578740 1 --2- 172.21.15.32:0/666097376 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55efa9b602d0 0x55efa9b61e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.961 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.961+0000 7fa66a578740 1 -- 172.21.15.32:0/666097376 >> 172.21.15.32:0/666097376 conn(0x55efa9b5c8a0 msgr2=0x55efa9b67a00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:37:59.987 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.987+0000 7fa66a578740 1 -- 172.21.15.32:0/666097376 shutdown_connections 2024-04-25T00:37:59.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.987+0000 7fa66a578740 1 -- 172.21.15.32:0/666097376 wait complete. 2024-04-25T00:37:59.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.987+0000 7fa66a578740 1 -- start start 2024-04-25T00:37:59.988 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.987+0000 7fa66a578740 1 Processor -- start 2024-04-25T00:37:59.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.990+0000 7fa66a578740 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55efa9b602d0 0x55efa9b9bab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.990+0000 7fa66a578740 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b60c80 0x55efa9b9bff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.990+0000 7fa66a578740 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55efa9bef6c0 0x55efa9bf1ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:37:59.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.990+0000 7fa66a578740 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fa64400e6e0 con 0x55efa9b602d0 2024-04-25T00:37:59.991 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.990+0000 7fa66a578740 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fa64400fb50 con 0x55efa9b60c80 2024-04-25T00:37:59.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.990+0000 7fa66a578740 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fa64400edb0 con 0x55efa9bef6c0 2024-04-25T00:37:59.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.990+0000 7fa66a578740 10 client.0 _ll_register_callbacks cb 0x7fa65c09c990 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-25T00:37:59.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.991+0000 7fa665d2d640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55efa9bef6c0 0x55efa9bf1ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.991+0000 7fa665d2d640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55efa9bef6c0 0x55efa9bf1ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:40058/0 (socket says 172.21.15.32:40058) 2024-04-25T00:37:59.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.991+0000 7fa665d2d640 1 -- 172.21.15.32:0/1703986356 learned_addr learned my addr 172.21.15.32:0/1703986356 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:37:59.992 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.991+0000 7fa664d2b640 1 --2- 172.21.15.32:0/1703986356 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b60c80 0x55efa9b9bff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.992+0000 7fa66552c640 1 --2- 172.21.15.32:0/1703986356 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55efa9b602d0 0x55efa9b9bab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:37:59.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.993+0000 7fa665d2d640 1 -- 172.21.15.32:0/1703986356 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b60c80 msgr2=0x55efa9b9bff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:59.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.993+0000 7fa665d2d640 1 --2- 172.21.15.32:0/1703986356 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b60c80 0x55efa9b9bff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.993+0000 7fa665d2d640 1 -- 172.21.15.32:0/1703986356 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55efa9b602d0 msgr2=0x55efa9b9bab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-25T00:37:59.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.993+0000 7fa665d2d640 1 --2- 172.21.15.32:0/1703986356 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55efa9b602d0 0x55efa9b9bab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:37:59.994 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.993+0000 7fa665d2d640 1 -- 172.21.15.32:0/1703986356 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55efa9bf2070 con 0x55efa9bef6c0 2024-04-25T00:37:59.994 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.994+0000 7fa665d2d640 1 --2- 172.21.15.32:0/1703986356 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55efa9bef6c0 0x55efa9bf1ab0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7fa658005e30 tx=0x7fa6580095b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=3ab56e15cfc7d3e1 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:37:59.998 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.998+0000 7fa648ff9640 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2024-04-25T00:37:59.999 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.998+0000 7fa648ff9640 1 -- 172.21.15.32:0/1703986356 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa658031db0 con 0x55efa9bef6c0 2024-04-25T00:37:59.999 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.998+0000 7fa66a578740 1 -- 172.21.15.32:0/1703986356 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x55efa9ad2bb0 con 0x55efa9bef6c0 2024-04-25T00:37:59.999 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:37:59.998+0000 7fa648ff9640 1 -- 172.21.15.32:0/1703986356 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 9466+0+0 (secure 0 0 0) 0x7fa658004440 con 0x55efa9bef6c0 2024-04-25T00:38:00.000 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:59 smithi032 ceph-mon[29357]: pgmap v143: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 621 MiB/s wr, 1.39k op/s 2024-04-25T00:38:00.000 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:37:59 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2540928517' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:38:00.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.001+0000 7fa648ff9640 1 -- 172.21.15.32:0/1703986356 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7fa658004640 con 0x55efa9bef6c0 2024-04-25T00:38:00.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.001+0000 7fa66a578740 10 client.25861 fetch_fsmap learned FSMap version 28 2024-04-25T00:38:00.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.001+0000 7fa66a578740 1 -- 172.21.15.32:0/1703986356 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x55efa9bf3400 con 0x55efa9bef6c0 2024-04-25T00:38:00.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.003+0000 7fa648ff9640 1 -- 172.21.15.32:0/1703986356 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x55efa9ad2bb0 con 0x55efa9bef6c0 2024-04-25T00:38:00.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.003+0000 7fa648ff9640 1 -- 172.21.15.32:0/1703986356 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7fa658054030 con 0x55efa9bef6c0 2024-04-25T00:38:00.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.003+0000 7fa66a578740 10 client.25861 fetch_fsmap finished waiting for FSMap version 28 2024-04-25T00:38:00.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.003+0000 7fa66a578740 10 client.25861 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-25T00:38:00.005 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.003+0000 7fa66a578740 20 client.25861 populate_metadata read hostname 'smithi032' 2024-04-25T00:38:00.005 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.003+0000 7fa66a578740 1 --2- 172.21.15.32:0/1703986356 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55efa9bf3c10 0x55efa9bf6030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:38:00.005 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.003+0000 7fa66a578740 4 client.25861 mds_command: new command op to 24506 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-25T00:38:00.005 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.003+0000 7fa66a578740 1 -- 172.21.15.32:0/1703986356 --> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55efa9accf00 con 0x55efa9bf3c10 2024-04-25T00:38:00.011 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.006+0000 7fa66552c640 1 --2- 172.21.15.32:0/1703986356 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55efa9bf3c10 0x55efa9bf6030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:38:00.011 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.007+0000 7fa648ff9640 10 client.25861 ms_handle_connect on v2:172.21.15.121:6838/258444853 2024-04-25T00:38:00.011 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.007+0000 7fa66552c640 1 --2- 172.21.15.32:0/1703986356 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55efa9bf3c10 0x55efa9bf6030 crc :-1 s=READY pgs=98 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=728d35f65f707a80 server_cookie=de421b75932cc695 in_seq=0 out_seq=0 2024-04-25T00:38:00.011 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.008+0000 7fa648ff9640 1 -- 172.21.15.32:0/1703986356 <== mds.0 v2:172.21.15.121:6838/258444853 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55efa9accf00 con 0x55efa9bf3c10 2024-04-25T00:38:00.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.008+0000 7fa648ff9640 10 client.25861 handle_command_reply: tid=0 2024-04-25T00:38:00.012 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.008+0000 7fa66a578740 1 client.25861 shutdown 2024-04-25T00:38:00.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.012+0000 7fa66a578740 1 -- 172.21.15.32:0/1703986356 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55efa9bef6c0 msgr2=0x55efa9bf1ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:38:00.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.012+0000 7fa66a578740 1 --2- 172.21.15.32:0/1703986356 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55efa9bef6c0 0x55efa9bf1ab0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7fa658005e30 tx=0x7fa6580095b0 comp rx=0 tx=0).stop 2024-04-25T00:38:00.017 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.016+0000 7fa66a578740 1 -- 172.21.15.32:0/1703986356 shutdown_connections 2024-04-25T00:38:00.017 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.016+0000 7fa66a578740 1 --2- 172.21.15.32:0/1703986356 >> [v2:172.21.15.121:6838/258444853,v1:172.21.15.121:6839/258444853] conn(0x55efa9bf3c10 0x55efa9bf6030 crc :-1 s=READY pgs=98 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:38:00.017 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.016+0000 7fa66a578740 1 --2- 172.21.15.32:0/1703986356 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55efa9bef6c0 0x55efa9bf1ab0 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:38:00.018 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.016+0000 7fa66a578740 1 --2- 172.21.15.32:0/1703986356 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x55efa9b60c80 0x55efa9b9bff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:38:00.018 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.016+0000 7fa66a578740 1 --2- 172.21.15.32:0/1703986356 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x55efa9b602d0 0x55efa9b9bab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:38:00.018 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.016+0000 7fa66a578740 1 -- 172.21.15.32:0/1703986356 >> 172.21.15.32:0/1703986356 conn(0x55efa9b5c8a0 msgr2=0x55efa9b5b2b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:38:00.018 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.016+0000 7fa648ff9640 0 client.25861 ms_handle_reset on v2:172.21.15.121:6838/258444853 2024-04-25T00:38:00.018 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.016+0000 7fa66a578740 1 -- 172.21.15.32:0/1703986356 shutdown_connections 2024-04-25T00:38:00.019 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.016+0000 7fa66a578740 1 -- 172.21.15.32:0/1703986356 wait complete. 2024-04-25T00:38:00.021 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.021+0000 7fa66a578740 20 client.25861 trim_cache size 0 max 16384 2024-04-25T00:38:00.028 INFO:teuthology.orchestra.run.smithi032.stdout:[] 2024-04-25T00:38:00.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.036+0000 7fa667790640 1 -- 172.21.15.32:0/4115355371 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa64c0a2ed0 msgr2=0x7fa64c0a5390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:38:00.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.036+0000 7fa667790640 1 --2- 172.21.15.32:0/4115355371 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa64c0a2ed0 0x7fa64c0a5390 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7fa660077c00 tx=0x7fa650005e50 comp rx=0 tx=0).stop 2024-04-25T00:38:00.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.036+0000 7fa667790640 1 -- 172.21.15.32:0/4115355371 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa660100600 msgr2=0x7fa660196dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:38:00.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.036+0000 7fa667790640 1 --2- 172.21.15.32:0/4115355371 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa660100600 0x7fa660196dc0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7fa65c00d770 tx=0x7fa65c00dc30 comp rx=0 tx=0).stop 2024-04-25T00:38:00.040 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.036+0000 7fa667790640 1 -- 172.21.15.32:0/4115355371 shutdown_connections 2024-04-25T00:38:00.041 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.036+0000 7fa667790640 1 --2- 172.21.15.32:0/4115355371 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fa64c0a2ed0 0x7fa64c0a5390 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:38:00.041 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.036+0000 7fa667790640 1 --2- 172.21.15.32:0/4115355371 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fa6601066a0 0x7fa66019b680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:38:00.041 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.036+0000 7fa667790640 1 --2- 172.21.15.32:0/4115355371 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fa660100fc0 0x7fa660197300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:38:00.041 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.036+0000 7fa667790640 1 --2- 172.21.15.32:0/4115355371 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa660100600 0x7fa660196dc0 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:38:00.041 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.036+0000 7fa667790640 1 -- 172.21.15.32:0/4115355371 >> 172.21.15.32:0/4115355371 conn(0x7fa6600776e0 msgr2=0x7fa6600ff0b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:38:00.041 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.036+0000 7fa667790640 1 -- 172.21.15.32:0/4115355371 shutdown_connections 2024-04-25T00:38:00.041 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:38:00.036+0000 7fa667790640 1 -- 172.21.15.32:0/4115355371 wait complete. 2024-04-25T00:38:00.075 ERROR:tasks.fwd_scrub:rank 1 damaged: [{'damage_type': 'backtrace', 'id': 3699566877, 'ino': 1099511627780, 'path': '/volumes/qa/sv_0/.meta'}, {'damage_type': 'backtrace', 'id': 4177872315, 'ino': 1099511627779, 'path': '/volumes/qa/sv_0/d23367ad-0ac2-4422-ae7b-65f85c49244e'}] 2024-04-25T00:38:00.075 ERROR:tasks.fwd_scrub.fs.[cephfs]:exception: Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_ceph_d2c45d977142e024a59b68a8f1dce8f0b9840551/qa/tasks/fwd_scrub.py", line 38, in _run self.do_scrub() File "/home/teuthworker/src/git.ceph.com_ceph_d2c45d977142e024a59b68a8f1dce8f0b9840551/qa/tasks/fwd_scrub.py", line 55, in do_scrub self._scrub() File "/home/teuthworker/src/git.ceph.com_ceph_d2c45d977142e024a59b68a8f1dce8f0b9840551/qa/tasks/fwd_scrub.py", line 79, in _scrub self._check_damage() File "/home/teuthworker/src/git.ceph.com_ceph_d2c45d977142e024a59b68a8f1dce8f0b9840551/qa/tasks/fwd_scrub.py", line 90, in _check_damage raise RuntimeError(f"rank damage found: {types}") RuntimeError: rank damage found: {'backtrace'} 2024-04-25T00:38:00.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:59 smithi121 ceph-mon[32231]: pgmap v143: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 621 MiB/s wr, 1.39k op/s 2024-04-25T00:38:00.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:37:59 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2540928517' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:38:00.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:59 smithi130 ceph-mon[32525]: pgmap v143: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 621 MiB/s wr, 1.39k op/s 2024-04-25T00:38:00.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:37:59 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2540928517' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:38:02.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:01 smithi121 ceph-mon[32231]: pgmap v144: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 2.0 MiB/s rd, 881 MiB/s wr, 1.89k op/s 2024-04-25T00:38:02.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:01 smithi130 ceph-mon[32525]: pgmap v144: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 2.0 MiB/s rd, 881 MiB/s wr, 1.89k op/s 2024-04-25T00:38:02.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:01 smithi032 ceph-mon[29357]: pgmap v144: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 2.0 MiB/s rd, 881 MiB/s wr, 1.89k op/s 2024-04-25T00:38:04.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:03 smithi121 ceph-mon[32231]: pgmap v145: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.5 MiB/s rd, 675 MiB/s wr, 1.26k op/s 2024-04-25T00:38:04.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:03 smithi130 ceph-mon[32525]: pgmap v145: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.5 MiB/s rd, 675 MiB/s wr, 1.26k op/s 2024-04-25T00:38:04.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:03 smithi032 ceph-mon[29357]: pgmap v145: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.5 MiB/s rd, 675 MiB/s wr, 1.26k op/s 2024-04-25T00:38:06.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:05 smithi130 ceph-mon[32525]: pgmap v146: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.9 MiB/s rd, 692 MiB/s wr, 1.27k op/s 2024-04-25T00:38:06.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:05 smithi032 ceph-mon[29357]: pgmap v146: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.9 MiB/s rd, 692 MiB/s wr, 1.27k op/s 2024-04-25T00:38:06.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:05 smithi121 ceph-mon[32231]: pgmap v146: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.9 MiB/s rd, 692 MiB/s wr, 1.27k op/s 2024-04-25T00:38:08.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:07 smithi130 ceph-mon[32525]: pgmap v147: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.8 MiB/s rd, 797 MiB/s wr, 1.62k op/s 2024-04-25T00:38:08.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:07 smithi032 ceph-mon[29357]: pgmap v147: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.8 MiB/s rd, 797 MiB/s wr, 1.62k op/s 2024-04-25T00:38:08.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:07 smithi121 ceph-mon[32231]: pgmap v147: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.8 MiB/s rd, 797 MiB/s wr, 1.62k op/s 2024-04-25T00:38:09.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:08 smithi130 ceph-mon[32525]: pgmap v148: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 562 MiB/s wr, 1.27k op/s 2024-04-25T00:38:09.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:08 smithi032 ceph-mon[29357]: pgmap v148: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 562 MiB/s wr, 1.27k op/s 2024-04-25T00:38:09.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:08 smithi121 ceph-mon[32231]: pgmap v148: 129 pgs: 129 active+clean; 40 GiB data, 86 GiB used, 987 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 562 MiB/s wr, 1.27k op/s 2024-04-25T00:38:11.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:11 smithi032 ceph-mon[29357]: pgmap v149: 129 pgs: 129 active+clean; 41 GiB data, 87 GiB used, 986 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 710 MiB/s wr, 2.65k op/s 2024-04-25T00:38:11.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:11 smithi121 ceph-mon[32231]: pgmap v149: 129 pgs: 129 active+clean; 41 GiB data, 87 GiB used, 986 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 710 MiB/s wr, 2.65k op/s 2024-04-25T00:38:11.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:11 smithi130 ceph-mon[32525]: pgmap v149: 129 pgs: 129 active+clean; 41 GiB data, 87 GiB used, 986 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 710 MiB/s wr, 2.65k op/s 2024-04-25T00:38:13.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:13 smithi032 ceph-mon[29357]: pgmap v150: 129 pgs: 129 active+clean; 41 GiB data, 88 GiB used, 985 GiB / 1.0 TiB avail; 642 KiB/s rd, 481 MiB/s wr, 2.49k op/s 2024-04-25T00:38:13.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:13 smithi121 ceph-mon[32231]: pgmap v150: 129 pgs: 129 active+clean; 41 GiB data, 88 GiB used, 985 GiB / 1.0 TiB avail; 642 KiB/s rd, 481 MiB/s wr, 2.49k op/s 2024-04-25T00:38:13.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:13 smithi130 ceph-mon[32525]: pgmap v150: 129 pgs: 129 active+clean; 41 GiB data, 88 GiB used, 985 GiB / 1.0 TiB avail; 642 KiB/s rd, 481 MiB/s wr, 2.49k op/s 2024-04-25T00:38:15.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:15 smithi032 ceph-mon[29357]: pgmap v151: 129 pgs: 129 active+clean; 41 GiB data, 88 GiB used, 985 GiB / 1.0 TiB avail; 641 KiB/s rd, 461 MiB/s wr, 2.63k op/s 2024-04-25T00:38:15.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:15 smithi121 ceph-mon[32231]: pgmap v151: 129 pgs: 129 active+clean; 41 GiB data, 88 GiB used, 985 GiB / 1.0 TiB avail; 641 KiB/s rd, 461 MiB/s wr, 2.63k op/s 2024-04-25T00:38:15.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:15 smithi130 ceph-mon[32525]: pgmap v151: 129 pgs: 129 active+clean; 41 GiB data, 88 GiB used, 985 GiB / 1.0 TiB avail; 641 KiB/s rd, 461 MiB/s wr, 2.63k op/s 2024-04-25T00:38:17.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:17 smithi032 ceph-mon[29357]: pgmap v152: 129 pgs: 129 active+clean; 42 GiB data, 89 GiB used, 984 GiB / 1.0 TiB avail; 267 KiB/s rd, 548 MiB/s wr, 4.12k op/s 2024-04-25T00:38:17.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:17 smithi121 ceph-mon[32231]: pgmap v152: 129 pgs: 129 active+clean; 42 GiB data, 89 GiB used, 984 GiB / 1.0 TiB avail; 267 KiB/s rd, 548 MiB/s wr, 4.12k op/s 2024-04-25T00:38:17.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:17 smithi130 ceph-mon[32525]: pgmap v152: 129 pgs: 129 active+clean; 42 GiB data, 89 GiB used, 984 GiB / 1.0 TiB avail; 267 KiB/s rd, 548 MiB/s wr, 4.12k op/s 2024-04-25T00:38:19.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:19 smithi032 ceph-mon[29357]: pgmap v153: 129 pgs: 129 active+clean; 42 GiB data, 89 GiB used, 984 GiB / 1.0 TiB avail; 140 KiB/s rd, 370 MiB/s wr, 3.72k op/s 2024-04-25T00:38:19.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:19 smithi121 ceph-mon[32231]: pgmap v153: 129 pgs: 129 active+clean; 42 GiB data, 89 GiB used, 984 GiB / 1.0 TiB avail; 140 KiB/s rd, 370 MiB/s wr, 3.72k op/s 2024-04-25T00:38:19.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:19 smithi130 ceph-mon[32525]: pgmap v153: 129 pgs: 129 active+clean; 42 GiB data, 89 GiB used, 984 GiB / 1.0 TiB avail; 140 KiB/s rd, 370 MiB/s wr, 3.72k op/s 2024-04-25T00:38:21.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:21 smithi032 ceph-mon[29357]: pgmap v154: 129 pgs: 129 active+clean; 44 GiB data, 92 GiB used, 980 GiB / 1.0 TiB avail; 161 KiB/s rd, 559 MiB/s wr, 4.79k op/s 2024-04-25T00:38:21.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:21 smithi121 ceph-mon[32231]: pgmap v154: 129 pgs: 129 active+clean; 44 GiB data, 92 GiB used, 980 GiB / 1.0 TiB avail; 161 KiB/s rd, 559 MiB/s wr, 4.79k op/s 2024-04-25T00:38:21.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:21 smithi130 ceph-mon[32525]: pgmap v154: 129 pgs: 129 active+clean; 44 GiB data, 92 GiB used, 980 GiB / 1.0 TiB avail; 161 KiB/s rd, 559 MiB/s wr, 4.79k op/s 2024-04-25T00:38:23.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:23 smithi121 ceph-mon[32231]: pgmap v155: 129 pgs: 129 active+clean; 44 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 180 KiB/s rd, 445 MiB/s wr, 3.64k op/s 2024-04-25T00:38:23.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:23 smithi130 ceph-mon[32525]: pgmap v155: 129 pgs: 129 active+clean; 44 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 180 KiB/s rd, 445 MiB/s wr, 3.64k op/s 2024-04-25T00:38:23.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:23 smithi032 ceph-mon[29357]: pgmap v155: 129 pgs: 129 active+clean; 44 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 180 KiB/s rd, 445 MiB/s wr, 3.64k op/s 2024-04-25T00:38:25.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:25 smithi121 ceph-mon[32231]: pgmap v156: 129 pgs: 129 active+clean; 44 GiB data, 94 GiB used, 979 GiB / 1.0 TiB avail; 109 KiB/s rd, 434 MiB/s wr, 3.44k op/s 2024-04-25T00:38:25.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:25 smithi130 ceph-mon[32525]: pgmap v156: 129 pgs: 129 active+clean; 44 GiB data, 94 GiB used, 979 GiB / 1.0 TiB avail; 109 KiB/s rd, 434 MiB/s wr, 3.44k op/s 2024-04-25T00:38:25.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:25 smithi032 ceph-mon[29357]: pgmap v156: 129 pgs: 129 active+clean; 44 GiB data, 94 GiB used, 979 GiB / 1.0 TiB avail; 109 KiB/s rd, 434 MiB/s wr, 3.44k op/s 2024-04-25T00:38:27.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:27 smithi121 ceph-mon[32231]: pgmap v157: 129 pgs: 129 active+clean; 45 GiB data, 96 GiB used, 977 GiB / 1.0 TiB avail; 375 KiB/s rd, 570 MiB/s wr, 4.23k op/s 2024-04-25T00:38:27.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:27 smithi130 ceph-mon[32525]: pgmap v157: 129 pgs: 129 active+clean; 45 GiB data, 96 GiB used, 977 GiB / 1.0 TiB avail; 375 KiB/s rd, 570 MiB/s wr, 4.23k op/s 2024-04-25T00:38:27.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:27 smithi032 ceph-mon[29357]: pgmap v157: 129 pgs: 129 active+clean; 45 GiB data, 96 GiB used, 977 GiB / 1.0 TiB avail; 375 KiB/s rd, 570 MiB/s wr, 4.23k op/s 2024-04-25T00:38:29.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:29 smithi121 ceph-mon[32231]: pgmap v158: 129 pgs: 129 active+clean; 45 GiB data, 97 GiB used, 976 GiB / 1.0 TiB avail; 327 KiB/s rd, 444 MiB/s wr, 2.76k op/s 2024-04-25T00:38:29.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:29 smithi121 ceph-mon[32231]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:38:29.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:29 smithi130 ceph-mon[32525]: pgmap v158: 129 pgs: 129 active+clean; 45 GiB data, 97 GiB used, 976 GiB / 1.0 TiB avail; 327 KiB/s rd, 444 MiB/s wr, 2.76k op/s 2024-04-25T00:38:29.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:29 smithi130 ceph-mon[32525]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:38:29.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:29 smithi032 ceph-mon[29357]: pgmap v158: 129 pgs: 129 active+clean; 45 GiB data, 97 GiB used, 976 GiB / 1.0 TiB avail; 327 KiB/s rd, 444 MiB/s wr, 2.76k op/s 2024-04-25T00:38:29.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:29 smithi032 ceph-mon[29357]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:38:31.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:31 smithi121 ceph-mon[32231]: pgmap v159: 129 pgs: 129 active+clean; 47 GiB data, 98 GiB used, 974 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 615 MiB/s wr, 3.03k op/s 2024-04-25T00:38:31.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:31 smithi130 ceph-mon[32525]: pgmap v159: 129 pgs: 129 active+clean; 47 GiB data, 98 GiB used, 974 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 615 MiB/s wr, 3.03k op/s 2024-04-25T00:38:31.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:31 smithi032 ceph-mon[29357]: pgmap v159: 129 pgs: 129 active+clean; 47 GiB data, 98 GiB used, 974 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 615 MiB/s wr, 3.03k op/s 2024-04-25T00:38:33.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:33 smithi121 ceph-mon[32231]: pgmap v160: 129 pgs: 129 active+clean; 47 GiB data, 99 GiB used, 973 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 474 MiB/s wr, 2.06k op/s 2024-04-25T00:38:33.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:33 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:38:33.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:33 smithi130 ceph-mon[32525]: pgmap v160: 129 pgs: 129 active+clean; 47 GiB data, 99 GiB used, 973 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 474 MiB/s wr, 2.06k op/s 2024-04-25T00:38:33.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:33 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:38:33.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:33 smithi032 ceph-mon[29357]: pgmap v160: 129 pgs: 129 active+clean; 47 GiB data, 99 GiB used, 973 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 474 MiB/s wr, 2.06k op/s 2024-04-25T00:38:33.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:33 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:38:35.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:35 smithi121 ceph-mon[32231]: pgmap v161: 129 pgs: 129 active+clean; 47 GiB data, 100 GiB used, 973 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 446 MiB/s wr, 1.85k op/s 2024-04-25T00:38:35.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:35 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:38:35.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:35 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:38:35.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:35 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:38:35.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:35 smithi130 ceph-mon[32525]: pgmap v161: 129 pgs: 129 active+clean; 47 GiB data, 100 GiB used, 973 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 446 MiB/s wr, 1.85k op/s 2024-04-25T00:38:35.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:35 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:38:35.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:35 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:38:35.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:35 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:38:35.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:35 smithi032 ceph-mon[29357]: pgmap v161: 129 pgs: 129 active+clean; 47 GiB data, 100 GiB used, 973 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 446 MiB/s wr, 1.85k op/s 2024-04-25T00:38:35.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:35 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:38:35.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:35 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:38:35.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:35 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:38:37.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:37 smithi130 ceph-mon[32525]: pgmap v162: 129 pgs: 129 active+clean; 48 GiB data, 103 GiB used, 970 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 615 MiB/s wr, 2.33k op/s 2024-04-25T00:38:37.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:37 smithi032 ceph-mon[29357]: pgmap v162: 129 pgs: 129 active+clean; 48 GiB data, 103 GiB used, 970 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 615 MiB/s wr, 2.33k op/s 2024-04-25T00:38:37.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:37 smithi121 ceph-mon[32231]: pgmap v162: 129 pgs: 129 active+clean; 48 GiB data, 103 GiB used, 970 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 615 MiB/s wr, 2.33k op/s 2024-04-25T00:38:39.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:39 smithi130 ceph-mon[32525]: pgmap v163: 129 pgs: 129 active+clean; 49 GiB data, 104 GiB used, 969 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 495 MiB/s wr, 1.45k op/s 2024-04-25T00:38:39.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:39 smithi032 ceph-mon[29357]: pgmap v163: 129 pgs: 129 active+clean; 49 GiB data, 104 GiB used, 969 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 495 MiB/s wr, 1.45k op/s 2024-04-25T00:38:39.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:39 smithi121 ceph-mon[32231]: pgmap v163: 129 pgs: 129 active+clean; 49 GiB data, 104 GiB used, 969 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 495 MiB/s wr, 1.45k op/s 2024-04-25T00:38:41.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:41 smithi130 ceph-mon[32525]: pgmap v164: 129 pgs: 129 active+clean; 50 GiB data, 106 GiB used, 967 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 671 MiB/s wr, 1.77k op/s 2024-04-25T00:38:41.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:41 smithi032 ceph-mon[29357]: pgmap v164: 129 pgs: 129 active+clean; 50 GiB data, 106 GiB used, 967 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 671 MiB/s wr, 1.77k op/s 2024-04-25T00:38:41.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:41 smithi121 ceph-mon[32231]: pgmap v164: 129 pgs: 129 active+clean; 50 GiB data, 106 GiB used, 967 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 671 MiB/s wr, 1.77k op/s 2024-04-25T00:38:43.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:43 smithi130 ceph-mon[32525]: pgmap v165: 129 pgs: 129 active+clean; 50 GiB data, 106 GiB used, 967 GiB / 1.0 TiB avail; 530 KiB/s rd, 533 MiB/s wr, 1.47k op/s 2024-04-25T00:38:43.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:43 smithi032 ceph-mon[29357]: pgmap v165: 129 pgs: 129 active+clean; 50 GiB data, 106 GiB used, 967 GiB / 1.0 TiB avail; 530 KiB/s rd, 533 MiB/s wr, 1.47k op/s 2024-04-25T00:38:43.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:43 smithi121 ceph-mon[32231]: pgmap v165: 129 pgs: 129 active+clean; 50 GiB data, 106 GiB used, 967 GiB / 1.0 TiB avail; 530 KiB/s rd, 533 MiB/s wr, 1.47k op/s 2024-04-25T00:38:45.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:45 smithi130 ceph-mon[32525]: pgmap v166: 129 pgs: 129 active+clean; 50 GiB data, 107 GiB used, 966 GiB / 1.0 TiB avail; 876 KiB/s rd, 525 MiB/s wr, 1.48k op/s 2024-04-25T00:38:45.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:45 smithi032 ceph-mon[29357]: pgmap v166: 129 pgs: 129 active+clean; 50 GiB data, 107 GiB used, 966 GiB / 1.0 TiB avail; 876 KiB/s rd, 525 MiB/s wr, 1.48k op/s 2024-04-25T00:38:45.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:45 smithi121 ceph-mon[32231]: pgmap v166: 129 pgs: 129 active+clean; 50 GiB data, 107 GiB used, 966 GiB / 1.0 TiB avail; 876 KiB/s rd, 525 MiB/s wr, 1.48k op/s 2024-04-25T00:38:47.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:47 smithi130 ceph-mon[32525]: pgmap v167: 129 pgs: 129 active+clean; 50 GiB data, 107 GiB used, 966 GiB / 1.0 TiB avail; 1.2 MiB/s rd, 744 MiB/s wr, 1.81k op/s 2024-04-25T00:38:47.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:47 smithi032 ceph-mon[29357]: pgmap v167: 129 pgs: 129 active+clean; 50 GiB data, 107 GiB used, 966 GiB / 1.0 TiB avail; 1.2 MiB/s rd, 744 MiB/s wr, 1.81k op/s 2024-04-25T00:38:47.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:47 smithi121 ceph-mon[32231]: pgmap v167: 129 pgs: 129 active+clean; 50 GiB data, 107 GiB used, 966 GiB / 1.0 TiB avail; 1.2 MiB/s rd, 744 MiB/s wr, 1.81k op/s 2024-04-25T00:38:49.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:49 smithi032 ceph-mon[29357]: pgmap v168: 129 pgs: 129 active+clean; 50 GiB data, 107 GiB used, 966 GiB / 1.0 TiB avail; 1.2 MiB/s rd, 605 MiB/s wr, 1.28k op/s 2024-04-25T00:38:49.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:49 smithi121 ceph-mon[32231]: pgmap v168: 129 pgs: 129 active+clean; 50 GiB data, 107 GiB used, 966 GiB / 1.0 TiB avail; 1.2 MiB/s rd, 605 MiB/s wr, 1.28k op/s 2024-04-25T00:38:49.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:49 smithi130 ceph-mon[32525]: pgmap v168: 129 pgs: 129 active+clean; 50 GiB data, 107 GiB used, 966 GiB / 1.0 TiB avail; 1.2 MiB/s rd, 605 MiB/s wr, 1.28k op/s 2024-04-25T00:38:51.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:51 smithi032 ceph-mon[29357]: pgmap v169: 129 pgs: 129 active+clean; 50 GiB data, 107 GiB used, 966 GiB / 1.0 TiB avail; 1.2 MiB/s rd, 774 MiB/s wr, 1.53k op/s 2024-04-25T00:38:51.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:51 smithi121 ceph-mon[32231]: pgmap v169: 129 pgs: 129 active+clean; 50 GiB data, 107 GiB used, 966 GiB / 1.0 TiB avail; 1.2 MiB/s rd, 774 MiB/s wr, 1.53k op/s 2024-04-25T00:38:51.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:51 smithi130 ceph-mon[32525]: pgmap v169: 129 pgs: 129 active+clean; 50 GiB data, 107 GiB used, 966 GiB / 1.0 TiB avail; 1.2 MiB/s rd, 774 MiB/s wr, 1.53k op/s 2024-04-25T00:38:53.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:53 smithi032 ceph-mon[29357]: pgmap v170: 129 pgs: 129 active+clean; 50 GiB data, 107 GiB used, 966 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 611 MiB/s wr, 1.16k op/s 2024-04-25T00:38:53.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:53 smithi121 ceph-mon[32231]: pgmap v170: 129 pgs: 129 active+clean; 50 GiB data, 107 GiB used, 966 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 611 MiB/s wr, 1.16k op/s 2024-04-25T00:38:53.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:53 smithi130 ceph-mon[32525]: pgmap v170: 129 pgs: 129 active+clean; 50 GiB data, 107 GiB used, 966 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 611 MiB/s wr, 1.16k op/s 2024-04-25T00:38:55.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:55 smithi032 ceph-mon[29357]: pgmap v171: 129 pgs: 129 active+clean; 50 GiB data, 107 GiB used, 966 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 613 MiB/s wr, 1.21k op/s 2024-04-25T00:38:55.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:55 smithi121 ceph-mon[32231]: pgmap v171: 129 pgs: 129 active+clean; 50 GiB data, 107 GiB used, 966 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 613 MiB/s wr, 1.21k op/s 2024-04-25T00:38:55.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:55 smithi130 ceph-mon[32525]: pgmap v171: 129 pgs: 129 active+clean; 50 GiB data, 107 GiB used, 966 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 613 MiB/s wr, 1.21k op/s 2024-04-25T00:38:57.694 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:57 smithi130 ceph-mon[32525]: pgmap v172: 129 pgs: 129 active+clean; 50 GiB data, 108 GiB used, 965 GiB / 1.0 TiB avail; 1.0 MiB/s rd, 777 MiB/s wr, 1.52k op/s 2024-04-25T00:38:57.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:57 smithi032 ceph-mon[29357]: pgmap v172: 129 pgs: 129 active+clean; 50 GiB data, 108 GiB used, 965 GiB / 1.0 TiB avail; 1.0 MiB/s rd, 777 MiB/s wr, 1.52k op/s 2024-04-25T00:38:57.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:57 smithi121 ceph-mon[32231]: pgmap v172: 129 pgs: 129 active+clean; 50 GiB data, 108 GiB used, 965 GiB / 1.0 TiB avail; 1.0 MiB/s rd, 777 MiB/s wr, 1.52k op/s 2024-04-25T00:38:59.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:38:59 smithi032 ceph-mon[29357]: pgmap v173: 129 pgs: 129 active+clean; 50 GiB data, 108 GiB used, 965 GiB / 1.0 TiB avail; 894 KiB/s rd, 613 MiB/s wr, 1.29k op/s 2024-04-25T00:38:59.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:38:59 smithi121 ceph-mon[32231]: pgmap v173: 129 pgs: 129 active+clean; 50 GiB data, 108 GiB used, 965 GiB / 1.0 TiB avail; 894 KiB/s rd, 613 MiB/s wr, 1.29k op/s 2024-04-25T00:38:59.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:38:59 smithi130 ceph-mon[32525]: pgmap v173: 129 pgs: 129 active+clean; 50 GiB data, 108 GiB used, 965 GiB / 1.0 TiB avail; 894 KiB/s rd, 613 MiB/s wr, 1.29k op/s 2024-04-25T00:39:01.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:01 smithi032 ceph-mon[29357]: pgmap v174: 129 pgs: 129 active+clean; 50 GiB data, 108 GiB used, 965 GiB / 1.0 TiB avail; 797 KiB/s rd, 823 MiB/s wr, 1.82k op/s 2024-04-25T00:39:01.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:01 smithi121 ceph-mon[32231]: pgmap v174: 129 pgs: 129 active+clean; 50 GiB data, 108 GiB used, 965 GiB / 1.0 TiB avail; 797 KiB/s rd, 823 MiB/s wr, 1.82k op/s 2024-04-25T00:39:01.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:01 smithi130 ceph-mon[32525]: pgmap v174: 129 pgs: 129 active+clean; 50 GiB data, 108 GiB used, 965 GiB / 1.0 TiB avail; 797 KiB/s rd, 823 MiB/s wr, 1.82k op/s 2024-04-25T00:39:03.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:03 smithi032 ceph-mon[29357]: pgmap v175: 129 pgs: 129 active+clean; 50 GiB data, 108 GiB used, 965 GiB / 1.0 TiB avail; 904 KiB/s rd, 644 MiB/s wr, 1.50k op/s 2024-04-25T00:39:03.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:03 smithi121 ceph-mon[32231]: pgmap v175: 129 pgs: 129 active+clean; 50 GiB data, 108 GiB used, 965 GiB / 1.0 TiB avail; 904 KiB/s rd, 644 MiB/s wr, 1.50k op/s 2024-04-25T00:39:03.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:03 smithi130 ceph-mon[32525]: pgmap v175: 129 pgs: 129 active+clean; 50 GiB data, 108 GiB used, 965 GiB / 1.0 TiB avail; 904 KiB/s rd, 644 MiB/s wr, 1.50k op/s 2024-04-25T00:39:05.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:05 smithi032 ceph-mon[29357]: pgmap v176: 129 pgs: 129 active+clean; 50 GiB data, 108 GiB used, 965 GiB / 1.0 TiB avail; 630 KiB/s rd, 655 MiB/s wr, 1.51k op/s 2024-04-25T00:39:05.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:05 smithi121 ceph-mon[32231]: pgmap v176: 129 pgs: 129 active+clean; 50 GiB data, 108 GiB used, 965 GiB / 1.0 TiB avail; 630 KiB/s rd, 655 MiB/s wr, 1.51k op/s 2024-04-25T00:39:05.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:05 smithi130 ceph-mon[32525]: pgmap v176: 129 pgs: 129 active+clean; 50 GiB data, 108 GiB used, 965 GiB / 1.0 TiB avail; 630 KiB/s rd, 655 MiB/s wr, 1.51k op/s 2024-04-25T00:39:07.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:07 smithi032 ceph-mon[29357]: pgmap v177: 129 pgs: 129 active+clean; 50 GiB data, 108 GiB used, 965 GiB / 1.0 TiB avail; 586 KiB/s rd, 824 MiB/s wr, 2.04k op/s 2024-04-25T00:39:07.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:07 smithi121 ceph-mon[32231]: pgmap v177: 129 pgs: 129 active+clean; 50 GiB data, 108 GiB used, 965 GiB / 1.0 TiB avail; 586 KiB/s rd, 824 MiB/s wr, 2.04k op/s 2024-04-25T00:39:07.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:07 smithi130 ceph-mon[32525]: pgmap v177: 129 pgs: 129 active+clean; 50 GiB data, 108 GiB used, 965 GiB / 1.0 TiB avail; 586 KiB/s rd, 824 MiB/s wr, 2.04k op/s 2024-04-25T00:39:09.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:09 smithi032 ceph-mon[29357]: pgmap v178: 129 pgs: 129 active+clean; 50 GiB data, 108 GiB used, 965 GiB / 1.0 TiB avail; 621 KiB/s rd, 665 MiB/s wr, 1.79k op/s 2024-04-25T00:39:09.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:09 smithi121 ceph-mon[32231]: pgmap v178: 129 pgs: 129 active+clean; 50 GiB data, 108 GiB used, 965 GiB / 1.0 TiB avail; 621 KiB/s rd, 665 MiB/s wr, 1.79k op/s 2024-04-25T00:39:09.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:09 smithi130 ceph-mon[32525]: pgmap v178: 129 pgs: 129 active+clean; 50 GiB data, 108 GiB used, 965 GiB / 1.0 TiB avail; 621 KiB/s rd, 665 MiB/s wr, 1.79k op/s 2024-04-25T00:39:11.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:11 smithi121 ceph-mon[32231]: pgmap v179: 129 pgs: 129 active+clean; 51 GiB data, 110 GiB used, 963 GiB / 1.0 TiB avail; 773 KiB/s rd, 781 MiB/s wr, 2.81k op/s 2024-04-25T00:39:11.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:11 smithi130 ceph-mon[32525]: pgmap v179: 129 pgs: 129 active+clean; 51 GiB data, 110 GiB used, 963 GiB / 1.0 TiB avail; 773 KiB/s rd, 781 MiB/s wr, 2.81k op/s 2024-04-25T00:39:11.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:11 smithi032 ceph-mon[29357]: pgmap v179: 129 pgs: 129 active+clean; 51 GiB data, 110 GiB used, 963 GiB / 1.0 TiB avail; 773 KiB/s rd, 781 MiB/s wr, 2.81k op/s 2024-04-25T00:39:13.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:13 smithi121 ceph-mon[32231]: pgmap v180: 129 pgs: 129 active+clean; 51 GiB data, 110 GiB used, 963 GiB / 1.0 TiB avail; 649 KiB/s rd, 521 MiB/s wr, 2.19k op/s 2024-04-25T00:39:13.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:13 smithi130 ceph-mon[32525]: pgmap v180: 129 pgs: 129 active+clean; 51 GiB data, 110 GiB used, 963 GiB / 1.0 TiB avail; 649 KiB/s rd, 521 MiB/s wr, 2.19k op/s 2024-04-25T00:39:13.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:13 smithi032 ceph-mon[29357]: pgmap v180: 129 pgs: 129 active+clean; 51 GiB data, 110 GiB used, 963 GiB / 1.0 TiB avail; 649 KiB/s rd, 521 MiB/s wr, 2.19k op/s 2024-04-25T00:39:15.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:15 smithi121 ceph-mon[32231]: pgmap v181: 129 pgs: 129 active+clean; 52 GiB data, 110 GiB used, 962 GiB / 1.0 TiB avail; 505 KiB/s rd, 526 MiB/s wr, 2.30k op/s 2024-04-25T00:39:15.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:15 smithi130 ceph-mon[32525]: pgmap v181: 129 pgs: 129 active+clean; 52 GiB data, 110 GiB used, 962 GiB / 1.0 TiB avail; 505 KiB/s rd, 526 MiB/s wr, 2.30k op/s 2024-04-25T00:39:15.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:15 smithi032 ceph-mon[29357]: pgmap v181: 129 pgs: 129 active+clean; 52 GiB data, 110 GiB used, 962 GiB / 1.0 TiB avail; 505 KiB/s rd, 526 MiB/s wr, 2.30k op/s 2024-04-25T00:39:17.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:17 smithi121 ceph-mon[32231]: pgmap v182: 129 pgs: 129 active+clean; 53 GiB data, 113 GiB used, 959 GiB / 1.0 TiB avail; 1.2 MiB/s rd, 633 MiB/s wr, 3.07k op/s 2024-04-25T00:39:17.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:17 smithi130 ceph-mon[32525]: pgmap v182: 129 pgs: 129 active+clean; 53 GiB data, 113 GiB used, 959 GiB / 1.0 TiB avail; 1.2 MiB/s rd, 633 MiB/s wr, 3.07k op/s 2024-04-25T00:39:17.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:17 smithi032 ceph-mon[29357]: pgmap v182: 129 pgs: 129 active+clean; 53 GiB data, 113 GiB used, 959 GiB / 1.0 TiB avail; 1.2 MiB/s rd, 633 MiB/s wr, 3.07k op/s 2024-04-25T00:39:19.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:19 smithi121 ceph-mon[32231]: pgmap v183: 129 pgs: 129 active+clean; 53 GiB data, 114 GiB used, 959 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 448 MiB/s wr, 2.56k op/s 2024-04-25T00:39:19.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:19 smithi130 ceph-mon[32525]: pgmap v183: 129 pgs: 129 active+clean; 53 GiB data, 114 GiB used, 959 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 448 MiB/s wr, 2.56k op/s 2024-04-25T00:39:19.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:19 smithi032 ceph-mon[29357]: pgmap v183: 129 pgs: 129 active+clean; 53 GiB data, 114 GiB used, 959 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 448 MiB/s wr, 2.56k op/s 2024-04-25T00:39:21.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:21 smithi121 ceph-mon[32231]: pgmap v184: 129 pgs: 129 active+clean; 54 GiB data, 115 GiB used, 957 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 518 MiB/s wr, 3.05k op/s 2024-04-25T00:39:21.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:21 smithi130 ceph-mon[32525]: pgmap v184: 129 pgs: 129 active+clean; 54 GiB data, 115 GiB used, 957 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 518 MiB/s wr, 3.05k op/s 2024-04-25T00:39:21.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:21 smithi032 ceph-mon[29357]: pgmap v184: 129 pgs: 129 active+clean; 54 GiB data, 115 GiB used, 957 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 518 MiB/s wr, 3.05k op/s 2024-04-25T00:39:23.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:23 smithi121 ceph-mon[32231]: pgmap v185: 129 pgs: 129 active+clean; 54 GiB data, 115 GiB used, 957 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 341 MiB/s wr, 1.92k op/s 2024-04-25T00:39:23.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:23 smithi130 ceph-mon[32525]: pgmap v185: 129 pgs: 129 active+clean; 54 GiB data, 115 GiB used, 957 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 341 MiB/s wr, 1.92k op/s 2024-04-25T00:39:24.000 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:23 smithi032 ceph-mon[29357]: pgmap v185: 129 pgs: 129 active+clean; 54 GiB data, 115 GiB used, 957 GiB / 1.0 TiB avail; 1.1 MiB/s rd, 341 MiB/s wr, 1.92k op/s 2024-04-25T00:39:25.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:25 smithi121 ceph-mon[32231]: pgmap v186: 129 pgs: 129 active+clean; 54 GiB data, 116 GiB used, 957 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 370 MiB/s wr, 1.98k op/s 2024-04-25T00:39:25.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:25 smithi130 ceph-mon[32525]: pgmap v186: 129 pgs: 129 active+clean; 54 GiB data, 116 GiB used, 957 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 370 MiB/s wr, 1.98k op/s 2024-04-25T00:39:26.002 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:25 smithi032 ceph-mon[29357]: pgmap v186: 129 pgs: 129 active+clean; 54 GiB data, 116 GiB used, 957 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 370 MiB/s wr, 1.98k op/s 2024-04-25T00:39:27.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:27 smithi121 ceph-mon[32231]: pgmap v187: 129 pgs: 129 active+clean; 56 GiB data, 118 GiB used, 955 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 530 MiB/s wr, 2.04k op/s 2024-04-25T00:39:27.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:27 smithi130 ceph-mon[32525]: pgmap v187: 129 pgs: 129 active+clean; 56 GiB data, 118 GiB used, 955 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 530 MiB/s wr, 2.04k op/s 2024-04-25T00:39:27.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:27 smithi032 ceph-mon[29357]: pgmap v187: 129 pgs: 129 active+clean; 56 GiB data, 118 GiB used, 955 GiB / 1.0 TiB avail; 1.4 MiB/s rd, 530 MiB/s wr, 2.04k op/s 2024-04-25T00:39:29.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:29 smithi121 ceph-mon[32231]: pgmap v188: 129 pgs: 129 active+clean; 56 GiB data, 119 GiB used, 954 GiB / 1.0 TiB avail; 921 KiB/s rd, 405 MiB/s wr, 1.18k op/s 2024-04-25T00:39:29.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:29 smithi130 ceph-mon[32525]: pgmap v188: 129 pgs: 129 active+clean; 56 GiB data, 119 GiB used, 954 GiB / 1.0 TiB avail; 921 KiB/s rd, 405 MiB/s wr, 1.18k op/s 2024-04-25T00:39:29.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:29 smithi032 ceph-mon[29357]: pgmap v188: 129 pgs: 129 active+clean; 56 GiB data, 119 GiB used, 954 GiB / 1.0 TiB avail; 921 KiB/s rd, 405 MiB/s wr, 1.18k op/s 2024-04-25T00:39:31.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:31 smithi121 ceph-mon[32231]: pgmap v189: 129 pgs: 129 active+clean; 56 GiB data, 120 GiB used, 953 GiB / 1.0 TiB avail; 1.2 MiB/s rd, 566 MiB/s wr, 1.34k op/s 2024-04-25T00:39:31.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:31 smithi130 ceph-mon[32525]: pgmap v189: 129 pgs: 129 active+clean; 56 GiB data, 120 GiB used, 953 GiB / 1.0 TiB avail; 1.2 MiB/s rd, 566 MiB/s wr, 1.34k op/s 2024-04-25T00:39:31.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:31 smithi032 ceph-mon[29357]: pgmap v189: 129 pgs: 129 active+clean; 56 GiB data, 120 GiB used, 953 GiB / 1.0 TiB avail; 1.2 MiB/s rd, 566 MiB/s wr, 1.34k op/s 2024-04-25T00:39:33.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:33 smithi121 ceph-mon[32231]: pgmap v190: 129 pgs: 129 active+clean; 56 GiB data, 120 GiB used, 953 GiB / 1.0 TiB avail; 855 KiB/s rd, 450 MiB/s wr, 680 op/s 2024-04-25T00:39:33.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:33 smithi121 ceph-mon[32231]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:39:33.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:33 smithi130 ceph-mon[32525]: pgmap v190: 129 pgs: 129 active+clean; 56 GiB data, 120 GiB used, 953 GiB / 1.0 TiB avail; 855 KiB/s rd, 450 MiB/s wr, 680 op/s 2024-04-25T00:39:33.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:33 smithi130 ceph-mon[32525]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:39:33.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:33 smithi032 ceph-mon[29357]: pgmap v190: 129 pgs: 129 active+clean; 56 GiB data, 120 GiB used, 953 GiB / 1.0 TiB avail; 855 KiB/s rd, 450 MiB/s wr, 680 op/s 2024-04-25T00:39:33.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:33 smithi032 ceph-mon[29357]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:39:35.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:35 smithi130 ceph-mon[32525]: pgmap v191: 129 pgs: 129 active+clean; 56 GiB data, 121 GiB used, 952 GiB / 1.0 TiB avail; 856 KiB/s rd, 485 MiB/s wr, 784 op/s 2024-04-25T00:39:35.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:35 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:39:35.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:35 smithi032 ceph-mon[29357]: pgmap v191: 129 pgs: 129 active+clean; 56 GiB data, 121 GiB used, 952 GiB / 1.0 TiB avail; 856 KiB/s rd, 485 MiB/s wr, 784 op/s 2024-04-25T00:39:35.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:35 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:39:36.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:35 smithi121 ceph-mon[32231]: pgmap v191: 129 pgs: 129 active+clean; 56 GiB data, 121 GiB used, 952 GiB / 1.0 TiB avail; 856 KiB/s rd, 485 MiB/s wr, 784 op/s 2024-04-25T00:39:36.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:35 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:39:36.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:36 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:39:36.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:36 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:39:36.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:36 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:39:36.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:36 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:39:36.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:36 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:39:36.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:36 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:39:37.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:36 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:39:37.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:36 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:39:37.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:36 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:39:37.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:37 smithi130 ceph-mon[32525]: pgmap v192: 129 pgs: 129 active+clean; 59 GiB data, 124 GiB used, 949 GiB / 1.0 TiB avail; 964 KiB/s rd, 679 MiB/s wr, 1.12k op/s 2024-04-25T00:39:38.001 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:37 smithi032 ceph-mon[29357]: pgmap v192: 129 pgs: 129 active+clean; 59 GiB data, 124 GiB used, 949 GiB / 1.0 TiB avail; 964 KiB/s rd, 679 MiB/s wr, 1.12k op/s 2024-04-25T00:39:38.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:37 smithi121 ceph-mon[32231]: pgmap v192: 129 pgs: 129 active+clean; 59 GiB data, 124 GiB used, 949 GiB / 1.0 TiB avail; 964 KiB/s rd, 679 MiB/s wr, 1.12k op/s 2024-04-25T00:39:39.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:39 smithi130 ceph-mon[32525]: pgmap v193: 129 pgs: 129 active+clean; 59 GiB data, 125 GiB used, 947 GiB / 1.0 TiB avail; 736 KiB/s rd, 526 MiB/s wr, 993 op/s 2024-04-25T00:39:40.002 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:39 smithi032 ceph-mon[29357]: pgmap v193: 129 pgs: 129 active+clean; 59 GiB data, 125 GiB used, 947 GiB / 1.0 TiB avail; 736 KiB/s rd, 526 MiB/s wr, 993 op/s 2024-04-25T00:39:40.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:39 smithi121 ceph-mon[32231]: pgmap v193: 129 pgs: 129 active+clean; 59 GiB data, 125 GiB used, 947 GiB / 1.0 TiB avail; 736 KiB/s rd, 526 MiB/s wr, 993 op/s 2024-04-25T00:39:41.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:41 smithi130 ceph-mon[32525]: pgmap v194: 129 pgs: 129 active+clean; 60 GiB data, 128 GiB used, 945 GiB / 1.0 TiB avail; 762 KiB/s rd, 677 MiB/s wr, 1.36k op/s 2024-04-25T00:39:41.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:41 smithi032 ceph-mon[29357]: pgmap v194: 129 pgs: 129 active+clean; 60 GiB data, 128 GiB used, 945 GiB / 1.0 TiB avail; 762 KiB/s rd, 677 MiB/s wr, 1.36k op/s 2024-04-25T00:39:42.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:41 smithi121 ceph-mon[32231]: pgmap v194: 129 pgs: 129 active+clean; 60 GiB data, 128 GiB used, 945 GiB / 1.0 TiB avail; 762 KiB/s rd, 677 MiB/s wr, 1.36k op/s 2024-04-25T00:39:43.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:43 smithi130 ceph-mon[32525]: pgmap v195: 129 pgs: 129 active+clean; 60 GiB data, 128 GiB used, 945 GiB / 1.0 TiB avail; 477 KiB/s rd, 487 MiB/s wr, 1.02k op/s 2024-04-25T00:39:43.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:43 smithi032 ceph-mon[29357]: pgmap v195: 129 pgs: 129 active+clean; 60 GiB data, 128 GiB used, 945 GiB / 1.0 TiB avail; 477 KiB/s rd, 487 MiB/s wr, 1.02k op/s 2024-04-25T00:39:44.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:43 smithi121 ceph-mon[32231]: pgmap v195: 129 pgs: 129 active+clean; 60 GiB data, 128 GiB used, 945 GiB / 1.0 TiB avail; 477 KiB/s rd, 487 MiB/s wr, 1.02k op/s 2024-04-25T00:39:45.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:45 smithi032 ceph-mon[29357]: pgmap v196: 129 pgs: 129 active+clean; 60 GiB data, 128 GiB used, 945 GiB / 1.0 TiB avail; 641 KiB/s rd, 540 MiB/s wr, 1.13k op/s 2024-04-25T00:39:46.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:45 smithi121 ceph-mon[32231]: pgmap v196: 129 pgs: 129 active+clean; 60 GiB data, 128 GiB used, 945 GiB / 1.0 TiB avail; 641 KiB/s rd, 540 MiB/s wr, 1.13k op/s 2024-04-25T00:39:46.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:45 smithi130 ceph-mon[32525]: pgmap v196: 129 pgs: 129 active+clean; 60 GiB data, 128 GiB used, 945 GiB / 1.0 TiB avail; 641 KiB/s rd, 540 MiB/s wr, 1.13k op/s 2024-04-25T00:39:47.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:47 smithi032 ceph-mon[29357]: pgmap v197: 129 pgs: 129 active+clean; 60 GiB data, 128 GiB used, 945 GiB / 1.0 TiB avail; 772 KiB/s rd, 769 MiB/s wr, 1.41k op/s 2024-04-25T00:39:48.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:47 smithi121 ceph-mon[32231]: pgmap v197: 129 pgs: 129 active+clean; 60 GiB data, 128 GiB used, 945 GiB / 1.0 TiB avail; 772 KiB/s rd, 769 MiB/s wr, 1.41k op/s 2024-04-25T00:39:48.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:47 smithi130 ceph-mon[32525]: pgmap v197: 129 pgs: 129 active+clean; 60 GiB data, 128 GiB used, 945 GiB / 1.0 TiB avail; 772 KiB/s rd, 769 MiB/s wr, 1.41k op/s 2024-04-25T00:39:49.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:49 smithi032 ceph-mon[29357]: pgmap v198: 129 pgs: 129 active+clean; 60 GiB data, 128 GiB used, 945 GiB / 1.0 TiB avail; 443 KiB/s rd, 564 MiB/s wr, 1.07k op/s 2024-04-25T00:39:50.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:49 smithi121 ceph-mon[32231]: pgmap v198: 129 pgs: 129 active+clean; 60 GiB data, 128 GiB used, 945 GiB / 1.0 TiB avail; 443 KiB/s rd, 564 MiB/s wr, 1.07k op/s 2024-04-25T00:39:50.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:49 smithi130 ceph-mon[32525]: pgmap v198: 129 pgs: 129 active+clean; 60 GiB data, 128 GiB used, 945 GiB / 1.0 TiB avail; 443 KiB/s rd, 564 MiB/s wr, 1.07k op/s 2024-04-25T00:39:51.999 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:51 smithi032 ceph-mon[29357]: pgmap v199: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 944 GiB / 1.0 TiB avail; 1.6 MiB/s rd, 757 MiB/s wr, 1.74k op/s 2024-04-25T00:39:52.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:51 smithi121 ceph-mon[32231]: pgmap v199: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 944 GiB / 1.0 TiB avail; 1.6 MiB/s rd, 757 MiB/s wr, 1.74k op/s 2024-04-25T00:39:52.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:51 smithi130 ceph-mon[32525]: pgmap v199: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 944 GiB / 1.0 TiB avail; 1.6 MiB/s rd, 757 MiB/s wr, 1.74k op/s 2024-04-25T00:39:53.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:53 smithi032 ceph-mon[29357]: pgmap v200: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 944 GiB / 1.0 TiB avail; 1.5 MiB/s rd, 603 MiB/s wr, 1.45k op/s 2024-04-25T00:39:54.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:53 smithi121 ceph-mon[32231]: pgmap v200: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 944 GiB / 1.0 TiB avail; 1.5 MiB/s rd, 603 MiB/s wr, 1.45k op/s 2024-04-25T00:39:54.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:53 smithi130 ceph-mon[32525]: pgmap v200: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 944 GiB / 1.0 TiB avail; 1.5 MiB/s rd, 603 MiB/s wr, 1.45k op/s 2024-04-25T00:39:55.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:55 smithi032 ceph-mon[29357]: pgmap v201: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 944 GiB / 1.0 TiB avail; 1.5 MiB/s rd, 620 MiB/s wr, 1.49k op/s 2024-04-25T00:39:56.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:55 smithi121 ceph-mon[32231]: pgmap v201: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 944 GiB / 1.0 TiB avail; 1.5 MiB/s rd, 620 MiB/s wr, 1.49k op/s 2024-04-25T00:39:56.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:55 smithi130 ceph-mon[32525]: pgmap v201: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 944 GiB / 1.0 TiB avail; 1.5 MiB/s rd, 620 MiB/s wr, 1.49k op/s 2024-04-25T00:39:57.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:57 smithi032 ceph-mon[29357]: pgmap v202: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 944 GiB / 1.0 TiB avail; 1.8 MiB/s rd, 812 MiB/s wr, 2.10k op/s 2024-04-25T00:39:58.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:57 smithi121 ceph-mon[32231]: pgmap v202: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 944 GiB / 1.0 TiB avail; 1.8 MiB/s rd, 812 MiB/s wr, 2.10k op/s 2024-04-25T00:39:58.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:57 smithi130 ceph-mon[32525]: pgmap v202: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 944 GiB / 1.0 TiB avail; 1.8 MiB/s rd, 812 MiB/s wr, 2.10k op/s 2024-04-25T00:39:59.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:39:59 smithi032 ceph-mon[29357]: pgmap v203: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 944 GiB / 1.0 TiB avail; 1.6 MiB/s rd, 563 MiB/s wr, 1.78k op/s 2024-04-25T00:40:00.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:39:59 smithi121 ceph-mon[32231]: pgmap v203: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 944 GiB / 1.0 TiB avail; 1.6 MiB/s rd, 563 MiB/s wr, 1.78k op/s 2024-04-25T00:40:00.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:39:59 smithi130 ceph-mon[32525]: pgmap v203: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 944 GiB / 1.0 TiB avail; 1.6 MiB/s rd, 563 MiB/s wr, 1.78k op/s 2024-04-25T00:40:00.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:00 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-a[29353]: 2024-04-25T00:39:59.998+0000 7fe95959b700 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 1 MDSs report damaged metadata 2024-04-25T00:40:00.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:00 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-a[29353]: 2024-04-25T00:39:59.998+0000 7fe95959b700 -1 log_channel(cluster) log [ERR] : [ERR] MDS_DAMAGE: 1 MDSs report damaged metadata 2024-04-25T00:40:00.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:00 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-a[29353]: 2024-04-25T00:39:59.998+0000 7fe95959b700 -1 log_channel(cluster) log [ERR] : mds.i(mds.1): Metadata damage detected 2024-04-25T00:40:01.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:00 smithi121 ceph-mon[32231]: Health detail: HEALTH_ERR 1 MDSs report damaged metadata 2024-04-25T00:40:01.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:00 smithi121 ceph-mon[32231]: [ERR] MDS_DAMAGE: 1 MDSs report damaged metadata 2024-04-25T00:40:01.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:00 smithi121 ceph-mon[32231]: mds.i(mds.1): Metadata damage detected 2024-04-25T00:40:01.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:00 smithi130 ceph-mon[32525]: Health detail: HEALTH_ERR 1 MDSs report damaged metadata 2024-04-25T00:40:01.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:00 smithi130 ceph-mon[32525]: [ERR] MDS_DAMAGE: 1 MDSs report damaged metadata 2024-04-25T00:40:01.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:00 smithi130 ceph-mon[32525]: mds.i(mds.1): Metadata damage detected 2024-04-25T00:40:01.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:00 smithi032 ceph-mon[29357]: Health detail: HEALTH_ERR 1 MDSs report damaged metadata 2024-04-25T00:40:01.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:00 smithi032 ceph-mon[29357]: [ERR] MDS_DAMAGE: 1 MDSs report damaged metadata 2024-04-25T00:40:01.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:00 smithi032 ceph-mon[29357]: mds.i(mds.1): Metadata damage detected 2024-04-25T00:40:02.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:01 smithi121 ceph-mon[32231]: pgmap v204: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 944 GiB / 1.0 TiB avail; 2.5 MiB/s rd, 777 MiB/s wr, 2.31k op/s 2024-04-25T00:40:02.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:01 smithi130 ceph-mon[32525]: pgmap v204: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 944 GiB / 1.0 TiB avail; 2.5 MiB/s rd, 777 MiB/s wr, 2.31k op/s 2024-04-25T00:40:02.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:01 smithi032 ceph-mon[29357]: pgmap v204: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 944 GiB / 1.0 TiB avail; 2.5 MiB/s rd, 777 MiB/s wr, 2.31k op/s 2024-04-25T00:40:04.074 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:03 smithi121 ceph-mon[32231]: pgmap v205: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 944 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 572 MiB/s wr, 1.58k op/s 2024-04-25T00:40:04.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:03 smithi130 ceph-mon[32525]: pgmap v205: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 944 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 572 MiB/s wr, 1.58k op/s 2024-04-25T00:40:04.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:03 smithi032 ceph-mon[29357]: pgmap v205: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 944 GiB / 1.0 TiB avail; 1.3 MiB/s rd, 572 MiB/s wr, 1.58k op/s 2024-04-25T00:40:06.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:05 smithi121 ceph-mon[32231]: pgmap v206: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 944 GiB / 1.0 TiB avail; 1.6 MiB/s rd, 558 MiB/s wr, 1.53k op/s 2024-04-25T00:40:06.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:05 smithi130 ceph-mon[32525]: pgmap v206: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 944 GiB / 1.0 TiB avail; 1.6 MiB/s rd, 558 MiB/s wr, 1.53k op/s 2024-04-25T00:40:06.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:05 smithi032 ceph-mon[29357]: pgmap v206: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 944 GiB / 1.0 TiB avail; 1.6 MiB/s rd, 558 MiB/s wr, 1.53k op/s 2024-04-25T00:40:08.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:07 smithi121 ceph-mon[32231]: pgmap v207: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 943 GiB / 1.0 TiB avail; 1.9 MiB/s rd, 777 MiB/s wr, 2.25k op/s 2024-04-25T00:40:08.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:07 smithi130 ceph-mon[32525]: pgmap v207: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 943 GiB / 1.0 TiB avail; 1.9 MiB/s rd, 777 MiB/s wr, 2.25k op/s 2024-04-25T00:40:08.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:07 smithi032 ceph-mon[29357]: pgmap v207: 129 pgs: 129 active+clean; 60 GiB data, 129 GiB used, 943 GiB / 1.0 TiB avail; 1.9 MiB/s rd, 777 MiB/s wr, 2.25k op/s 2024-04-25T00:40:09.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:08 smithi121 ceph-mon[32231]: osdmap e95: 12 total, 12 up, 12 in 2024-04-25T00:40:09.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:08 smithi130 ceph-mon[32525]: osdmap e95: 12 total, 12 up, 12 in 2024-04-25T00:40:09.149 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:08 smithi032 ceph-mon[29357]: osdmap e95: 12 total, 12 up, 12 in 2024-04-25T00:40:10.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:09 smithi121 ceph-mon[32231]: pgmap v209: 129 pgs: 129 active+clean; 61 GiB data, 130 GiB used, 943 GiB / 1.0 TiB avail; 1.9 MiB/s rd, 668 MiB/s wr, 2.01k op/s 2024-04-25T00:40:10.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:09 smithi130 ceph-mon[32525]: pgmap v209: 129 pgs: 129 active+clean; 61 GiB data, 130 GiB used, 943 GiB / 1.0 TiB avail; 1.9 MiB/s rd, 668 MiB/s wr, 2.01k op/s 2024-04-25T00:40:10.165 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:09 smithi032 ceph-mon[29357]: pgmap v209: 129 pgs: 129 active+clean; 61 GiB data, 130 GiB used, 943 GiB / 1.0 TiB avail; 1.9 MiB/s rd, 668 MiB/s wr, 2.01k op/s 2024-04-25T00:40:12.061 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:11 smithi032 ceph-mon[29357]: pgmap v210: 129 pgs: 129 active+clean; 61 GiB data, 131 GiB used, 942 GiB / 1.0 TiB avail; 992 KiB/s rd, 459 MiB/s wr, 1.89k op/s 2024-04-25T00:40:12.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:11 smithi121 ceph-mon[32231]: pgmap v210: 129 pgs: 129 active+clean; 61 GiB data, 131 GiB used, 942 GiB / 1.0 TiB avail; 992 KiB/s rd, 459 MiB/s wr, 1.89k op/s 2024-04-25T00:40:12.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:11 smithi130 ceph-mon[32525]: pgmap v210: 129 pgs: 129 active+clean; 61 GiB data, 131 GiB used, 942 GiB / 1.0 TiB avail; 992 KiB/s rd, 459 MiB/s wr, 1.89k op/s 2024-04-25T00:40:14.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:13 smithi121 ceph-mon[32231]: pgmap v211: 129 pgs: 129 active+clean; 61 GiB data, 132 GiB used, 941 GiB / 1.0 TiB avail; 991 KiB/s rd, 485 MiB/s wr, 2.28k op/s 2024-04-25T00:40:14.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:13 smithi130 ceph-mon[32525]: pgmap v211: 129 pgs: 129 active+clean; 61 GiB data, 132 GiB used, 941 GiB / 1.0 TiB avail; 991 KiB/s rd, 485 MiB/s wr, 2.28k op/s 2024-04-25T00:40:14.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:13 smithi032 ceph-mon[29357]: pgmap v211: 129 pgs: 129 active+clean; 61 GiB data, 132 GiB used, 941 GiB / 1.0 TiB avail; 991 KiB/s rd, 485 MiB/s wr, 2.28k op/s 2024-04-25T00:40:16.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:15 smithi121 ceph-mon[32231]: pgmap v212: 129 pgs: 129 active+clean; 61 GiB data, 132 GiB used, 941 GiB / 1.0 TiB avail; 645 KiB/s rd, 463 MiB/s wr, 2.22k op/s 2024-04-25T00:40:16.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:15 smithi130 ceph-mon[32525]: pgmap v212: 129 pgs: 129 active+clean; 61 GiB data, 132 GiB used, 941 GiB / 1.0 TiB avail; 645 KiB/s rd, 463 MiB/s wr, 2.22k op/s 2024-04-25T00:40:16.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:15 smithi032 ceph-mon[29357]: pgmap v212: 129 pgs: 129 active+clean; 61 GiB data, 132 GiB used, 941 GiB / 1.0 TiB avail; 645 KiB/s rd, 463 MiB/s wr, 2.22k op/s 2024-04-25T00:40:18.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:17 smithi032 ceph-mon[29357]: pgmap v213: 129 pgs: 129 active+clean; 61 GiB data, 132 GiB used, 941 GiB / 1.0 TiB avail; 281 KiB/s rd, 264 MiB/s wr, 1.98k op/s 2024-04-25T00:40:18.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:17 smithi032 ceph-mon[29357]: osdmap e96: 12 total, 12 up, 12 in 2024-04-25T00:40:18.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:17 smithi121 ceph-mon[32231]: pgmap v213: 129 pgs: 129 active+clean; 61 GiB data, 132 GiB used, 941 GiB / 1.0 TiB avail; 281 KiB/s rd, 264 MiB/s wr, 1.98k op/s 2024-04-25T00:40:18.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:17 smithi121 ceph-mon[32231]: osdmap e96: 12 total, 12 up, 12 in 2024-04-25T00:40:18.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:17 smithi130 ceph-mon[32525]: pgmap v213: 129 pgs: 129 active+clean; 61 GiB data, 132 GiB used, 941 GiB / 1.0 TiB avail; 281 KiB/s rd, 264 MiB/s wr, 1.98k op/s 2024-04-25T00:40:18.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:17 smithi130 ceph-mon[32525]: osdmap e96: 12 total, 12 up, 12 in 2024-04-25T00:40:20.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:19 smithi032 ceph-mon[29357]: pgmap v215: 129 pgs: 129 active+clean; 61 GiB data, 132 GiB used, 941 GiB / 1.0 TiB avail; 223 KiB/s rd, 235 MiB/s wr, 1.92k op/s 2024-04-25T00:40:20.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:19 smithi121 ceph-mon[32231]: pgmap v215: 129 pgs: 129 active+clean; 61 GiB data, 132 GiB used, 941 GiB / 1.0 TiB avail; 223 KiB/s rd, 235 MiB/s wr, 1.92k op/s 2024-04-25T00:40:20.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:19 smithi130 ceph-mon[32525]: pgmap v215: 129 pgs: 129 active+clean; 61 GiB data, 132 GiB used, 941 GiB / 1.0 TiB avail; 223 KiB/s rd, 235 MiB/s wr, 1.92k op/s 2024-04-25T00:40:22.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:21 smithi032 ceph-mon[29357]: pgmap v216: 129 pgs: 129 active+clean; 61 GiB data, 132 GiB used, 941 GiB / 1.0 TiB avail; 140 KiB/s rd, 197 MiB/s wr, 1.68k op/s 2024-04-25T00:40:22.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:21 smithi121 ceph-mon[32231]: pgmap v216: 129 pgs: 129 active+clean; 61 GiB data, 132 GiB used, 941 GiB / 1.0 TiB avail; 140 KiB/s rd, 197 MiB/s wr, 1.68k op/s 2024-04-25T00:40:22.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:21 smithi130 ceph-mon[32525]: pgmap v216: 129 pgs: 129 active+clean; 61 GiB data, 132 GiB used, 941 GiB / 1.0 TiB avail; 140 KiB/s rd, 197 MiB/s wr, 1.68k op/s 2024-04-25T00:40:24.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:23 smithi032 ceph-mon[29357]: pgmap v217: 129 pgs: 129 active+clean; 61 GiB data, 132 GiB used, 941 GiB / 1.0 TiB avail; 140 KiB/s rd, 167 MiB/s wr, 1.53k op/s 2024-04-25T00:40:24.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:23 smithi121 ceph-mon[32231]: pgmap v217: 129 pgs: 129 active+clean; 61 GiB data, 132 GiB used, 941 GiB / 1.0 TiB avail; 140 KiB/s rd, 167 MiB/s wr, 1.53k op/s 2024-04-25T00:40:24.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:23 smithi130 ceph-mon[32525]: pgmap v217: 129 pgs: 129 active+clean; 61 GiB data, 132 GiB used, 941 GiB / 1.0 TiB avail; 140 KiB/s rd, 167 MiB/s wr, 1.53k op/s 2024-04-25T00:40:26.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:25 smithi032 ceph-mon[29357]: pgmap v218: 129 pgs: 129 active+clean; 61 GiB data, 132 GiB used, 941 GiB / 1.0 TiB avail; 16 KiB/s rd, 165 MiB/s wr, 1.52k op/s 2024-04-25T00:40:26.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:25 smithi121 ceph-mon[32231]: pgmap v218: 129 pgs: 129 active+clean; 61 GiB data, 132 GiB used, 941 GiB / 1.0 TiB avail; 16 KiB/s rd, 165 MiB/s wr, 1.52k op/s 2024-04-25T00:40:26.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:25 smithi130 ceph-mon[32525]: pgmap v218: 129 pgs: 129 active+clean; 61 GiB data, 132 GiB used, 941 GiB / 1.0 TiB avail; 16 KiB/s rd, 165 MiB/s wr, 1.52k op/s 2024-04-25T00:40:28.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:27 smithi032 ceph-mon[29357]: pgmap v219: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 16 KiB/s rd, 107 MiB/s wr, 1.04k op/s 2024-04-25T00:40:28.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:27 smithi121 ceph-mon[32231]: pgmap v219: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 16 KiB/s rd, 107 MiB/s wr, 1.04k op/s 2024-04-25T00:40:28.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:27 smithi130 ceph-mon[32525]: pgmap v219: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 16 KiB/s rd, 107 MiB/s wr, 1.04k op/s 2024-04-25T00:40:30.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:29 smithi032 ceph-mon[29357]: pgmap v220: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 21 KiB/s rd, 88 MiB/s wr, 851 op/s 2024-04-25T00:40:30.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:29 smithi121 ceph-mon[32231]: pgmap v220: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 21 KiB/s rd, 88 MiB/s wr, 851 op/s 2024-04-25T00:40:30.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:29 smithi130 ceph-mon[32525]: pgmap v220: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 21 KiB/s rd, 88 MiB/s wr, 851 op/s 2024-04-25T00:40:32.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:31 smithi032 ceph-mon[29357]: pgmap v221: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 11 KiB/s rd, 77 MiB/s wr, 740 op/s 2024-04-25T00:40:32.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:31 smithi121 ceph-mon[32231]: pgmap v221: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 11 KiB/s rd, 77 MiB/s wr, 740 op/s 2024-04-25T00:40:32.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:31 smithi130 ceph-mon[32525]: pgmap v221: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 11 KiB/s rd, 77 MiB/s wr, 740 op/s 2024-04-25T00:40:34.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:33 smithi032 ceph-mon[29357]: pgmap v222: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 9.4 KiB/s rd, 51 MiB/s wr, 462 op/s 2024-04-25T00:40:34.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:33 smithi121 ceph-mon[32231]: pgmap v222: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 9.4 KiB/s rd, 51 MiB/s wr, 462 op/s 2024-04-25T00:40:34.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:33 smithi130 ceph-mon[32525]: pgmap v222: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 9.4 KiB/s rd, 51 MiB/s wr, 462 op/s 2024-04-25T00:40:35.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:35 smithi121 ceph-mon[32231]: pgmap v223: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 9.6 KiB/s rd, 25 MiB/s wr, 188 op/s 2024-04-25T00:40:35.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:35 smithi130 ceph-mon[32525]: pgmap v223: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 9.6 KiB/s rd, 25 MiB/s wr, 188 op/s 2024-04-25T00:40:35.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:35 smithi032 ceph-mon[29357]: pgmap v223: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 9.6 KiB/s rd, 25 MiB/s wr, 188 op/s 2024-04-25T00:40:37.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:37 smithi121 ceph-mon[32231]: pgmap v224: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 11 KiB/s rd, 25 MiB/s wr, 189 op/s 2024-04-25T00:40:37.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:37 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:40:37.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:37 smithi130 ceph-mon[32525]: pgmap v224: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 11 KiB/s rd, 25 MiB/s wr, 189 op/s 2024-04-25T00:40:37.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:37 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:40:37.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:37 smithi032 ceph-mon[29357]: pgmap v224: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 11 KiB/s rd, 25 MiB/s wr, 189 op/s 2024-04-25T00:40:37.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:37 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:40:38.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:38 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:40:38.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:38 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:40:38.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:38 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:40:38.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:38 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:40:38.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:38 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:40:38.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:38 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:40:38.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:38 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:40:38.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:38 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:40:38.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:38 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:40:40.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:39 smithi121 ceph-mon[32231]: pgmap v225: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 15 KiB/s rd, 2.9 MiB/s wr, 28 op/s 2024-04-25T00:40:40.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:39 smithi130 ceph-mon[32525]: pgmap v225: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 15 KiB/s rd, 2.9 MiB/s wr, 28 op/s 2024-04-25T00:40:40.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:39 smithi032 ceph-mon[29357]: pgmap v225: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 15 KiB/s rd, 2.9 MiB/s wr, 28 op/s 2024-04-25T00:40:42.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:41 smithi121 ceph-mon[32231]: pgmap v226: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 9.7 KiB/s rd, 6.2 KiB/s wr, 7 op/s 2024-04-25T00:40:42.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:41 smithi130 ceph-mon[32525]: pgmap v226: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 9.7 KiB/s rd, 6.2 KiB/s wr, 7 op/s 2024-04-25T00:40:42.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:41 smithi032 ceph-mon[29357]: pgmap v226: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 9.7 KiB/s rd, 6.2 KiB/s wr, 7 op/s 2024-04-25T00:40:44.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:43 smithi121 ceph-mon[32231]: pgmap v227: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 8.5 KiB/s rd, 6.0 KiB/s wr, 5 op/s 2024-04-25T00:40:44.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:43 smithi130 ceph-mon[32525]: pgmap v227: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 8.5 KiB/s rd, 6.0 KiB/s wr, 5 op/s 2024-04-25T00:40:44.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:43 smithi032 ceph-mon[29357]: pgmap v227: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 8.5 KiB/s rd, 6.0 KiB/s wr, 5 op/s 2024-04-25T00:40:46.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:45 smithi121 ceph-mon[32231]: pgmap v228: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 8.5 KiB/s rd, 8.8 KiB/s wr, 6 op/s 2024-04-25T00:40:46.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:45 smithi130 ceph-mon[32525]: pgmap v228: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 8.5 KiB/s rd, 8.8 KiB/s wr, 6 op/s 2024-04-25T00:40:46.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:45 smithi032 ceph-mon[29357]: pgmap v228: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 8.5 KiB/s rd, 8.8 KiB/s wr, 6 op/s 2024-04-25T00:40:48.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:47 smithi121 ceph-mon[32231]: pgmap v229: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 9.6 KiB/s rd, 6.7 KiB/s wr, 7 op/s 2024-04-25T00:40:48.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:47 smithi130 ceph-mon[32525]: pgmap v229: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 9.6 KiB/s rd, 6.7 KiB/s wr, 7 op/s 2024-04-25T00:40:48.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:47 smithi032 ceph-mon[29357]: pgmap v229: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 9.6 KiB/s rd, 6.7 KiB/s wr, 7 op/s 2024-04-25T00:40:50.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:49 smithi121 ceph-mon[32231]: pgmap v230: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 11 KiB/s rd, 6.7 KiB/s wr, 6 op/s 2024-04-25T00:40:50.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:49 smithi130 ceph-mon[32525]: pgmap v230: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 11 KiB/s rd, 6.7 KiB/s wr, 6 op/s 2024-04-25T00:40:50.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:49 smithi032 ceph-mon[29357]: pgmap v230: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 11 KiB/s rd, 6.7 KiB/s wr, 6 op/s 2024-04-25T00:40:52.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:51 smithi121 ceph-mon[32231]: pgmap v231: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 6.6 KiB/s rd, 2.9 KiB/s wr, 7 op/s 2024-04-25T00:40:52.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:51 smithi130 ceph-mon[32525]: pgmap v231: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 6.6 KiB/s rd, 2.9 KiB/s wr, 7 op/s 2024-04-25T00:40:52.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:51 smithi032 ceph-mon[29357]: pgmap v231: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 6.6 KiB/s rd, 2.9 KiB/s wr, 7 op/s 2024-04-25T00:40:54.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:53 smithi130 ceph-mon[32525]: pgmap v232: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 5.3 KiB/s rd, 2.9 KiB/s wr, 5 op/s 2024-04-25T00:40:54.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:53 smithi032 ceph-mon[29357]: pgmap v232: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 5.3 KiB/s rd, 2.9 KiB/s wr, 5 op/s 2024-04-25T00:40:54.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:53 smithi121 ceph-mon[32231]: pgmap v232: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 5.3 KiB/s rd, 2.9 KiB/s wr, 5 op/s 2024-04-25T00:40:56.140 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:55 smithi121 ceph-mon[32231]: pgmap v233: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 5.3 KiB/s rd, 3.7 KiB/s wr, 6 op/s 2024-04-25T00:40:56.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:55 smithi130 ceph-mon[32525]: pgmap v233: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 5.3 KiB/s rd, 3.7 KiB/s wr, 6 op/s 2024-04-25T00:40:56.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:55 smithi032 ceph-mon[29357]: pgmap v233: 129 pgs: 129 active+clean; 62 GiB data, 132 GiB used, 940 GiB / 1.0 TiB avail; 5.3 KiB/s rd, 3.7 KiB/s wr, 6 op/s 2024-04-25T00:40:58.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:57 smithi130 ceph-mon[32525]: pgmap v234: 129 pgs: 129 active+clean; 62 GiB data, 133 GiB used, 940 GiB / 1.0 TiB avail; 6.6 KiB/s rd, 14 MiB/s wr, 104 op/s 2024-04-25T00:40:58.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:57 smithi032 ceph-mon[29357]: pgmap v234: 129 pgs: 129 active+clean; 62 GiB data, 133 GiB used, 940 GiB / 1.0 TiB avail; 6.6 KiB/s rd, 14 MiB/s wr, 104 op/s 2024-04-25T00:40:58.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:57 smithi121 ceph-mon[32231]: pgmap v234: 129 pgs: 129 active+clean; 62 GiB data, 133 GiB used, 940 GiB / 1.0 TiB avail; 6.6 KiB/s rd, 14 MiB/s wr, 104 op/s 2024-04-25T00:41:00.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:40:59 smithi130 ceph-mon[32525]: pgmap v235: 129 pgs: 129 active+clean; 62 GiB data, 133 GiB used, 940 GiB / 1.0 TiB avail; 6.2 KiB/s rd, 15 MiB/s wr, 115 op/s 2024-04-25T00:41:00.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:40:59 smithi032 ceph-mon[29357]: pgmap v235: 129 pgs: 129 active+clean; 62 GiB data, 133 GiB used, 940 GiB / 1.0 TiB avail; 6.2 KiB/s rd, 15 MiB/s wr, 115 op/s 2024-04-25T00:41:00.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:40:59 smithi121 ceph-mon[32231]: pgmap v235: 129 pgs: 129 active+clean; 62 GiB data, 133 GiB used, 940 GiB / 1.0 TiB avail; 6.2 KiB/s rd, 15 MiB/s wr, 115 op/s 2024-04-25T00:41:02.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:01 smithi130 ceph-mon[32525]: pgmap v236: 129 pgs: 129 active+clean; 62 GiB data, 133 GiB used, 940 GiB / 1.0 TiB avail; 4.6 KiB/s rd, 15 MiB/s wr, 115 op/s 2024-04-25T00:41:02.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:01 smithi032 ceph-mon[29357]: pgmap v236: 129 pgs: 129 active+clean; 62 GiB data, 133 GiB used, 940 GiB / 1.0 TiB avail; 4.6 KiB/s rd, 15 MiB/s wr, 115 op/s 2024-04-25T00:41:02.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:01 smithi121 ceph-mon[32231]: pgmap v236: 129 pgs: 129 active+clean; 62 GiB data, 133 GiB used, 940 GiB / 1.0 TiB avail; 4.6 KiB/s rd, 15 MiB/s wr, 115 op/s 2024-04-25T00:41:04.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:03 smithi130 ceph-mon[32525]: pgmap v237: 129 pgs: 129 active+clean; 62 GiB data, 133 GiB used, 940 GiB / 1.0 TiB avail; 12 KiB/s rd, 15 MiB/s wr, 128 op/s 2024-04-25T00:41:04.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:03 smithi032 ceph-mon[29357]: pgmap v237: 129 pgs: 129 active+clean; 62 GiB data, 133 GiB used, 940 GiB / 1.0 TiB avail; 12 KiB/s rd, 15 MiB/s wr, 128 op/s 2024-04-25T00:41:04.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:03 smithi121 ceph-mon[32231]: pgmap v237: 129 pgs: 129 active+clean; 62 GiB data, 133 GiB used, 940 GiB / 1.0 TiB avail; 12 KiB/s rd, 15 MiB/s wr, 128 op/s 2024-04-25T00:41:06.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:05 smithi130 ceph-mon[32525]: pgmap v238: 129 pgs: 129 active+clean; 62 GiB data, 133 GiB used, 940 GiB / 1.0 TiB avail; 16 KiB/s rd, 15 MiB/s wr, 129 op/s 2024-04-25T00:41:06.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:05 smithi032 ceph-mon[29357]: pgmap v238: 129 pgs: 129 active+clean; 62 GiB data, 133 GiB used, 940 GiB / 1.0 TiB avail; 16 KiB/s rd, 15 MiB/s wr, 129 op/s 2024-04-25T00:41:06.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:05 smithi121 ceph-mon[32231]: pgmap v238: 129 pgs: 129 active+clean; 62 GiB data, 133 GiB used, 940 GiB / 1.0 TiB avail; 16 KiB/s rd, 15 MiB/s wr, 129 op/s 2024-04-25T00:41:08.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:07 smithi130 ceph-mon[32525]: pgmap v239: 129 pgs: 129 active+clean; 62 GiB data, 133 GiB used, 940 GiB / 1.0 TiB avail; 19 KiB/s rd, 16 MiB/s wr, 133 op/s 2024-04-25T00:41:08.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:07 smithi032 ceph-mon[29357]: pgmap v239: 129 pgs: 129 active+clean; 62 GiB data, 133 GiB used, 940 GiB / 1.0 TiB avail; 19 KiB/s rd, 16 MiB/s wr, 133 op/s 2024-04-25T00:41:08.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:07 smithi121 ceph-mon[32231]: pgmap v239: 129 pgs: 129 active+clean; 62 GiB data, 133 GiB used, 940 GiB / 1.0 TiB avail; 19 KiB/s rd, 16 MiB/s wr, 133 op/s 2024-04-25T00:41:09.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:08 smithi130 ceph-mon[32525]: osdmap e97: 12 total, 12 up, 12 in 2024-04-25T00:41:09.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:08 smithi032 ceph-mon[29357]: osdmap e97: 12 total, 12 up, 12 in 2024-04-25T00:41:09.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:08 smithi121 ceph-mon[32231]: osdmap e97: 12 total, 12 up, 12 in 2024-04-25T00:41:10.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:09 smithi130 ceph-mon[32525]: pgmap v241: 129 pgs: 2 active+clean+snaptrim, 119 active+clean, 8 active+clean+snaptrim_wait; 62 GiB data, 133 GiB used, 940 GiB / 1.0 TiB avail; 20 KiB/s rd, 54 KiB/s wr, 27 op/s 2024-04-25T00:41:10.148 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:09 smithi032 ceph-mon[29357]: pgmap v241: 129 pgs: 2 active+clean+snaptrim, 119 active+clean, 8 active+clean+snaptrim_wait; 62 GiB data, 133 GiB used, 940 GiB / 1.0 TiB avail; 20 KiB/s rd, 54 KiB/s wr, 27 op/s 2024-04-25T00:41:10.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:09 smithi121 ceph-mon[32231]: pgmap v241: 129 pgs: 2 active+clean+snaptrim, 119 active+clean, 8 active+clean+snaptrim_wait; 62 GiB data, 133 GiB used, 940 GiB / 1.0 TiB avail; 20 KiB/s rd, 54 KiB/s wr, 27 op/s 2024-04-25T00:41:12.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:11 smithi130 ceph-mon[32525]: pgmap v242: 129 pgs: 8 active+clean+snaptrim, 102 active+clean, 19 active+clean+snaptrim_wait; 58 GiB data, 127 GiB used, 946 GiB / 1.0 TiB avail; 20 KiB/s rd, 54 KiB/s wr, 27 op/s 2024-04-25T00:41:12.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:11 smithi032 ceph-mon[29357]: pgmap v242: 129 pgs: 8 active+clean+snaptrim, 102 active+clean, 19 active+clean+snaptrim_wait; 58 GiB data, 127 GiB used, 946 GiB / 1.0 TiB avail; 20 KiB/s rd, 54 KiB/s wr, 27 op/s 2024-04-25T00:41:12.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:11 smithi121 ceph-mon[32231]: pgmap v242: 129 pgs: 8 active+clean+snaptrim, 102 active+clean, 19 active+clean+snaptrim_wait; 58 GiB data, 127 GiB used, 946 GiB / 1.0 TiB avail; 20 KiB/s rd, 54 KiB/s wr, 27 op/s 2024-04-25T00:41:14.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:13 smithi032 ceph-mon[29357]: pgmap v243: 129 pgs: 11 active+clean+snaptrim, 92 active+clean, 26 active+clean+snaptrim_wait; 55 GiB data, 122 GiB used, 951 GiB / 1.0 TiB avail; 9.0 KiB/s rd, 28 KiB/s wr, 9 op/s 2024-04-25T00:41:14.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:13 smithi121 ceph-mon[32231]: pgmap v243: 129 pgs: 11 active+clean+snaptrim, 92 active+clean, 26 active+clean+snaptrim_wait; 55 GiB data, 122 GiB used, 951 GiB / 1.0 TiB avail; 9.0 KiB/s rd, 28 KiB/s wr, 9 op/s 2024-04-25T00:41:14.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:13 smithi130 ceph-mon[32525]: pgmap v243: 129 pgs: 11 active+clean+snaptrim, 92 active+clean, 26 active+clean+snaptrim_wait; 55 GiB data, 122 GiB used, 951 GiB / 1.0 TiB avail; 9.0 KiB/s rd, 28 KiB/s wr, 9 op/s 2024-04-25T00:41:16.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:15 smithi032 ceph-mon[29357]: pgmap v244: 129 pgs: 9 active+clean+snaptrim, 102 active+clean, 18 active+clean+snaptrim_wait; 54 GiB data, 120 GiB used, 953 GiB / 1.0 TiB avail; 4.4 KiB/s rd, 27 KiB/s wr, 9 op/s 2024-04-25T00:41:16.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:15 smithi121 ceph-mon[32231]: pgmap v244: 129 pgs: 9 active+clean+snaptrim, 102 active+clean, 18 active+clean+snaptrim_wait; 54 GiB data, 120 GiB used, 953 GiB / 1.0 TiB avail; 4.4 KiB/s rd, 27 KiB/s wr, 9 op/s 2024-04-25T00:41:16.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:15 smithi130 ceph-mon[32525]: pgmap v244: 129 pgs: 9 active+clean+snaptrim, 102 active+clean, 18 active+clean+snaptrim_wait; 54 GiB data, 120 GiB used, 953 GiB / 1.0 TiB avail; 4.4 KiB/s rd, 27 KiB/s wr, 9 op/s 2024-04-25T00:41:18.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:17 smithi032 ceph-mon[29357]: pgmap v245: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.3 KiB/s rd, 204 B/s wr, 6 op/s 2024-04-25T00:41:18.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:17 smithi121 ceph-mon[32231]: pgmap v245: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.3 KiB/s rd, 204 B/s wr, 6 op/s 2024-04-25T00:41:18.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:17 smithi130 ceph-mon[32525]: pgmap v245: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.3 KiB/s rd, 204 B/s wr, 6 op/s 2024-04-25T00:41:20.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:19 smithi032 ceph-mon[29357]: pgmap v246: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.2 KiB/s rd, 197 B/s wr, 7 op/s 2024-04-25T00:41:20.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:19 smithi121 ceph-mon[32231]: pgmap v246: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.2 KiB/s rd, 197 B/s wr, 7 op/s 2024-04-25T00:41:20.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:19 smithi130 ceph-mon[32525]: pgmap v246: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.2 KiB/s rd, 197 B/s wr, 7 op/s 2024-04-25T00:41:22.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:21 smithi032 ceph-mon[29357]: pgmap v247: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.9 KiB/s rd, 85 B/s wr, 6 op/s 2024-04-25T00:41:22.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:21 smithi032 ceph-mon[29357]: osdmap e98: 12 total, 12 up, 12 in 2024-04-25T00:41:22.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:21 smithi121 ceph-mon[32231]: pgmap v247: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.9 KiB/s rd, 85 B/s wr, 6 op/s 2024-04-25T00:41:22.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:21 smithi121 ceph-mon[32231]: osdmap e98: 12 total, 12 up, 12 in 2024-04-25T00:41:22.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:21 smithi130 ceph-mon[32525]: pgmap v247: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.9 KiB/s rd, 85 B/s wr, 6 op/s 2024-04-25T00:41:22.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:21 smithi130 ceph-mon[32525]: osdmap e98: 12 total, 12 up, 12 in 2024-04-25T00:41:24.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:23 smithi032 ceph-mon[29357]: pgmap v249: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.2 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:41:24.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:23 smithi121 ceph-mon[32231]: pgmap v249: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.2 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:41:24.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:23 smithi130 ceph-mon[32525]: pgmap v249: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.2 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:41:26.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:25 smithi032 ceph-mon[29357]: pgmap v250: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-04-25T00:41:26.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:25 smithi121 ceph-mon[32231]: pgmap v250: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-04-25T00:41:26.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:25 smithi130 ceph-mon[32525]: pgmap v250: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-04-25T00:41:27.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:26 smithi032 ceph-mon[29357]: pgmap v251: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-04-25T00:41:27.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:26 smithi121 ceph-mon[32231]: pgmap v251: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-04-25T00:41:27.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:26 smithi130 ceph-mon[32525]: pgmap v251: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-04-25T00:41:29.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:29 smithi121 ceph-mon[32231]: pgmap v252: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-04-25T00:41:29.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:29 smithi130 ceph-mon[32525]: pgmap v252: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-04-25T00:41:29.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:29 smithi032 ceph-mon[29357]: pgmap v252: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-04-25T00:41:31.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:31 smithi032 ceph-mon[29357]: pgmap v253: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-04-25T00:41:31.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:31 smithi121 ceph-mon[32231]: pgmap v253: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-04-25T00:41:31.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:31 smithi130 ceph-mon[32525]: pgmap v253: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-04-25T00:41:33.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:33 smithi121 ceph-mon[32231]: pgmap v254: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-25T00:41:33.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:33 smithi130 ceph-mon[32525]: pgmap v254: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-25T00:41:33.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:33 smithi032 ceph-mon[29357]: pgmap v254: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-25T00:41:35.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:35 smithi121 ceph-mon[32231]: pgmap v255: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:41:35.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:35 smithi130 ceph-mon[32525]: pgmap v255: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:41:35.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:35 smithi032 ceph-mon[29357]: pgmap v255: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:41:37.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:37 smithi121 ceph-mon[32231]: pgmap v256: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:41:37.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:37 smithi130 ceph-mon[32525]: pgmap v256: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:41:37.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:37 smithi032 ceph-mon[29357]: pgmap v256: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:41:38.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:38 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:41:38.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:38 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:41:38.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:38 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:41:39.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:39 smithi121 ceph-mon[32231]: pgmap v257: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:41:39.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:39 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:41:39.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:39 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:41:39.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:39 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:41:39.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:39 smithi130 ceph-mon[32525]: pgmap v257: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:41:39.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:39 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:41:39.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:39 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:41:39.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:39 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:41:39.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:39 smithi032 ceph-mon[29357]: pgmap v257: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:41:39.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:39 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:41:39.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:39 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:41:39.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:39 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:41:41.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:41 smithi121 ceph-mon[32231]: pgmap v258: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:41:41.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:41 smithi130 ceph-mon[32525]: pgmap v258: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:41:41.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:41 smithi032 ceph-mon[29357]: pgmap v258: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:41:43.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:43 smithi121 ceph-mon[32231]: pgmap v259: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:41:43.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:43 smithi130 ceph-mon[32525]: pgmap v259: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:41:43.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:43 smithi032 ceph-mon[29357]: pgmap v259: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:41:45.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:45 smithi121 ceph-mon[32231]: pgmap v260: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:41:45.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:45 smithi130 ceph-mon[32525]: pgmap v260: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:41:45.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:45 smithi032 ceph-mon[29357]: pgmap v260: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:41:47.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:47 smithi121 ceph-mon[32231]: pgmap v261: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:41:47.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:47 smithi130 ceph-mon[32525]: pgmap v261: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:41:47.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:47 smithi032 ceph-mon[29357]: pgmap v261: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:41:49.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:49 smithi121 ceph-mon[32231]: pgmap v262: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:41:49.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:49 smithi130 ceph-mon[32525]: pgmap v262: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:41:49.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:49 smithi032 ceph-mon[29357]: pgmap v262: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:41:51.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:51 smithi121 ceph-mon[32231]: pgmap v263: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:41:51.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:51 smithi130 ceph-mon[32525]: pgmap v263: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:41:51.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:51 smithi032 ceph-mon[29357]: pgmap v263: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:41:53.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:53 smithi121 ceph-mon[32231]: pgmap v264: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:41:53.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:53 smithi130 ceph-mon[32525]: pgmap v264: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:41:53.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:53 smithi032 ceph-mon[29357]: pgmap v264: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:41:55.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:55 smithi121 ceph-mon[32231]: pgmap v265: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:41:55.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:55 smithi130 ceph-mon[32525]: pgmap v265: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:41:55.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:55 smithi032 ceph-mon[29357]: pgmap v265: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:41:57.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:57 smithi121 ceph-mon[32231]: pgmap v266: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:41:57.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:57 smithi130 ceph-mon[32525]: pgmap v266: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:41:57.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:57 smithi032 ceph-mon[29357]: pgmap v266: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:41:59.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:41:59 smithi130 ceph-mon[32525]: pgmap v267: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:41:59.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:41:59 smithi032 ceph-mon[29357]: pgmap v267: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:41:59.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:41:59 smithi121 ceph-mon[32231]: pgmap v267: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:42:01.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:01 smithi130 ceph-mon[32525]: pgmap v268: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:42:01.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:01 smithi032 ceph-mon[29357]: pgmap v268: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:42:01.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:01 smithi121 ceph-mon[32231]: pgmap v268: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:42:03.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:03 smithi130 ceph-mon[32525]: pgmap v269: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 11 KiB/s rd, 21 KiB/s wr, 19 op/s 2024-04-25T00:42:03.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:03 smithi032 ceph-mon[29357]: pgmap v269: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 11 KiB/s rd, 21 KiB/s wr, 19 op/s 2024-04-25T00:42:03.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:03 smithi121 ceph-mon[32231]: pgmap v269: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 11 KiB/s rd, 21 KiB/s wr, 19 op/s 2024-04-25T00:42:05.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:05 smithi130 ceph-mon[32525]: pgmap v270: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 12 KiB/s rd, 22 KiB/s wr, 20 op/s 2024-04-25T00:42:05.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:05 smithi032 ceph-mon[29357]: pgmap v270: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 12 KiB/s rd, 22 KiB/s wr, 20 op/s 2024-04-25T00:42:05.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:05 smithi121 ceph-mon[32231]: pgmap v270: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 12 KiB/s rd, 22 KiB/s wr, 20 op/s 2024-04-25T00:42:07.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:07 smithi130 ceph-mon[32525]: pgmap v271: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 14 KiB/s rd, 45 KiB/s wr, 24 op/s 2024-04-25T00:42:07.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:07 smithi032 ceph-mon[29357]: pgmap v271: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 14 KiB/s rd, 45 KiB/s wr, 24 op/s 2024-04-25T00:42:07.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:07 smithi121 ceph-mon[32231]: pgmap v271: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 14 KiB/s rd, 45 KiB/s wr, 24 op/s 2024-04-25T00:42:09.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:09 smithi130 ceph-mon[32525]: pgmap v272: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 13 KiB/s rd, 45 KiB/s wr, 23 op/s 2024-04-25T00:42:09.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:09 smithi130 ceph-mon[32525]: osdmap e99: 12 total, 12 up, 12 in 2024-04-25T00:42:09.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:09 smithi032 ceph-mon[29357]: pgmap v272: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 13 KiB/s rd, 45 KiB/s wr, 23 op/s 2024-04-25T00:42:09.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:09 smithi032 ceph-mon[29357]: osdmap e99: 12 total, 12 up, 12 in 2024-04-25T00:42:09.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:09 smithi121 ceph-mon[32231]: pgmap v272: 129 pgs: 129 active+clean; 52 GiB data, 113 GiB used, 960 GiB / 1.0 TiB avail; 13 KiB/s rd, 45 KiB/s wr, 23 op/s 2024-04-25T00:42:09.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:09 smithi121 ceph-mon[32231]: osdmap e99: 12 total, 12 up, 12 in 2024-04-25T00:42:11.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:11 smithi130 ceph-mon[32525]: pgmap v274: 129 pgs: 10 active+clean+snaptrim, 21 active+clean+snaptrim_wait, 98 active+clean; 49 GiB data, 109 GiB used, 963 GiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 27 op/s 2024-04-25T00:42:11.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:11 smithi032 ceph-mon[29357]: pgmap v274: 129 pgs: 10 active+clean+snaptrim, 21 active+clean+snaptrim_wait, 98 active+clean; 49 GiB data, 109 GiB used, 963 GiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 27 op/s 2024-04-25T00:42:11.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:11 smithi121 ceph-mon[32231]: pgmap v274: 129 pgs: 10 active+clean+snaptrim, 21 active+clean+snaptrim_wait, 98 active+clean; 49 GiB data, 109 GiB used, 963 GiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 27 op/s 2024-04-25T00:42:13.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:13 smithi130 ceph-mon[32525]: pgmap v275: 129 pgs: 14 active+clean+snaptrim, 34 active+clean+snaptrim_wait, 81 active+clean; 46 GiB data, 106 GiB used, 967 GiB / 1.0 TiB avail; 5.2 KiB/s rd, 28 KiB/s wr, 9 op/s 2024-04-25T00:42:13.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:13 smithi032 ceph-mon[29357]: pgmap v275: 129 pgs: 14 active+clean+snaptrim, 34 active+clean+snaptrim_wait, 81 active+clean; 46 GiB data, 106 GiB used, 967 GiB / 1.0 TiB avail; 5.2 KiB/s rd, 28 KiB/s wr, 9 op/s 2024-04-25T00:42:13.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:13 smithi121 ceph-mon[32231]: pgmap v275: 129 pgs: 14 active+clean+snaptrim, 34 active+clean+snaptrim_wait, 81 active+clean; 46 GiB data, 106 GiB used, 967 GiB / 1.0 TiB avail; 5.2 KiB/s rd, 28 KiB/s wr, 9 op/s 2024-04-25T00:42:15.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:15 smithi130 ceph-mon[32525]: pgmap v276: 129 pgs: 14 active+clean+snaptrim, 34 active+clean+snaptrim_wait, 81 active+clean; 46 GiB data, 104 GiB used, 969 GiB / 1.0 TiB avail; 4.4 KiB/s rd, 27 KiB/s wr, 9 op/s 2024-04-25T00:42:15.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:15 smithi032 ceph-mon[29357]: pgmap v276: 129 pgs: 14 active+clean+snaptrim, 34 active+clean+snaptrim_wait, 81 active+clean; 46 GiB data, 104 GiB used, 969 GiB / 1.0 TiB avail; 4.4 KiB/s rd, 27 KiB/s wr, 9 op/s 2024-04-25T00:42:15.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:15 smithi121 ceph-mon[32231]: pgmap v276: 129 pgs: 14 active+clean+snaptrim, 34 active+clean+snaptrim_wait, 81 active+clean; 46 GiB data, 104 GiB used, 969 GiB / 1.0 TiB avail; 4.4 KiB/s rd, 27 KiB/s wr, 9 op/s 2024-04-25T00:42:18.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:17 smithi032 ceph-mon[29357]: pgmap v277: 129 pgs: 129 active+clean; 42 GiB data, 94 GiB used, 979 GiB / 1.0 TiB avail; 3.2 KiB/s rd, 204 B/s wr, 6 op/s 2024-04-25T00:42:18.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:17 smithi032 ceph-mon[29357]: osdmap e100: 12 total, 12 up, 12 in 2024-04-25T00:42:18.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:17 smithi121 ceph-mon[32231]: pgmap v277: 129 pgs: 129 active+clean; 42 GiB data, 94 GiB used, 979 GiB / 1.0 TiB avail; 3.2 KiB/s rd, 204 B/s wr, 6 op/s 2024-04-25T00:42:18.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:17 smithi121 ceph-mon[32231]: osdmap e100: 12 total, 12 up, 12 in 2024-04-25T00:42:18.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:17 smithi130 ceph-mon[32525]: pgmap v277: 129 pgs: 129 active+clean; 42 GiB data, 94 GiB used, 979 GiB / 1.0 TiB avail; 3.2 KiB/s rd, 204 B/s wr, 6 op/s 2024-04-25T00:42:18.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:17 smithi130 ceph-mon[32525]: osdmap e100: 12 total, 12 up, 12 in 2024-04-25T00:42:20.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:19 smithi032 ceph-mon[29357]: pgmap v279: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.1 KiB/s rd, 103 B/s wr, 6 op/s 2024-04-25T00:42:20.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:19 smithi121 ceph-mon[32231]: pgmap v279: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.1 KiB/s rd, 103 B/s wr, 6 op/s 2024-04-25T00:42:20.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:19 smithi130 ceph-mon[32525]: pgmap v279: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.1 KiB/s rd, 103 B/s wr, 6 op/s 2024-04-25T00:42:22.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:21 smithi032 ceph-mon[29357]: pgmap v280: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:42:22.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:21 smithi121 ceph-mon[32231]: pgmap v280: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:42:22.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:21 smithi130 ceph-mon[32525]: pgmap v280: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:42:24.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:23 smithi032 ceph-mon[29357]: pgmap v281: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:42:24.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:23 smithi121 ceph-mon[32231]: pgmap v281: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:42:24.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:23 smithi130 ceph-mon[32525]: pgmap v281: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:42:26.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:25 smithi032 ceph-mon[29357]: pgmap v282: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-25T00:42:26.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:25 smithi121 ceph-mon[32231]: pgmap v282: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-25T00:42:26.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:25 smithi130 ceph-mon[32525]: pgmap v282: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-25T00:42:28.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:27 smithi032 ceph-mon[29357]: pgmap v283: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-25T00:42:28.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:27 smithi121 ceph-mon[32231]: pgmap v283: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-25T00:42:28.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:27 smithi130 ceph-mon[32525]: pgmap v283: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-25T00:42:29.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:28 smithi032 ceph-mon[29357]: pgmap v284: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.6 KiB/s rd, 6 op/s 2024-04-25T00:42:29.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:28 smithi130 ceph-mon[32525]: pgmap v284: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.6 KiB/s rd, 6 op/s 2024-04-25T00:42:29.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:28 smithi121 ceph-mon[32231]: pgmap v284: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.6 KiB/s rd, 6 op/s 2024-04-25T00:42:31.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:31 smithi121 ceph-mon[32231]: pgmap v285: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:42:31.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:31 smithi130 ceph-mon[32525]: pgmap v285: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:42:31.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:31 smithi032 ceph-mon[29357]: pgmap v285: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:42:33.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:33 smithi121 ceph-mon[32231]: pgmap v286: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:42:33.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:33 smithi130 ceph-mon[32525]: pgmap v286: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:42:33.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:33 smithi032 ceph-mon[29357]: pgmap v286: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:42:35.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:35 smithi121 ceph-mon[32231]: pgmap v287: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:42:35.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:35 smithi130 ceph-mon[32525]: pgmap v287: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:42:35.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:35 smithi032 ceph-mon[29357]: pgmap v287: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:42:37.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:37 smithi121 ceph-mon[32231]: pgmap v288: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:42:37.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:37 smithi130 ceph-mon[32525]: pgmap v288: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:42:37.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:37 smithi032 ceph-mon[29357]: pgmap v288: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:42:39.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:39 smithi121 ceph-mon[32231]: pgmap v289: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.4 KiB/s rd, 5 op/s 2024-04-25T00:42:39.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:39 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:42:39.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:39 smithi130 ceph-mon[32525]: pgmap v289: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.4 KiB/s rd, 5 op/s 2024-04-25T00:42:39.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:39 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:42:39.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:39 smithi032 ceph-mon[29357]: pgmap v289: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.4 KiB/s rd, 5 op/s 2024-04-25T00:42:39.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:39 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:42:40.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:40 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:42:40.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:40 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:42:40.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:40 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:42:40.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:40 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:42:40.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:40 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:42:40.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:40 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:42:40.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:40 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:42:40.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:40 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:42:40.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:40 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:42:41.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:41 smithi032 ceph-mon[29357]: pgmap v290: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:42:41.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:41 smithi121 ceph-mon[32231]: pgmap v290: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:42:41.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:41 smithi130 ceph-mon[32525]: pgmap v290: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:42:43.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:43 smithi032 ceph-mon[29357]: pgmap v291: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.4 KiB/s rd, 4 op/s 2024-04-25T00:42:43.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:43 smithi121 ceph-mon[32231]: pgmap v291: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.4 KiB/s rd, 4 op/s 2024-04-25T00:42:43.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:43 smithi130 ceph-mon[32525]: pgmap v291: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.4 KiB/s rd, 4 op/s 2024-04-25T00:42:45.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:45 smithi032 ceph-mon[29357]: pgmap v292: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.4 KiB/s rd, 5 op/s 2024-04-25T00:42:45.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:45 smithi121 ceph-mon[32231]: pgmap v292: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.4 KiB/s rd, 5 op/s 2024-04-25T00:42:45.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:45 smithi130 ceph-mon[32525]: pgmap v292: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.4 KiB/s rd, 5 op/s 2024-04-25T00:42:48.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:47 smithi032 ceph-mon[29357]: pgmap v293: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:42:48.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:47 smithi121 ceph-mon[32231]: pgmap v293: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:42:48.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:47 smithi130 ceph-mon[32525]: pgmap v293: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:42:50.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:49 smithi032 ceph-mon[29357]: pgmap v294: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:42:50.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:49 smithi121 ceph-mon[32231]: pgmap v294: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:42:50.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:49 smithi130 ceph-mon[32525]: pgmap v294: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:42:52.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:51 smithi032 ceph-mon[29357]: pgmap v295: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:42:52.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:51 smithi121 ceph-mon[32231]: pgmap v295: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:42:52.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:51 smithi130 ceph-mon[32525]: pgmap v295: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:42:54.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:53 smithi032 ceph-mon[29357]: pgmap v296: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:42:54.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:53 smithi121 ceph-mon[32231]: pgmap v296: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:42:54.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:53 smithi130 ceph-mon[32525]: pgmap v296: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:42:55.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:55 smithi121 ceph-mon[32231]: pgmap v297: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:42:55.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:55 smithi130 ceph-mon[32525]: pgmap v297: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:42:55.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:55 smithi032 ceph-mon[29357]: pgmap v297: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:42:57.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:57 smithi121 ceph-mon[32231]: pgmap v298: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:42:57.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:57 smithi130 ceph-mon[32525]: pgmap v298: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:42:57.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:57 smithi032 ceph-mon[29357]: pgmap v298: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:42:59.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:42:59 smithi121 ceph-mon[32231]: pgmap v299: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:42:59.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:42:59 smithi130 ceph-mon[32525]: pgmap v299: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:42:59.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:42:59 smithi032 ceph-mon[29357]: pgmap v299: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:43:01.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:01 smithi121 ceph-mon[32231]: pgmap v300: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:43:01.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:01 smithi130 ceph-mon[32525]: pgmap v300: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:43:01.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:01 smithi032 ceph-mon[29357]: pgmap v300: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:43:03.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:03 smithi121 ceph-mon[32231]: pgmap v301: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 11 KiB/s rd, 21 KiB/s wr, 19 op/s 2024-04-25T00:43:03.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:03 smithi130 ceph-mon[32525]: pgmap v301: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 11 KiB/s rd, 21 KiB/s wr, 19 op/s 2024-04-25T00:43:03.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:03 smithi032 ceph-mon[29357]: pgmap v301: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 11 KiB/s rd, 21 KiB/s wr, 19 op/s 2024-04-25T00:43:05.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:05 smithi121 ceph-mon[32231]: pgmap v302: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 12 KiB/s rd, 22 KiB/s wr, 20 op/s 2024-04-25T00:43:05.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:05 smithi130 ceph-mon[32525]: pgmap v302: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 12 KiB/s rd, 22 KiB/s wr, 20 op/s 2024-04-25T00:43:05.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:05 smithi032 ceph-mon[29357]: pgmap v302: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 12 KiB/s rd, 22 KiB/s wr, 20 op/s 2024-04-25T00:43:07.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:07 smithi121 ceph-mon[32231]: pgmap v303: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 14 KiB/s rd, 45 KiB/s wr, 24 op/s 2024-04-25T00:43:07.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:07 smithi130 ceph-mon[32525]: pgmap v303: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 14 KiB/s rd, 45 KiB/s wr, 24 op/s 2024-04-25T00:43:07.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:07 smithi032 ceph-mon[29357]: pgmap v303: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 14 KiB/s rd, 45 KiB/s wr, 24 op/s 2024-04-25T00:43:09.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:09 smithi130 ceph-mon[32525]: pgmap v304: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 13 KiB/s rd, 45 KiB/s wr, 23 op/s 2024-04-25T00:43:09.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:09 smithi130 ceph-mon[32525]: osdmap e101: 12 total, 12 up, 12 in 2024-04-25T00:43:09.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:09 smithi032 ceph-mon[29357]: pgmap v304: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 13 KiB/s rd, 45 KiB/s wr, 23 op/s 2024-04-25T00:43:09.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:09 smithi032 ceph-mon[29357]: osdmap e101: 12 total, 12 up, 12 in 2024-04-25T00:43:09.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:09 smithi121 ceph-mon[32231]: pgmap v304: 129 pgs: 129 active+clean; 42 GiB data, 93 GiB used, 980 GiB / 1.0 TiB avail; 13 KiB/s rd, 45 KiB/s wr, 23 op/s 2024-04-25T00:43:09.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:09 smithi121 ceph-mon[32231]: osdmap e101: 12 total, 12 up, 12 in 2024-04-25T00:43:11.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:11 smithi130 ceph-mon[32525]: pgmap v306: 129 pgs: 8 active+clean+snaptrim, 22 active+clean+snaptrim_wait, 99 active+clean; 39 GiB data, 89 GiB used, 983 GiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 27 op/s 2024-04-25T00:43:11.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:11 smithi032 ceph-mon[29357]: pgmap v306: 129 pgs: 8 active+clean+snaptrim, 22 active+clean+snaptrim_wait, 99 active+clean; 39 GiB data, 89 GiB used, 983 GiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 27 op/s 2024-04-25T00:43:11.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:11 smithi121 ceph-mon[32231]: pgmap v306: 129 pgs: 8 active+clean+snaptrim, 22 active+clean+snaptrim_wait, 99 active+clean; 39 GiB data, 89 GiB used, 983 GiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 27 op/s 2024-04-25T00:43:13.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:13 smithi130 ceph-mon[32525]: pgmap v307: 129 pgs: 12 active+clean+snaptrim, 35 active+clean+snaptrim_wait, 82 active+clean; 36 GiB data, 85 GiB used, 988 GiB / 1.0 TiB avail; 5.2 KiB/s rd, 28 KiB/s wr, 9 op/s 2024-04-25T00:43:13.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:13 smithi032 ceph-mon[29357]: pgmap v307: 129 pgs: 12 active+clean+snaptrim, 35 active+clean+snaptrim_wait, 82 active+clean; 36 GiB data, 85 GiB used, 988 GiB / 1.0 TiB avail; 5.2 KiB/s rd, 28 KiB/s wr, 9 op/s 2024-04-25T00:43:13.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:13 smithi121 ceph-mon[32231]: pgmap v307: 129 pgs: 12 active+clean+snaptrim, 35 active+clean+snaptrim_wait, 82 active+clean; 36 GiB data, 85 GiB used, 988 GiB / 1.0 TiB avail; 5.2 KiB/s rd, 28 KiB/s wr, 9 op/s 2024-04-25T00:43:15.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:15 smithi130 ceph-mon[32525]: pgmap v308: 129 pgs: 12 active+clean+snaptrim, 35 active+clean+snaptrim_wait, 82 active+clean; 36 GiB data, 83 GiB used, 990 GiB / 1.0 TiB avail; 4.4 KiB/s rd, 27 KiB/s wr, 9 op/s 2024-04-25T00:43:15.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:15 smithi032 ceph-mon[29357]: pgmap v308: 129 pgs: 12 active+clean+snaptrim, 35 active+clean+snaptrim_wait, 82 active+clean; 36 GiB data, 83 GiB used, 990 GiB / 1.0 TiB avail; 4.4 KiB/s rd, 27 KiB/s wr, 9 op/s 2024-04-25T00:43:15.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:15 smithi121 ceph-mon[32231]: pgmap v308: 129 pgs: 12 active+clean+snaptrim, 35 active+clean+snaptrim_wait, 82 active+clean; 36 GiB data, 83 GiB used, 990 GiB / 1.0 TiB avail; 4.4 KiB/s rd, 27 KiB/s wr, 9 op/s 2024-04-25T00:43:18.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:17 smithi032 ceph-mon[29357]: pgmap v309: 129 pgs: 129 active+clean; 32 GiB data, 74 GiB used, 999 GiB / 1.0 TiB avail; 3.2 KiB/s rd, 204 B/s wr, 6 op/s 2024-04-25T00:43:18.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:17 smithi032 ceph-mon[29357]: osdmap e102: 12 total, 12 up, 12 in 2024-04-25T00:43:18.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:17 smithi121 ceph-mon[32231]: pgmap v309: 129 pgs: 129 active+clean; 32 GiB data, 74 GiB used, 999 GiB / 1.0 TiB avail; 3.2 KiB/s rd, 204 B/s wr, 6 op/s 2024-04-25T00:43:18.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:17 smithi121 ceph-mon[32231]: osdmap e102: 12 total, 12 up, 12 in 2024-04-25T00:43:18.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:17 smithi130 ceph-mon[32525]: pgmap v309: 129 pgs: 129 active+clean; 32 GiB data, 74 GiB used, 999 GiB / 1.0 TiB avail; 3.2 KiB/s rd, 204 B/s wr, 6 op/s 2024-04-25T00:43:18.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:17 smithi130 ceph-mon[32525]: osdmap e102: 12 total, 12 up, 12 in 2024-04-25T00:43:20.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:19 smithi032 ceph-mon[29357]: pgmap v311: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.1 KiB/s rd, 103 B/s wr, 6 op/s 2024-04-25T00:43:20.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:19 smithi121 ceph-mon[32231]: pgmap v311: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.1 KiB/s rd, 103 B/s wr, 6 op/s 2024-04-25T00:43:20.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:19 smithi130 ceph-mon[32525]: pgmap v311: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.1 KiB/s rd, 103 B/s wr, 6 op/s 2024-04-25T00:43:22.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:21 smithi032 ceph-mon[29357]: pgmap v312: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.1 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:43:22.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:21 smithi121 ceph-mon[32231]: pgmap v312: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.1 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:43:22.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:21 smithi130 ceph-mon[32525]: pgmap v312: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.1 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:43:23.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:22 smithi032 ceph-mon[29357]: pgmap v313: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.1 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:43:23.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:22 smithi121 ceph-mon[32231]: pgmap v313: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.1 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:43:23.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:22 smithi130 ceph-mon[32525]: pgmap v313: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.1 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:43:25.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:25 smithi121 ceph-mon[32231]: pgmap v314: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-04-25T00:43:25.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:25 smithi130 ceph-mon[32525]: pgmap v314: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-04-25T00:43:25.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:25 smithi032 ceph-mon[29357]: pgmap v314: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-04-25T00:43:27.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:27 smithi121 ceph-mon[32231]: pgmap v315: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-04-25T00:43:27.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:27 smithi130 ceph-mon[32525]: pgmap v315: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-04-25T00:43:27.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:27 smithi032 ceph-mon[29357]: pgmap v315: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-04-25T00:43:29.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:29 smithi121 ceph-mon[32231]: pgmap v316: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.7 KiB/s rd, 6 op/s 2024-04-25T00:43:29.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:29 smithi130 ceph-mon[32525]: pgmap v316: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.7 KiB/s rd, 6 op/s 2024-04-25T00:43:29.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:29 smithi032 ceph-mon[29357]: pgmap v316: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.7 KiB/s rd, 6 op/s 2024-04-25T00:43:31.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:31 smithi121 ceph-mon[32231]: pgmap v317: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:43:31.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:31 smithi130 ceph-mon[32525]: pgmap v317: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:43:31.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:31 smithi032 ceph-mon[29357]: pgmap v317: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:43:33.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:33 smithi121 ceph-mon[32231]: pgmap v318: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:43:33.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:33 smithi130 ceph-mon[32525]: pgmap v318: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:43:33.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:33 smithi032 ceph-mon[29357]: pgmap v318: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:43:35.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:35 smithi130 ceph-mon[32525]: pgmap v319: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:43:35.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:35 smithi032 ceph-mon[29357]: pgmap v319: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:43:35.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:35 smithi121 ceph-mon[32231]: pgmap v319: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:43:37.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:37 smithi130 ceph-mon[32525]: pgmap v320: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:43:37.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:37 smithi032 ceph-mon[29357]: pgmap v320: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:43:37.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:37 smithi121 ceph-mon[32231]: pgmap v320: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:43:39.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:39 smithi130 ceph-mon[32525]: pgmap v321: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:43:39.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:39 smithi032 ceph-mon[29357]: pgmap v321: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:43:39.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:39 smithi121 ceph-mon[32231]: pgmap v321: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:43:40.609 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:40 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:43:40.687 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:40 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:43:40.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:40 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:43:41.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:41 smithi121 ceph-mon[32231]: pgmap v322: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:43:41.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:41 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:43:41.588 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:41 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:43:41.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:41 smithi130 ceph-mon[32525]: pgmap v322: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:43:41.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:41 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:43:41.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:41 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:43:41.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:41 smithi032 ceph-mon[29357]: pgmap v322: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:43:41.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:41 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:43:41.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:41 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:43:43.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:42 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:43:43.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:42 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:43:43.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:42 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:42 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:42 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:42 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:42 smithi121 ceph-mon[32231]: Adjusting osd_memory_target on smithi130 to 176.2M 2024-04-25T00:43:43.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:42 smithi121 ceph-mon[32231]: Unable to set osd_memory_target on smithi130 to 184770252: error parsing value: Value '184770252' is below minimum 939524096 2024-04-25T00:43:43.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:42 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:43:43.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:42 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:43:43.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:42 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:42 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:42 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:42 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:42 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:42 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:42 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:42 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:42 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:43:43.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:42 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:43:43.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:42 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:43:43.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:42 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:43:43.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:42 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:43:43.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:42 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:42 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:42 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:42 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:42 smithi130 ceph-mon[32525]: Adjusting osd_memory_target on smithi130 to 176.2M 2024-04-25T00:43:43.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:42 smithi130 ceph-mon[32525]: Unable to set osd_memory_target on smithi130 to 184770252: error parsing value: Value '184770252' is below minimum 939524096 2024-04-25T00:43:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:42 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:43:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:42 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:43:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:42 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:42 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:42 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:42 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:42 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:42 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:42 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:42 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:42 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:43:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:42 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:43:43.143 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:42 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:43:43.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:42 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:43:43.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:42 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:43:43.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:42 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:42 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:42 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:42 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:42 smithi032 ceph-mon[29357]: Adjusting osd_memory_target on smithi130 to 176.2M 2024-04-25T00:43:43.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:42 smithi032 ceph-mon[29357]: Unable to set osd_memory_target on smithi130 to 184770252: error parsing value: Value '184770252' is below minimum 939524096 2024-04-25T00:43:43.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:42 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:43:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:42 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:43:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:42 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:42 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:42 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:42 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:42 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:42 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:42 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:42 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-04-25T00:43:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:42 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:43:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:42 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:43:43.250 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:42 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:43:44.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:43 smithi032 ceph-mon[29357]: pgmap v323: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:43:44.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:43 smithi032 ceph-mon[29357]: Adjusting osd_memory_target on smithi032 to 177.2M 2024-04-25T00:43:44.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:43 smithi032 ceph-mon[29357]: Unable to set osd_memory_target on smithi032 to 185897062: error parsing value: Value '185897062' is below minimum 939524096 2024-04-25T00:43:44.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:43 smithi032 ceph-mon[29357]: Adjusting osd_memory_target on smithi121 to 176.1M 2024-04-25T00:43:44.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:43 smithi032 ceph-mon[29357]: Unable to set osd_memory_target on smithi121 to 184727961: error parsing value: Value '184727961' is below minimum 939524096 2024-04-25T00:43:44.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:43 smithi121 ceph-mon[32231]: pgmap v323: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:43:44.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:43 smithi121 ceph-mon[32231]: Adjusting osd_memory_target on smithi032 to 177.2M 2024-04-25T00:43:44.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:43 smithi121 ceph-mon[32231]: Unable to set osd_memory_target on smithi032 to 185897062: error parsing value: Value '185897062' is below minimum 939524096 2024-04-25T00:43:44.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:43 smithi121 ceph-mon[32231]: Adjusting osd_memory_target on smithi121 to 176.1M 2024-04-25T00:43:44.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:43 smithi121 ceph-mon[32231]: Unable to set osd_memory_target on smithi121 to 184727961: error parsing value: Value '184727961' is below minimum 939524096 2024-04-25T00:43:44.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:43 smithi130 ceph-mon[32525]: pgmap v323: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:43:44.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:43 smithi130 ceph-mon[32525]: Adjusting osd_memory_target on smithi032 to 177.2M 2024-04-25T00:43:44.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:43 smithi130 ceph-mon[32525]: Unable to set osd_memory_target on smithi032 to 185897062: error parsing value: Value '185897062' is below minimum 939524096 2024-04-25T00:43:44.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:43 smithi130 ceph-mon[32525]: Adjusting osd_memory_target on smithi121 to 176.1M 2024-04-25T00:43:44.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:43 smithi130 ceph-mon[32525]: Unable to set osd_memory_target on smithi121 to 184727961: error parsing value: Value '184727961' is below minimum 939524096 2024-04-25T00:43:46.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:45 smithi032 ceph-mon[29357]: pgmap v324: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:43:46.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:45 smithi121 ceph-mon[32231]: pgmap v324: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:43:46.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:45 smithi130 ceph-mon[32525]: pgmap v324: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:43:48.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:47 smithi032 ceph-mon[29357]: pgmap v325: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:43:48.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:47 smithi121 ceph-mon[32231]: pgmap v325: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:43:48.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:47 smithi130 ceph-mon[32525]: pgmap v325: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:43:50.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:49 smithi032 ceph-mon[29357]: pgmap v326: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:43:50.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:49 smithi121 ceph-mon[32231]: pgmap v326: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:43:50.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:49 smithi130 ceph-mon[32525]: pgmap v326: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:43:52.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:51 smithi032 ceph-mon[29357]: pgmap v327: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:43:52.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:51 smithi121 ceph-mon[32231]: pgmap v327: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:43:52.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:51 smithi130 ceph-mon[32525]: pgmap v327: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:43:54.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:53 smithi032 ceph-mon[29357]: pgmap v328: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:43:54.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:53 smithi121 ceph-mon[32231]: pgmap v328: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:43:54.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:53 smithi130 ceph-mon[32525]: pgmap v328: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:43:56.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:55 smithi032 ceph-mon[29357]: pgmap v329: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:43:56.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:55 smithi121 ceph-mon[32231]: pgmap v329: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:43:56.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:55 smithi130 ceph-mon[32525]: pgmap v329: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:43:57.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:57 smithi121 ceph-mon[32231]: pgmap v330: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:43:57.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:57 smithi130 ceph-mon[32525]: pgmap v330: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:43:57.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:57 smithi032 ceph-mon[29357]: pgmap v330: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:43:59.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:43:59 smithi121 ceph-mon[32231]: pgmap v331: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:43:59.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:43:59 smithi130 ceph-mon[32525]: pgmap v331: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:43:59.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:43:59 smithi032 ceph-mon[29357]: pgmap v331: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:44:01.587 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:01 smithi121 ceph-mon[32231]: pgmap v332: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:44:01.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:01 smithi130 ceph-mon[32525]: pgmap v332: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:44:01.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:01 smithi032 ceph-mon[29357]: pgmap v332: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:44:03.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:03 smithi130 ceph-mon[32525]: pgmap v333: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 11 KiB/s rd, 21 KiB/s wr, 19 op/s 2024-04-25T00:44:03.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:03 smithi032 ceph-mon[29357]: pgmap v333: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 11 KiB/s rd, 21 KiB/s wr, 19 op/s 2024-04-25T00:44:03.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:03 smithi121 ceph-mon[32231]: pgmap v333: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 11 KiB/s rd, 21 KiB/s wr, 19 op/s 2024-04-25T00:44:05.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:05 smithi130 ceph-mon[32525]: pgmap v334: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 12 KiB/s rd, 22 KiB/s wr, 20 op/s 2024-04-25T00:44:05.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:05 smithi032 ceph-mon[29357]: pgmap v334: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 12 KiB/s rd, 22 KiB/s wr, 20 op/s 2024-04-25T00:44:05.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:05 smithi121 ceph-mon[32231]: pgmap v334: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 12 KiB/s rd, 22 KiB/s wr, 20 op/s 2024-04-25T00:44:07.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:07 smithi130 ceph-mon[32525]: pgmap v335: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 14 KiB/s rd, 44 KiB/s wr, 23 op/s 2024-04-25T00:44:07.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:07 smithi032 ceph-mon[29357]: pgmap v335: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 14 KiB/s rd, 44 KiB/s wr, 23 op/s 2024-04-25T00:44:07.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:07 smithi121 ceph-mon[32231]: pgmap v335: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 14 KiB/s rd, 44 KiB/s wr, 23 op/s 2024-04-25T00:44:09.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:09 smithi130 ceph-mon[32525]: pgmap v336: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 13 KiB/s rd, 45 KiB/s wr, 23 op/s 2024-04-25T00:44:09.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:09 smithi130 ceph-mon[32525]: osdmap e103: 12 total, 12 up, 12 in 2024-04-25T00:44:09.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:09 smithi032 ceph-mon[29357]: pgmap v336: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 13 KiB/s rd, 45 KiB/s wr, 23 op/s 2024-04-25T00:44:09.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:09 smithi032 ceph-mon[29357]: osdmap e103: 12 total, 12 up, 12 in 2024-04-25T00:44:09.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:09 smithi121 ceph-mon[32231]: pgmap v336: 129 pgs: 129 active+clean; 32 GiB data, 73 GiB used, 999 GiB / 1.0 TiB avail; 13 KiB/s rd, 45 KiB/s wr, 23 op/s 2024-04-25T00:44:09.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:09 smithi121 ceph-mon[32231]: osdmap e103: 12 total, 12 up, 12 in 2024-04-25T00:44:11.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:11 smithi130 ceph-mon[32525]: pgmap v338: 129 pgs: 9 active+clean+snaptrim, 21 active+clean+snaptrim_wait, 99 active+clean; 30 GiB data, 70 GiB used, 1003 GiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 26 op/s 2024-04-25T00:44:11.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:11 smithi032 ceph-mon[29357]: pgmap v338: 129 pgs: 9 active+clean+snaptrim, 21 active+clean+snaptrim_wait, 99 active+clean; 30 GiB data, 70 GiB used, 1003 GiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 26 op/s 2024-04-25T00:44:11.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:11 smithi121 ceph-mon[32231]: pgmap v338: 129 pgs: 9 active+clean+snaptrim, 21 active+clean+snaptrim_wait, 99 active+clean; 30 GiB data, 70 GiB used, 1003 GiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 26 op/s 2024-04-25T00:44:13.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:13 smithi130 ceph-mon[32525]: pgmap v339: 129 pgs: 15 active+clean+snaptrim, 34 active+clean+snaptrim_wait, 80 active+clean; 26 GiB data, 65 GiB used, 1008 GiB / 1.0 TiB avail; 5.0 KiB/s rd, 28 KiB/s wr, 9 op/s 2024-04-25T00:44:13.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:13 smithi032 ceph-mon[29357]: pgmap v339: 129 pgs: 15 active+clean+snaptrim, 34 active+clean+snaptrim_wait, 80 active+clean; 26 GiB data, 65 GiB used, 1008 GiB / 1.0 TiB avail; 5.0 KiB/s rd, 28 KiB/s wr, 9 op/s 2024-04-25T00:44:13.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:13 smithi121 ceph-mon[32231]: pgmap v339: 129 pgs: 15 active+clean+snaptrim, 34 active+clean+snaptrim_wait, 80 active+clean; 26 GiB data, 65 GiB used, 1008 GiB / 1.0 TiB avail; 5.0 KiB/s rd, 28 KiB/s wr, 9 op/s 2024-04-25T00:44:15.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:15 smithi130 ceph-mon[32525]: pgmap v340: 129 pgs: 15 active+clean+snaptrim, 34 active+clean+snaptrim_wait, 80 active+clean; 26 GiB data, 63 GiB used, 1010 GiB / 1.0 TiB avail; 4.4 KiB/s rd, 27 KiB/s wr, 9 op/s 2024-04-25T00:44:15.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:15 smithi032 ceph-mon[29357]: pgmap v340: 129 pgs: 15 active+clean+snaptrim, 34 active+clean+snaptrim_wait, 80 active+clean; 26 GiB data, 63 GiB used, 1010 GiB / 1.0 TiB avail; 4.4 KiB/s rd, 27 KiB/s wr, 9 op/s 2024-04-25T00:44:15.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:15 smithi121 ceph-mon[32231]: pgmap v340: 129 pgs: 15 active+clean+snaptrim, 34 active+clean+snaptrim_wait, 80 active+clean; 26 GiB data, 63 GiB used, 1010 GiB / 1.0 TiB avail; 4.4 KiB/s rd, 27 KiB/s wr, 9 op/s 2024-04-25T00:44:18.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:17 smithi032 ceph-mon[29357]: pgmap v341: 129 pgs: 1 active+clean+snaptrim, 128 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 3.2 KiB/s rd, 204 B/s wr, 6 op/s 2024-04-25T00:44:18.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:17 smithi032 ceph-mon[29357]: osdmap e104: 12 total, 12 up, 12 in 2024-04-25T00:44:18.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:17 smithi121 ceph-mon[32231]: pgmap v341: 129 pgs: 1 active+clean+snaptrim, 128 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 3.2 KiB/s rd, 204 B/s wr, 6 op/s 2024-04-25T00:44:18.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:17 smithi121 ceph-mon[32231]: osdmap e104: 12 total, 12 up, 12 in 2024-04-25T00:44:18.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:17 smithi130 ceph-mon[32525]: pgmap v341: 129 pgs: 1 active+clean+snaptrim, 128 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 3.2 KiB/s rd, 204 B/s wr, 6 op/s 2024-04-25T00:44:18.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:17 smithi130 ceph-mon[32525]: osdmap e104: 12 total, 12 up, 12 in 2024-04-25T00:44:19.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:19 smithi121 ceph-mon[32231]: pgmap v343: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 3.1 KiB/s rd, 103 B/s wr, 6 op/s 2024-04-25T00:44:19.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:19 smithi130 ceph-mon[32525]: pgmap v343: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 3.1 KiB/s rd, 103 B/s wr, 6 op/s 2024-04-25T00:44:19.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:19 smithi032 ceph-mon[29357]: pgmap v343: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 3.1 KiB/s rd, 103 B/s wr, 6 op/s 2024-04-25T00:44:21.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:21 smithi130 ceph-mon[32525]: pgmap v344: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 3.1 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:44:21.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:21 smithi032 ceph-mon[29357]: pgmap v344: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 3.1 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:44:21.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:21 smithi121 ceph-mon[32231]: pgmap v344: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 3.1 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:44:23.248 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:23 smithi032 ceph-mon[29357]: osdmap e105: 12 total, 12 up, 12 in 2024-04-25T00:44:23.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:23 smithi121 ceph-mon[32231]: osdmap e105: 12 total, 12 up, 12 in 2024-04-25T00:44:23.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:23 smithi130 ceph-mon[32525]: osdmap e105: 12 total, 12 up, 12 in 2024-04-25T00:44:24.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:24 smithi121 ceph-mon[32231]: pgmap v346: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:44:24.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:24 smithi130 ceph-mon[32525]: pgmap v346: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:44:24.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:24 smithi032 ceph-mon[29357]: pgmap v346: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:44:25.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:25 smithi121 ceph-mon[32231]: pgmap v347: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 1.9 KiB/s rd, 4 op/s 2024-04-25T00:44:25.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:25 smithi130 ceph-mon[32525]: pgmap v347: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 1.9 KiB/s rd, 4 op/s 2024-04-25T00:44:25.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:25 smithi032 ceph-mon[29357]: pgmap v347: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 1.9 KiB/s rd, 4 op/s 2024-04-25T00:44:27.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:27 smithi130 ceph-mon[32525]: pgmap v348: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 3.2 KiB/s rd, 6 op/s 2024-04-25T00:44:27.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:27 smithi032 ceph-mon[29357]: pgmap v348: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 3.2 KiB/s rd, 6 op/s 2024-04-25T00:44:27.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:27 smithi121 ceph-mon[32231]: pgmap v348: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 3.2 KiB/s rd, 6 op/s 2024-04-25T00:44:29.640 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:29 smithi121 ceph-mon[32231]: pgmap v349: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-04-25T00:44:29.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:29 smithi130 ceph-mon[32525]: pgmap v349: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-04-25T00:44:29.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:29 smithi032 ceph-mon[29357]: pgmap v349: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-04-25T00:44:31.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:31 smithi130 ceph-mon[32525]: pgmap v350: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-04-25T00:44:31.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:31 smithi032 ceph-mon[29357]: pgmap v350: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-04-25T00:44:31.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:31 smithi121 ceph-mon[32231]: pgmap v350: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-04-25T00:44:33.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:33 smithi130 ceph-mon[32525]: pgmap v351: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-25T00:44:33.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:33 smithi032 ceph-mon[29357]: pgmap v351: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-25T00:44:33.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:33 smithi121 ceph-mon[32231]: pgmap v351: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-25T00:44:35.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:35 smithi130 ceph-mon[32525]: pgmap v352: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:44:35.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:35 smithi032 ceph-mon[29357]: pgmap v352: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:44:35.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:35 smithi121 ceph-mon[32231]: pgmap v352: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:44:37.544 INFO:tasks.workunit.client.0.smithi032.stdout:stopping iogen 2024-04-25T00:44:37.544 INFO:tasks.workunit.client.0.smithi032.stderr:+ echo 'stopping iogen' 2024-04-25T00:44:37.544 INFO:tasks.workunit.client.0.smithi032.stderr:+ ./iogen -k 2024-04-25T00:44:37.549 INFO:tasks.workunit.client.0.smithi032.stdout:OK 2024-04-25T00:44:37.550 INFO:tasks.workunit.client.0.smithi032.stderr:+ echo OK 2024-04-25T00:44:37.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:37 smithi130 ceph-mon[32525]: pgmap v353: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:44:37.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:37 smithi032 ceph-mon[29357]: pgmap v353: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:44:37.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:37 smithi121 ceph-mon[32231]: pgmap v353: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:44:39.569 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-04-25T00:44:39.569 DEBUG:teuthology.orchestra.run.smithi032:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-04-25T00:44:39.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:39 smithi130 ceph-mon[32525]: pgmap v354: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:44:39.697 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:39 smithi032 ceph-mon[29357]: pgmap v354: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:44:39.698 INFO:tasks.workunit:Stopping ['suites/iogen.sh'] on client.0... 2024-04-25T00:44:39.698 DEBUG:teuthology.orchestra.run.smithi032:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-04-25T00:44:39.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:39 smithi121 ceph-mon[32231]: pgmap v354: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-25T00:44:40.005 DEBUG:teuthology.parallel:result is None 2024-04-25T00:44:40.005 DEBUG:teuthology.orchestra.run.smithi032:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-04-25T00:44:40.046 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-04-25T00:44:40.046 DEBUG:teuthology.run_tasks:Unwinding manager full_sequential_finally 2024-04-25T00:44:40.058 INFO:teuthology.task.full_sequential_finally:In full_sequential_finally, running task exec... 2024-04-25T00:44:40.058 INFO:teuthology.task.exec:Executing custom commands... 2024-04-25T00:44:40.058 INFO:teuthology.task.exec:Running commands on role mon.a host ubuntu@smithi032.front.sepia.ceph.com 2024-04-25T00:44:40.059 DEBUG:teuthology.orchestra.run.smithi032:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'date +%s > END_TIME' 2024-04-25T00:44:40.123 DEBUG:teuthology.orchestra.run.smithi032:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'START_TIME=$(cat START_TIME); END_TIME=$(cat END_TIME); DIFF_TIME=$((600-(END_TIME-START_TIME))); if [ "$DIFF_TIME" -gt 0 ]; then sleep "$DIFF_TIME"; fi' 2024-04-25T00:44:40.197 DEBUG:teuthology.orchestra.run.smithi032:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule status --fs=cephfs --path=/' 2024-04-25T00:44:40.343 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.340+0000 7f4b4081c640 1 -- 172.21.15.32:0/2629115150 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4b3c106630 msgr2=0x7f4b3c106a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:40.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.340+0000 7f4b4081c640 1 --2- 172.21.15.32:0/2629115150 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4b3c106630 0x7f4b3c106a10 secure :-1 s=READY pgs=645 cs=0 l=1 rev1=1 crypto rx=0x7f4b2c009930 tx=0x7f4b2c02a420 comp rx=0 tx=0).stop 2024-04-25T00:44:40.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.341+0000 7f4b4081c640 1 -- 172.21.15.32:0/2629115150 shutdown_connections 2024-04-25T00:44:40.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.341+0000 7f4b4081c640 1 --2- 172.21.15.32:0/2629115150 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b3c101030 0x7f4b3c10d4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:40.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.341+0000 7f4b4081c640 1 --2- 172.21.15.32:0/2629115150 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4b3c100670 0x7f4b3c100af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:40.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.341+0000 7f4b4081c640 1 --2- 172.21.15.32:0/2629115150 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4b3c106630 0x7f4b3c106a10 unknown :-1 s=CLOSED pgs=645 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:40.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.341+0000 7f4b4081c640 1 -- 172.21.15.32:0/2629115150 >> 172.21.15.32:0/2629115150 conn(0x7f4b3c0777c0 msgr2=0x7f4b3c077bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:44:40.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.341+0000 7f4b4081c640 1 -- 172.21.15.32:0/2629115150 shutdown_connections 2024-04-25T00:44:40.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.341+0000 7f4b4081c640 1 -- 172.21.15.32:0/2629115150 wait complete. 2024-04-25T00:44:40.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.341+0000 7f4b4081c640 1 Processor -- start 2024-04-25T00:44:40.344 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.341+0000 7f4b4081c640 1 -- start start 2024-04-25T00:44:40.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.342+0000 7f4b4081c640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b3c100670 0x7f4b3c196ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:40.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.342+0000 7f4b4081c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4b3c101030 0x7f4b3c1973e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:40.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.342+0000 7f4b4081c640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4b3c106630 0x7f4b3c19b710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:40.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.342+0000 7f4b4081c640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f4b3c110a40 con 0x7f4b3c101030 2024-04-25T00:44:40.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.342+0000 7f4b4081c640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f4b3c1108c0 con 0x7f4b3c106630 2024-04-25T00:44:40.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.342+0000 7f4b4081c640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f4b3c110bc0 con 0x7f4b3c100670 2024-04-25T00:44:40.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.342+0000 7f4b33fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4b3c101030 0x7f4b3c1973e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:40.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.342+0000 7f4b33fff640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4b3c101030 0x7f4b3c1973e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:38360/0 (socket says 172.21.15.32:38360) 2024-04-25T00:44:40.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.342+0000 7f4b33fff640 1 -- 172.21.15.32:0/3039625272 learned_addr learned my addr 172.21.15.32:0/3039625272 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:44:40.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.342+0000 7f4b3ad9d640 1 --2- 172.21.15.32:0/3039625272 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4b3c106630 0x7f4b3c19b710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:40.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.342+0000 7f4b3a59c640 1 --2- 172.21.15.32:0/3039625272 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b3c100670 0x7f4b3c196ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:40.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.342+0000 7f4b33fff640 1 -- 172.21.15.32:0/3039625272 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b3c100670 msgr2=0x7f4b3c196ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:40.345 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.342+0000 7f4b33fff640 1 --2- 172.21.15.32:0/3039625272 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b3c100670 0x7f4b3c196ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:40.346 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.342+0000 7f4b33fff640 1 -- 172.21.15.32:0/3039625272 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4b3c106630 msgr2=0x7f4b3c19b710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:40.346 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.342+0000 7f4b33fff640 1 --2- 172.21.15.32:0/3039625272 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4b3c106630 0x7f4b3c19b710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:40.346 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.342+0000 7f4b33fff640 1 -- 172.21.15.32:0/3039625272 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4b3c19be40 con 0x7f4b3c101030 2024-04-25T00:44:40.346 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.342+0000 7f4b3a59c640 1 --2- 172.21.15.32:0/3039625272 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b3c100670 0x7f4b3c196ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:44:40.346 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.342+0000 7f4b3ad9d640 1 --2- 172.21.15.32:0/3039625272 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4b3c106630 0x7f4b3c19b710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:44:40.346 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.342+0000 7f4b33fff640 1 --2- 172.21.15.32:0/3039625272 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4b3c101030 0x7f4b3c1973e0 secure :-1 s=READY pgs=646 cs=0 l=1 rev1=1 crypto rx=0x7f4b2400d950 tx=0x7f4b2400de10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:44:40.346 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.342+0000 7f4b337fe640 1 -- 172.21.15.32:0/3039625272 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4b24017070 con 0x7f4b3c101030 2024-04-25T00:44:40.346 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.342+0000 7f4b4081c640 1 -- 172.21.15.32:0/3039625272 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4b3c19c130 con 0x7f4b3c101030 2024-04-25T00:44:40.346 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.342+0000 7f4b337fe640 1 -- 172.21.15.32:0/3039625272 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4b24009c30 con 0x7f4b3c101030 2024-04-25T00:44:40.346 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.342+0000 7f4b337fe640 1 -- 172.21.15.32:0/3039625272 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4b2400f6b0 con 0x7f4b3c101030 2024-04-25T00:44:40.346 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.343+0000 7f4b4081c640 1 -- 172.21.15.32:0/3039625272 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4b3c1a39c0 con 0x7f4b3c101030 2024-04-25T00:44:40.347 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.344+0000 7f4b4081c640 1 -- 172.21.15.32:0/3039625272 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4b3c04f500 con 0x7f4b3c101030 2024-04-25T00:44:40.347 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.345+0000 7f4b337fe640 1 -- 172.21.15.32:0/3039625272 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f4b240109d0 con 0x7f4b3c101030 2024-04-25T00:44:40.347 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.346+0000 7f4b337fe640 1 --2- 172.21.15.32:0/3039625272 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f4b280a2da0 0x7f4b280a5260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:40.348 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.346+0000 7f4b3a59c640 1 --2- 172.21.15.32:0/3039625272 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f4b280a2da0 0x7f4b280a5260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:40.348 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.346+0000 7f4b337fe640 1 -- 172.21.15.32:0/3039625272 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(105..105 src has 1..105) ==== 9510+0+0 (secure 0 0 0) 0x7f4b240cf4f0 con 0x7f4b3c101030 2024-04-25T00:44:40.348 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.346+0000 7f4b3a59c640 1 --2- 172.21.15.32:0/3039625272 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f4b280a2da0 0x7f4b280a5260 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f4b2c000c00 tx=0x7f4b2c032c70 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:44:40.351 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.350+0000 7f4b337fe640 1 -- 172.21.15.32:0/3039625272 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f4b24009de0 con 0x7f4b3c101030 2024-04-25T00:44:40.558 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.558+0000 7f4b4081c640 1 -- 172.21.15.32:0/3039625272 --> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] -- mgr_command(tid 0: {"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}) -- 0x7f4b3c061b80 con 0x7f4b280a2da0 2024-04-25T00:44:40.566 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.566+0000 7f4b337fe640 1 -- 172.21.15.32:0/3039625272 <== mgr.15483 v2:172.21.15.32:6800/3307874911 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+350 (secure 0 0 0) 0x7f4b3c061b80 con 0x7f4b280a2da0 2024-04-25T00:44:40.566 INFO:teuthology.orchestra.run.smithi032.stdout:{"fs": "cephfs", "subvol": null, "group": null, "path": "/", "rel_path": "/", "schedule": "1m", "retention": {"m": 6, "h": 3}, "start": "2024-04-25T00:00:00", "created": "2024-04-25T00:33:24", "first": "2024-04-25T00:34:00", "last": "2024-04-25T00:44:00", "last_pruned": "2024-04-25T00:44:00", "created_count": 11, "pruned_count": 5, "active": true} 2024-04-25T00:44:40.569 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.568+0000 7f4b4081c640 1 -- 172.21.15.32:0/3039625272 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f4b280a2da0 msgr2=0x7f4b280a5260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:40.569 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.568+0000 7f4b4081c640 1 --2- 172.21.15.32:0/3039625272 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f4b280a2da0 0x7f4b280a5260 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f4b2c000c00 tx=0x7f4b2c032c70 comp rx=0 tx=0).stop 2024-04-25T00:44:40.569 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.568+0000 7f4b4081c640 1 -- 172.21.15.32:0/3039625272 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4b3c101030 msgr2=0x7f4b3c1973e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:40.569 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.568+0000 7f4b4081c640 1 --2- 172.21.15.32:0/3039625272 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4b3c101030 0x7f4b3c1973e0 secure :-1 s=READY pgs=646 cs=0 l=1 rev1=1 crypto rx=0x7f4b2400d950 tx=0x7f4b2400de10 comp rx=0 tx=0).stop 2024-04-25T00:44:40.569 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.568+0000 7f4b4081c640 1 -- 172.21.15.32:0/3039625272 shutdown_connections 2024-04-25T00:44:40.569 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.568+0000 7f4b4081c640 1 --2- 172.21.15.32:0/3039625272 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f4b280a2da0 0x7f4b280a5260 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:40.569 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.568+0000 7f4b4081c640 1 --2- 172.21.15.32:0/3039625272 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f4b3c106630 0x7f4b3c19b710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:40.569 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.568+0000 7f4b4081c640 1 --2- 172.21.15.32:0/3039625272 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f4b3c101030 0x7f4b3c1973e0 unknown :-1 s=CLOSED pgs=646 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:40.569 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.568+0000 7f4b4081c640 1 --2- 172.21.15.32:0/3039625272 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b3c100670 0x7f4b3c196ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:40.569 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.568+0000 7f4b4081c640 1 -- 172.21.15.32:0/3039625272 >> 172.21.15.32:0/3039625272 conn(0x7f4b3c0777c0 msgr2=0x7f4b3c103190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:44:40.570 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.568+0000 7f4b4081c640 1 -- 172.21.15.32:0/3039625272 shutdown_connections 2024-04-25T00:44:40.570 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.568+0000 7f4b4081c640 1 -- 172.21.15.32:0/3039625272 wait complete. 2024-04-25T00:44:40.580 DEBUG:teuthology.orchestra.run.smithi032:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule list --fs=cephfs --path=/ --recursive=true' 2024-04-25T00:44:40.686 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.684+0000 7f6c0f8ca640 1 -- 172.21.15.32:0/1663763133 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6c08073b10 msgr2=0x7f6c08073ef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:40.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.684+0000 7f6c0f8ca640 1 --2- 172.21.15.32:0/1663763133 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6c08073b10 0x7f6c08073ef0 secure :-1 s=READY pgs=647 cs=0 l=1 rev1=1 crypto rx=0x7f6c04009900 tx=0x7f6c0402a480 comp rx=0 tx=0).stop 2024-04-25T00:44:40.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.684+0000 7f6c0f8ca640 1 -- 172.21.15.32:0/1663763133 shutdown_connections 2024-04-25T00:44:40.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.684+0000 7f6c0f8ca640 1 --2- 172.21.15.32:0/1663763133 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6c0810dfe0 0x7f6c081103f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:40.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.684+0000 7f6c0f8ca640 1 --2- 172.21.15.32:0/1663763133 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6c08074430 0x7f6c0810daa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:40.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.684+0000 7f6c0f8ca640 1 --2- 172.21.15.32:0/1663763133 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6c08073b10 0x7f6c08073ef0 unknown :-1 s=CLOSED pgs=647 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:40.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.684+0000 7f6c0f8ca640 1 -- 172.21.15.32:0/1663763133 >> 172.21.15.32:0/1663763133 conn(0x7f6c080fd210 msgr2=0x7f6c080ff630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:44:40.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.684+0000 7f6c0f8ca640 1 -- 172.21.15.32:0/1663763133 shutdown_connections 2024-04-25T00:44:40.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.684+0000 7f6c0f8ca640 1 -- 172.21.15.32:0/1663763133 wait complete. 2024-04-25T00:44:40.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.684+0000 7f6c0f8ca640 1 Processor -- start 2024-04-25T00:44:40.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.684+0000 7f6c0f8ca640 1 -- start start 2024-04-25T00:44:40.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.685+0000 7f6c0f8ca640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6c08073b10 0x7f6c0819a220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:40.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.685+0000 7f6c0f8ca640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6c08074430 0x7f6c0819a760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:40.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.685+0000 7f6c0f8ca640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6c0810dfe0 0x7f6c0819eae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:40.687 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.685+0000 7f6c0f8ca640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f6c08113dc0 con 0x7f6c08074430 2024-04-25T00:44:40.688 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.685+0000 7f6c0f8ca640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f6c08113c40 con 0x7f6c0810dfe0 2024-04-25T00:44:40.688 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.685+0000 7f6c0f8ca640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f6c08113f40 con 0x7f6c08073b10 2024-04-25T00:44:40.688 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.685+0000 7f6c0ce65640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6c08074430 0x7f6c0819a760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:40.688 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.685+0000 7f6c0ce65640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6c08074430 0x7f6c0819a760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:38370/0 (socket says 172.21.15.32:38370) 2024-04-25T00:44:40.688 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.685+0000 7f6c0ce65640 1 -- 172.21.15.32:0/3935485173 learned_addr learned my addr 172.21.15.32:0/3935485173 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:44:40.688 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.685+0000 7f6c0de67640 1 --2- 172.21.15.32:0/3935485173 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6c0810dfe0 0x7f6c0819eae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:40.688 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.685+0000 7f6c0d666640 1 --2- 172.21.15.32:0/3935485173 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6c08073b10 0x7f6c0819a220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:40.688 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.685+0000 7f6c0ce65640 1 -- 172.21.15.32:0/3935485173 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6c08073b10 msgr2=0x7f6c0819a220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:40.688 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.685+0000 7f6c0ce65640 1 --2- 172.21.15.32:0/3935485173 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6c08073b10 0x7f6c0819a220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:40.688 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.685+0000 7f6c0ce65640 1 -- 172.21.15.32:0/3935485173 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6c0810dfe0 msgr2=0x7f6c0819eae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:40.688 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.685+0000 7f6c0ce65640 1 --2- 172.21.15.32:0/3935485173 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6c0810dfe0 0x7f6c0819eae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:40.688 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.685+0000 7f6c0ce65640 1 -- 172.21.15.32:0/3935485173 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6c0819f210 con 0x7f6c08074430 2024-04-25T00:44:40.688 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.685+0000 7f6c0ce65640 1 --2- 172.21.15.32:0/3935485173 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6c08074430 0x7f6c0819a760 secure :-1 s=READY pgs=648 cs=0 l=1 rev1=1 crypto rx=0x7f6bf800b4a0 tx=0x7f6bf800b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:44:40.688 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.685+0000 7f6bfe7fc640 1 -- 172.21.15.32:0/3935485173 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6bf8014070 con 0x7f6c08074430 2024-04-25T00:44:40.688 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.686+0000 7f6bfe7fc640 1 -- 172.21.15.32:0/3935485173 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6bf8010af0 con 0x7f6c08074430 2024-04-25T00:44:40.689 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.686+0000 7f6c0f8ca640 1 -- 172.21.15.32:0/3935485173 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6c0819f500 con 0x7f6c08074430 2024-04-25T00:44:40.689 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.686+0000 7f6bfe7fc640 1 -- 172.21.15.32:0/3935485173 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6bf8017d10 con 0x7f6c08074430 2024-04-25T00:44:40.689 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.686+0000 7f6c0f8ca640 1 -- 172.21.15.32:0/3935485173 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f6c081a6d90 con 0x7f6c08074430 2024-04-25T00:44:40.689 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.686+0000 7f6c0f8ca640 1 -- 172.21.15.32:0/3935485173 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6c0804f500 con 0x7f6c08074430 2024-04-25T00:44:40.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.688+0000 7f6bfe7fc640 1 -- 172.21.15.32:0/3935485173 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f6bf8017070 con 0x7f6c08074430 2024-04-25T00:44:40.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.688+0000 7f6bfe7fc640 1 --2- 172.21.15.32:0/3935485173 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f6bf40a2e20 0x7f6bf40a52e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:40.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.688+0000 7f6c0d666640 1 --2- 172.21.15.32:0/3935485173 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f6bf40a2e20 0x7f6bf40a52e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:40.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.688+0000 7f6bfe7fc640 1 -- 172.21.15.32:0/3935485173 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(105..105 src has 1..105) ==== 9510+0+0 (secure 0 0 0) 0x7f6bf80ced40 con 0x7f6c08074430 2024-04-25T00:44:40.690 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.689+0000 7f6c0d666640 1 --2- 172.21.15.32:0/3935485173 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f6bf40a2e20 0x7f6bf40a52e0 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f6c04009230 tx=0x7f6c040058d0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:44:40.693 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.692+0000 7f6bfe7fc640 1 -- 172.21.15.32:0/3935485173 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f6bf809b040 con 0x7f6c08074430 2024-04-25T00:44:40.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.900+0000 7f6c0f8ca640 1 -- 172.21.15.32:0/3935485173 --> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] -- mgr_command(tid 0: {"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}) -- 0x7f6c08061b80 con 0x7f6bf40a2e20 2024-04-25T00:44:40.908 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.907+0000 7f6bfe7fc640 1 -- 172.21.15.32:0/3935485173 <== mgr.15483 v2:172.21.15.32:6800/3307874911 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+10 (secure 0 0 0) 0x7f6c08061b80 con 0x7f6bf40a2e20 2024-04-25T00:44:40.908 INFO:teuthology.orchestra.run.smithi032.stdout:/ 1m 6m3h 2024-04-25T00:44:40.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.909+0000 7f6c0f8ca640 1 -- 172.21.15.32:0/3935485173 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f6bf40a2e20 msgr2=0x7f6bf40a52e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:40.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.909+0000 7f6c0f8ca640 1 --2- 172.21.15.32:0/3935485173 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f6bf40a2e20 0x7f6bf40a52e0 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f6c04009230 tx=0x7f6c040058d0 comp rx=0 tx=0).stop 2024-04-25T00:44:40.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.909+0000 7f6c0f8ca640 1 -- 172.21.15.32:0/3935485173 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6c08074430 msgr2=0x7f6c0819a760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:40.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.909+0000 7f6c0f8ca640 1 --2- 172.21.15.32:0/3935485173 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6c08074430 0x7f6c0819a760 secure :-1 s=READY pgs=648 cs=0 l=1 rev1=1 crypto rx=0x7f6bf800b4a0 tx=0x7f6bf800b960 comp rx=0 tx=0).stop 2024-04-25T00:44:40.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.909+0000 7f6c0f8ca640 1 -- 172.21.15.32:0/3935485173 shutdown_connections 2024-04-25T00:44:40.910 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.909+0000 7f6c0f8ca640 1 --2- 172.21.15.32:0/3935485173 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f6bf40a2e20 0x7f6bf40a52e0 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:40.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.909+0000 7f6c0f8ca640 1 --2- 172.21.15.32:0/3935485173 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f6c0810dfe0 0x7f6c0819eae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:40.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.909+0000 7f6c0f8ca640 1 --2- 172.21.15.32:0/3935485173 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f6c08074430 0x7f6c0819a760 unknown :-1 s=CLOSED pgs=648 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:40.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.909+0000 7f6c0f8ca640 1 --2- 172.21.15.32:0/3935485173 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6c08073b10 0x7f6c0819a220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:40.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.909+0000 7f6c0f8ca640 1 -- 172.21.15.32:0/3935485173 >> 172.21.15.32:0/3935485173 conn(0x7f6c080fd210 msgr2=0x7f6c080fdee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:44:40.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.910+0000 7f6c0f8ca640 1 -- 172.21.15.32:0/3935485173 shutdown_connections 2024-04-25T00:44:40.911 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:40.910+0000 7f6c0f8ca640 1 -- 172.21.15.32:0/3935485173 wait complete. 2024-04-25T00:44:40.921 DEBUG:teuthology.run_tasks:Unwinding manager fwd_scrub 2024-04-25T00:44:40.932 INFO:tasks.fwd_scrub:joining ForwardScrubbers 2024-04-25T00:44:40.932 ERROR:teuthology.run_tasks:Manager failed: fwd_scrub Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_ceph_d2c45d977142e024a59b68a8f1dce8f0b9840551/qa/tasks/fwd_scrub.py", line 164, in task stop_all_fwd_scrubbers(ctx.ceph[config['cluster']].thrashers) File "/home/teuthworker/src/git.ceph.com_ceph_d2c45d977142e024a59b68a8f1dce8f0b9840551/qa/tasks/fwd_scrub.py", line 99, in stop_all_fwd_scrubbers raise RuntimeError(f"error during scrub thrashing: {thrasher.exception}") RuntimeError: error during scrub thrashing: rank damage found: {'backtrace'} 2024-04-25T00:44:40.933 DEBUG:teuthology.run_tasks:Unwinding manager check-counter 2024-04-25T00:44:40.943 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-25T00:44:41.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.045+0000 7f173fa12640 1 -- 172.21.15.32:0/1731632218 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1738102a20 msgr2=0x7f1738102e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:41.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.045+0000 7f173fa12640 1 --2- 172.21.15.32:0/1731632218 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1738102a20 0x7f1738102e00 secure :-1 s=READY pgs=649 cs=0 l=1 rev1=1 crypto rx=0x7f1734009900 tx=0x7f173402a480 comp rx=0 tx=0).stop 2024-04-25T00:44:41.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.045+0000 7f173fa12640 1 -- 172.21.15.32:0/1731632218 shutdown_connections 2024-04-25T00:44:41.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.045+0000 7f173fa12640 1 --2- 172.21.15.32:0/1731632218 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f173810bd60 0x7f173810e1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.045+0000 7f173fa12640 1 --2- 172.21.15.32:0/1731632218 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1738103340 0x7f173810b820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.045+0000 7f173fa12640 1 --2- 172.21.15.32:0/1731632218 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1738102a20 0x7f1738102e00 unknown :-1 s=CLOSED pgs=649 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.048 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.045+0000 7f173fa12640 1 -- 172.21.15.32:0/1731632218 >> 172.21.15.32:0/1731632218 conn(0x7f17380777a0 msgr2=0x7f1738077bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:44:41.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.045+0000 7f173fa12640 1 -- 172.21.15.32:0/1731632218 shutdown_connections 2024-04-25T00:44:41.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.045+0000 7f173fa12640 1 -- 172.21.15.32:0/1731632218 wait complete. 2024-04-25T00:44:41.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.046+0000 7f173fa12640 1 Processor -- start 2024-04-25T00:44:41.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.046+0000 7f173fa12640 1 -- start start 2024-04-25T00:44:41.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.046+0000 7f173fa12640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1738102a20 0x7f17381980b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:41.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.046+0000 7f173fa12640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1738103340 0x7f17381985f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:41.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.046+0000 7f173fa12640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f173810bd60 0x7f173819c920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:41.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.046+0000 7f173fa12640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f1738111b70 con 0x7f1738103340 2024-04-25T00:44:41.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.046+0000 7f173fa12640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f17381119f0 con 0x7f173810bd60 2024-04-25T00:44:41.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.046+0000 7f173fa12640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f1738111cf0 con 0x7f1738102a20 2024-04-25T00:44:41.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.046+0000 7f173cfad640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1738103340 0x7f17381985f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:41.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.046+0000 7f173cfad640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1738103340 0x7f17381985f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:38390/0 (socket says 172.21.15.32:38390) 2024-04-25T00:44:41.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.046+0000 7f173cfad640 1 -- 172.21.15.32:0/3733129847 learned_addr learned my addr 172.21.15.32:0/3733129847 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:44:41.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.046+0000 7f173d7ae640 1 --2- 172.21.15.32:0/3733129847 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1738102a20 0x7f17381980b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:41.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.046+0000 7f173cfad640 1 -- 172.21.15.32:0/3733129847 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1738102a20 msgr2=0x7f17381980b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:41.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.046+0000 7f173dfaf640 1 --2- 172.21.15.32:0/3733129847 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f173810bd60 0x7f173819c920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:41.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.046+0000 7f173cfad640 1 --2- 172.21.15.32:0/3733129847 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1738102a20 0x7f17381980b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.046+0000 7f173cfad640 1 -- 172.21.15.32:0/3733129847 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f173810bd60 msgr2=0x7f173819c920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:41.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.046+0000 7f173cfad640 1 --2- 172.21.15.32:0/3733129847 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f173810bd60 0x7f173819c920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.046+0000 7f173cfad640 1 -- 172.21.15.32:0/3733129847 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f173819d050 con 0x7f1738103340 2024-04-25T00:44:41.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.046+0000 7f173dfaf640 1 --2- 172.21.15.32:0/3733129847 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f173810bd60 0x7f173819c920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:44:41.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.047+0000 7f173cfad640 1 --2- 172.21.15.32:0/3733129847 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1738103340 0x7f17381985f0 secure :-1 s=READY pgs=650 cs=0 l=1 rev1=1 crypto rx=0x7f172c00d950 tx=0x7f172c00de10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:44:41.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.047+0000 7f172a7fc640 1 -- 172.21.15.32:0/3733129847 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f172c017070 con 0x7f1738103340 2024-04-25T00:44:41.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.047+0000 7f172a7fc640 1 -- 172.21.15.32:0/3733129847 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f172c009c30 con 0x7f1738103340 2024-04-25T00:44:41.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.047+0000 7f173fa12640 1 -- 172.21.15.32:0/3733129847 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f173819d340 con 0x7f1738103340 2024-04-25T00:44:41.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.047+0000 7f172a7fc640 1 -- 172.21.15.32:0/3733129847 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f172c00f6b0 con 0x7f1738103340 2024-04-25T00:44:41.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.047+0000 7f173fa12640 1 -- 172.21.15.32:0/3733129847 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f17381a4bd0 con 0x7f1738103340 2024-04-25T00:44:41.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.048+0000 7f173fa12640 1 -- 172.21.15.32:0/3733129847 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f17380689b0 con 0x7f1738103340 2024-04-25T00:44:41.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.049+0000 7f172a7fc640 1 -- 172.21.15.32:0/3733129847 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f172c0109d0 con 0x7f1738103340 2024-04-25T00:44:41.052 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.049+0000 7f172a7fc640 1 --2- 172.21.15.32:0/3733129847 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f17240a2d00 0x7f17240a51c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:41.052 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.050+0000 7f172a7fc640 1 -- 172.21.15.32:0/3733129847 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(105..105 src has 1..105) ==== 9510+0+0 (secure 0 0 0) 0x7f172c0cf500 con 0x7f1738103340 2024-04-25T00:44:41.052 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.050+0000 7f173d7ae640 1 --2- 172.21.15.32:0/3733129847 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f17240a2d00 0x7f17240a51c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:41.052 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.050+0000 7f173d7ae640 1 --2- 172.21.15.32:0/3733129847 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f17240a2d00 0x7f17240a51c0 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f17340096c0 tx=0x7f1734000bc0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:44:41.053 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.053+0000 7f172a7fc640 1 -- 172.21.15.32:0/3733129847 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f172c009de0 con 0x7f1738103340 2024-04-25T00:44:41.240 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.239+0000 7f173fa12640 1 -- 172.21.15.32:0/3733129847 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f17381993b0 con 0x7f1738103340 2024-04-25T00:44:41.241 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.241+0000 7f172a7fc640 1 -- 172.21.15.32:0/3733129847 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v105) ==== 74+0+25337 (secure 0 0 0) 0x7f17381993b0 con 0x7f1738103340 2024-04-25T00:44:41.242 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:44:41.242 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":105,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:44:21.996063+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-25T00:30:46.239556+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-25T00:30:47.257182+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"103","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":17,"snap_epoch":103,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2024-04-25T00:30:51.193975+0000","flags":8197,"flags_names":"hashpspool,ec_overwrites,selfmanaged_snaps","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"103","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":17,"snap_epoch":103,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.b","mappings":[{"from":8,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":1}]},{"pgid":"2.f","mappings":[{"from":8,"to":11}]},{"pgid":"2.10","mappings":[{"from":2,"to":4}]},{"pgid":"3.6","mappings":[{"from":3,"to":6}]},{"pgid":"3.17","mappings":[{"from":4,"to":7}]},{"pgid":"3.1b","mappings":[{"from":5,"to":9}]},{"pgid":"3.20","mappings":[{"from":5,"to":9}]},{"pgid":"3.23","mappings":[{"from":4,"to":8}]},{"pgid":"3.2c","mappings":[{"from":4,"to":9}]},{"pgid":"4.c","mappings":[{"from":9,"to":10}]},{"pgid":"4.12","mappings":[{"from":9,"to":10}]},{"pgid":"4.1c","mappings":[{"from":9,"to":8}]},{"pgid":"4.1d","mappings":[{"from":5,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:6801/585810931":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/1096527590":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/1767429535":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/3915353181":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6800/585810931":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/1694990206":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-num-failure-domains":"0","crush-osds-per-failure-domain":"0","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":4,"snaps":[{"begin":6,"length":1},{"begin":17,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:44:41.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.243+0000 7f173fa12640 1 -- 172.21.15.32:0/3733129847 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f17240a2d00 msgr2=0x7f17240a51c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:41.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.243+0000 7f173fa12640 1 --2- 172.21.15.32:0/3733129847 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f17240a2d00 0x7f17240a51c0 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f17340096c0 tx=0x7f1734000bc0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.243+0000 7f173fa12640 1 -- 172.21.15.32:0/3733129847 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1738103340 msgr2=0x7f17381985f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:41.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.243+0000 7f173fa12640 1 --2- 172.21.15.32:0/3733129847 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1738103340 0x7f17381985f0 secure :-1 s=READY pgs=650 cs=0 l=1 rev1=1 crypto rx=0x7f172c00d950 tx=0x7f172c00de10 comp rx=0 tx=0).stop 2024-04-25T00:44:41.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.243+0000 7f173fa12640 1 -- 172.21.15.32:0/3733129847 shutdown_connections 2024-04-25T00:44:41.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.243+0000 7f173fa12640 1 --2- 172.21.15.32:0/3733129847 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f17240a2d00 0x7f17240a51c0 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.243+0000 7f173fa12640 1 --2- 172.21.15.32:0/3733129847 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f173810bd60 0x7f173819c920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.243+0000 7f173fa12640 1 --2- 172.21.15.32:0/3733129847 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1738103340 0x7f17381985f0 unknown :-1 s=CLOSED pgs=650 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.243+0000 7f173fa12640 1 --2- 172.21.15.32:0/3733129847 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1738102a20 0x7f17381980b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.244 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.243+0000 7f173fa12640 1 -- 172.21.15.32:0/3733129847 >> 172.21.15.32:0/3733129847 conn(0x7f17380777a0 msgr2=0x7f17380fd760 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:44:41.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.243+0000 7f173fa12640 1 -- 172.21.15.32:0/3733129847 shutdown_connections 2024-04-25T00:44:41.245 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.243+0000 7f173fa12640 1 -- 172.21.15.32:0/3733129847 wait complete. 2024-04-25T00:44:41.255 INFO:tasks.check_counter.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-25T00:26:07.306839+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-04-25T00:30:46.239556+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '84', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.8799999952316284, 'score_stable': 1.8799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.8799999952316284, 'raw_score_stable': 1.8799999952316284, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-04-25T00:30:47.257182+0000', 'flags': 8193, 'flags_names': 'hashpspool,selfmanaged_snaps', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '103', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 17, 'snap_epoch': 103, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.05000000074505806}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_data_ec', 'create_time': '2024-04-25T00:30:51.193975+0000', 'flags': 8197, 'flags_names': 'hashpspool,ec_overwrites,selfmanaged_snaps', 'type': 3, 'size': 4, 'min_size': 3, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '103', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 17, 'snap_epoch': 103, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'cephfs_data_ec', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 8192, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}] 2024-04-25T00:44:41.256 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-25T00:44:41.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.360+0000 7fc5474da640 1 -- 172.21.15.32:0/3584598365 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc540107730 msgr2=0x7fc540107b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:41.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.360+0000 7fc5474da640 1 --2- 172.21.15.32:0/3584598365 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc540107730 0x7fc540107b10 secure :-1 s=READY pgs=651 cs=0 l=1 rev1=1 crypto rx=0x7fc53c009930 tx=0x7fc53c02a420 comp rx=0 tx=0).stop 2024-04-25T00:44:41.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.360+0000 7fc5474da640 1 -- 172.21.15.32:0/3584598365 shutdown_connections 2024-04-25T00:44:41.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.360+0000 7fc5474da640 1 --2- 172.21.15.32:0/3584598365 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc54006a780 0x7fc54010e5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.360+0000 7fc5474da640 1 --2- 172.21.15.32:0/3584598365 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc540069dc0 0x7fc54006a240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.360+0000 7fc5474da640 1 --2- 172.21.15.32:0/3584598365 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc540107730 0x7fc540107b10 unknown :-1 s=CLOSED pgs=651 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.360+0000 7fc5474da640 1 -- 172.21.15.32:0/3584598365 >> 172.21.15.32:0/3584598365 conn(0x7fc5400776a0 msgr2=0x7fc540077ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:44:41.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.360+0000 7fc5474da640 1 -- 172.21.15.32:0/3584598365 shutdown_connections 2024-04-25T00:44:41.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.361+0000 7fc5474da640 1 -- 172.21.15.32:0/3584598365 wait complete. 2024-04-25T00:44:41.363 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.361+0000 7fc5474da640 1 Processor -- start 2024-04-25T00:44:41.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.361+0000 7fc5474da640 1 -- start start 2024-04-25T00:44:41.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.361+0000 7fc5474da640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc540069dc0 0x7fc540198000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:41.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.361+0000 7fc5474da640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc54006a780 0x7fc540198540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:41.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.361+0000 7fc5474da640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc540107730 0x7fc54019c8c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:41.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.361+0000 7fc5474da640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fc540111b40 con 0x7fc540107730 2024-04-25T00:44:41.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.361+0000 7fc5474da640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fc5401119c0 con 0x7fc540069dc0 2024-04-25T00:44:41.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.361+0000 7fc5474da640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fc540111cc0 con 0x7fc54006a780 2024-04-25T00:44:41.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.361+0000 7fc545a77640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc540107730 0x7fc54019c8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:41.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.361+0000 7fc545a77640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc540107730 0x7fc54019c8c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:38412/0 (socket says 172.21.15.32:38412) 2024-04-25T00:44:41.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.361+0000 7fc545a77640 1 -- 172.21.15.32:0/1191696468 learned_addr learned my addr 172.21.15.32:0/1191696468 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:44:41.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.361+0000 7fc545276640 1 --2- 172.21.15.32:0/1191696468 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc540069dc0 0x7fc540198000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:41.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.362+0000 7fc544a75640 1 --2- 172.21.15.32:0/1191696468 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc54006a780 0x7fc540198540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:41.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.362+0000 7fc545a77640 1 -- 172.21.15.32:0/1191696468 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc54006a780 msgr2=0x7fc540198540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:41.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.362+0000 7fc545a77640 1 --2- 172.21.15.32:0/1191696468 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc54006a780 0x7fc540198540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.362+0000 7fc545a77640 1 -- 172.21.15.32:0/1191696468 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc540069dc0 msgr2=0x7fc540198000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:41.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.362+0000 7fc545a77640 1 --2- 172.21.15.32:0/1191696468 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc540069dc0 0x7fc540198000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.362+0000 7fc545a77640 1 -- 172.21.15.32:0/1191696468 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc54019cff0 con 0x7fc540107730 2024-04-25T00:44:41.365 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.362+0000 7fc545a77640 1 --2- 172.21.15.32:0/1191696468 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc540107730 0x7fc54019c8c0 secure :-1 s=READY pgs=652 cs=0 l=1 rev1=1 crypto rx=0x7fc53800cce0 tx=0x7fc53800a620 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:44:41.366 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.365+0000 7fc5367fc640 1 -- 172.21.15.32:0/1191696468 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc53800f450 con 0x7fc540107730 2024-04-25T00:44:41.366 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.365+0000 7fc5474da640 1 -- 172.21.15.32:0/1191696468 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc54019d2e0 con 0x7fc540107730 2024-04-25T00:44:41.366 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.365+0000 7fc5367fc640 1 -- 172.21.15.32:0/1191696468 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc53800fad0 con 0x7fc540107730 2024-04-25T00:44:41.366 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.365+0000 7fc5367fc640 1 -- 172.21.15.32:0/1191696468 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc538018c10 con 0x7fc540107730 2024-04-25T00:44:41.366 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.365+0000 7fc5474da640 1 -- 172.21.15.32:0/1191696468 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc5401a4b70 con 0x7fc540107730 2024-04-25T00:44:41.367 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.366+0000 7fc5474da640 1 -- 172.21.15.32:0/1191696468 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc5400689b0 con 0x7fc540107730 2024-04-25T00:44:41.367 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.367+0000 7fc5367fc640 1 -- 172.21.15.32:0/1191696468 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7fc53800f5f0 con 0x7fc540107730 2024-04-25T00:44:41.368 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.368+0000 7fc5367fc640 1 --2- 172.21.15.32:0/1191696468 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fc52c0a2d00 0x7fc52c0a51c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:41.368 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.368+0000 7fc5367fc640 1 -- 172.21.15.32:0/1191696468 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(105..105 src has 1..105) ==== 9510+0+0 (secure 0 0 0) 0x7fc5380cff20 con 0x7fc540107730 2024-04-25T00:44:41.368 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.368+0000 7fc545276640 1 --2- 172.21.15.32:0/1191696468 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fc52c0a2d00 0x7fc52c0a51c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:41.369 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.369+0000 7fc545276640 1 --2- 172.21.15.32:0/1191696468 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fc52c0a2d00 0x7fc52c0a51c0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7fc53c009900 tx=0x7fc53c0058d0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:44:41.372 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.372+0000 7fc5367fc640 1 -- 172.21.15.32:0/1191696468 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fc538014030 con 0x7fc540107730 2024-04-25T00:44:41.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:41 smithi130 ceph-mon[32525]: pgmap v355: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 6.2 KiB/s rd, 255 B/s wr, 7 op/s 2024-04-25T00:44:41.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:41 smithi130 ceph-mon[32525]: from='client.16338 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:44:41.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:41 smithi130 ceph-mon[32525]: from='client.16344 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:44:41.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:41 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3733129847' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:44:41.726 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.725+0000 7fc5474da640 1 -- 172.21.15.32:0/1191696468 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7fc540109700 con 0x7fc540107730 2024-04-25T00:44:41.728 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.728+0000 7fc5367fc640 1 -- 172.21.15.32:0/1191696468 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v105) ==== 93+0+10 (secure 0 0 0) 0x7fc53809c760 con 0x7fc540107730 2024-04-25T00:44:41.728 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 1 2024-04-25T00:44:41.732 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.731+0000 7fc5474da640 1 -- 172.21.15.32:0/1191696468 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fc52c0a2d00 msgr2=0x7fc52c0a51c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:41.732 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.731+0000 7fc5474da640 1 --2- 172.21.15.32:0/1191696468 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fc52c0a2d00 0x7fc52c0a51c0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7fc53c009900 tx=0x7fc53c0058d0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.732 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.731+0000 7fc5474da640 1 -- 172.21.15.32:0/1191696468 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc540107730 msgr2=0x7fc54019c8c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:41.732 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.731+0000 7fc5474da640 1 --2- 172.21.15.32:0/1191696468 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc540107730 0x7fc54019c8c0 secure :-1 s=READY pgs=652 cs=0 l=1 rev1=1 crypto rx=0x7fc53800cce0 tx=0x7fc53800a620 comp rx=0 tx=0).stop 2024-04-25T00:44:41.732 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.731+0000 7fc5474da640 1 -- 172.21.15.32:0/1191696468 shutdown_connections 2024-04-25T00:44:41.732 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.731+0000 7fc5474da640 1 --2- 172.21.15.32:0/1191696468 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fc52c0a2d00 0x7fc52c0a51c0 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.732 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.731+0000 7fc5474da640 1 --2- 172.21.15.32:0/1191696468 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc540107730 0x7fc54019c8c0 unknown :-1 s=CLOSED pgs=652 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.732 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.731+0000 7fc5474da640 1 --2- 172.21.15.32:0/1191696468 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc54006a780 0x7fc540198540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.732 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.731+0000 7fc5474da640 1 --2- 172.21.15.32:0/1191696468 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc540069dc0 0x7fc540198000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.732 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.732+0000 7fc5474da640 1 -- 172.21.15.32:0/1191696468 >> 172.21.15.32:0/1191696468 conn(0x7fc5400776a0 msgr2=0x7fc5400fe5b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:44:41.732 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.732+0000 7fc5474da640 1 -- 172.21.15.32:0/1191696468 shutdown_connections 2024-04-25T00:44:41.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.732+0000 7fc5474da640 1 -- 172.21.15.32:0/1191696468 wait complete. 2024-04-25T00:44:41.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:41 smithi032 ceph-mon[29357]: pgmap v355: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 6.2 KiB/s rd, 255 B/s wr, 7 op/s 2024-04-25T00:44:41.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:41 smithi032 ceph-mon[29357]: from='client.16338 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:44:41.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:41 smithi032 ceph-mon[29357]: from='client.16344 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:44:41.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:41 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3733129847' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:44:41.751 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-04-25T00:44:41.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:41 smithi121 ceph-mon[32231]: pgmap v355: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 6.2 KiB/s rd, 255 B/s wr, 7 op/s 2024-04-25T00:44:41.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:41 smithi121 ceph-mon[32231]: from='client.16338 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:44:41.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:41 smithi121 ceph-mon[32231]: from='client.16344 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T00:44:41.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:41 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3733129847' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:44:41.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.936+0000 7ff345f7e640 1 -- 172.21.15.32:0/1899726895 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff34010dfc0 msgr2=0x7ff34010e440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:41.937 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.936+0000 7ff345f7e640 1 --2- 172.21.15.32:0/1899726895 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff34010dfc0 0x7ff34010e440 secure :-1 s=READY pgs=653 cs=0 l=1 rev1=1 crypto rx=0x7ff338009cd0 tx=0x7ff33802a930 comp rx=0 tx=0).stop 2024-04-25T00:44:41.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.938+0000 7ff345f7e640 1 -- 172.21.15.32:0/1899726895 shutdown_connections 2024-04-25T00:44:41.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.938+0000 7ff345f7e640 1 --2- 172.21.15.32:0/1899726895 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff34011af20 0x7ff34011d310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.939+0000 7ff345f7e640 1 --2- 172.21.15.32:0/1899726895 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff34010dfc0 0x7ff34010e440 unknown :-1 s=CLOSED pgs=653 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.939+0000 7ff345f7e640 1 --2- 172.21.15.32:0/1899726895 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff34010ff90 0x7ff34010da80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.939 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.939+0000 7ff345f7e640 1 -- 172.21.15.32:0/1899726895 >> 172.21.15.32:0/1899726895 conn(0x7ff34006c850 msgr2=0x7ff34006cc60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:44:41.942 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.942+0000 7ff345f7e640 1 -- 172.21.15.32:0/1899726895 shutdown_connections 2024-04-25T00:44:41.944 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.944+0000 7ff345f7e640 1 -- 172.21.15.32:0/1899726895 wait complete. 2024-04-25T00:44:41.947 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.945+0000 7ff345f7e640 1 Processor -- start 2024-04-25T00:44:41.948 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.948+0000 7ff345f7e640 1 -- start start 2024-04-25T00:44:41.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.948+0000 7ff345f7e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff34010dfc0 0x7ff3401a6fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:41.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.948+0000 7ff345f7e640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff34010ff90 0x7ff3401a7500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:41.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.948+0000 7ff345f7e640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff34011af20 0x7ff3401ab880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:41.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.948+0000 7ff345f7e640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7ff3401209c0 con 0x7ff34010dfc0 2024-04-25T00:44:41.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.948+0000 7ff345f7e640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7ff340120840 con 0x7ff34011af20 2024-04-25T00:44:41.949 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.948+0000 7ff345f7e640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7ff340120b40 con 0x7ff34010ff90 2024-04-25T00:44:41.951 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.951+0000 7ff33ffff640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff34011af20 0x7ff3401ab880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:41.951 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.951+0000 7ff33ffff640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff34011af20 0x7ff3401ab880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:39126/0 (socket says 172.21.15.32:39126) 2024-04-25T00:44:41.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.951+0000 7ff33ffff640 1 -- 172.21.15.32:0/3422064912 learned_addr learned my addr 172.21.15.32:0/3422064912 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:44:41.952 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.951+0000 7ff33effd640 1 --2- 172.21.15.32:0/3422064912 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff34010ff90 0x7ff3401a7500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:41.953 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.952+0000 7ff33f7fe640 1 --2- 172.21.15.32:0/3422064912 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff34010dfc0 0x7ff3401a6fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:41.953 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.952+0000 7ff33f7fe640 1 -- 172.21.15.32:0/3422064912 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff34010ff90 msgr2=0x7ff3401a7500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:41.953 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.952+0000 7ff33f7fe640 1 --2- 172.21.15.32:0/3422064912 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff34010ff90 0x7ff3401a7500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.953 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.952+0000 7ff33f7fe640 1 -- 172.21.15.32:0/3422064912 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff34011af20 msgr2=0x7ff3401ab880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:41.953 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.952+0000 7ff33f7fe640 1 --2- 172.21.15.32:0/3422064912 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff34011af20 0x7ff3401ab880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:41.953 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.952+0000 7ff33f7fe640 1 -- 172.21.15.32:0/3422064912 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff3401abfb0 con 0x7ff34010dfc0 2024-04-25T00:44:41.957 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.953+0000 7ff33f7fe640 1 --2- 172.21.15.32:0/3422064912 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff34010dfc0 0x7ff3401a6fc0 secure :-1 s=READY pgs=654 cs=0 l=1 rev1=1 crypto rx=0x7ff334009fd0 tx=0x7ff33400b870 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:44:41.960 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.958+0000 7ff33cff9640 1 -- 172.21.15.32:0/3422064912 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff334015020 con 0x7ff34010dfc0 2024-04-25T00:44:41.960 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.958+0000 7ff345f7e640 1 -- 172.21.15.32:0/3422064912 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff3401ac2a0 con 0x7ff34010dfc0 2024-04-25T00:44:41.960 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.958+0000 7ff345f7e640 1 -- 172.21.15.32:0/3422064912 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff3401b3b30 con 0x7ff34010dfc0 2024-04-25T00:44:41.960 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.958+0000 7ff33cff9640 1 -- 172.21.15.32:0/3422064912 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff334011d10 con 0x7ff34010dfc0 2024-04-25T00:44:41.960 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.958+0000 7ff33cff9640 1 -- 172.21.15.32:0/3422064912 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff33400f2e0 con 0x7ff34010dfc0 2024-04-25T00:44:41.966 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.963+0000 7ff345f7e640 1 -- 172.21.15.32:0/3422064912 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff32c005290 con 0x7ff34010dfc0 2024-04-25T00:44:41.969 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.969+0000 7ff33cff9640 1 -- 172.21.15.32:0/3422064912 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7ff334018020 con 0x7ff34010dfc0 2024-04-25T00:44:41.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.970+0000 7ff33cff9640 1 --2- 172.21.15.32:0/3422064912 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7ff3280a2ed0 0x7ff3280a5390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:41.970 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.970+0000 7ff33cff9640 1 -- 172.21.15.32:0/3422064912 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(105..105 src has 1..105) ==== 9510+0+0 (secure 0 0 0) 0x7ff3340cec00 con 0x7ff34010dfc0 2024-04-25T00:44:41.973 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.973+0000 7ff33effd640 1 --2- 172.21.15.32:0/3422064912 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7ff3280a2ed0 0x7ff3280a5390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:41.975 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.975+0000 7ff33cff9640 1 -- 172.21.15.32:0/3422064912 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7ff33409b4f0 con 0x7ff34010dfc0 2024-04-25T00:44:41.976 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:41.976+0000 7ff33effd640 1 --2- 172.21.15.32:0/3422064912 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7ff3280a2ed0 0x7ff3280a5390 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7ff3380098d0 tx=0x7ff338009900 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:44:42.350 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.349+0000 7ff345f7e640 1 -- 172.21.15.32:0/3422064912 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) -- 0x7ff32c005850 con 0x7ff34010dfc0 2024-04-25T00:44:42.352 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.351+0000 7ff33cff9640 1 -- 172.21.15.32:0/3422064912 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v105) ==== 104+0+11 (secure 0 0 0) 0x7ff33409adf0 con 0x7ff34010dfc0 2024-04-25T00:44:42.352 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 32 2024-04-25T00:44:42.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.355+0000 7ff345f7e640 1 -- 172.21.15.32:0/3422064912 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7ff3280a2ed0 msgr2=0x7ff3280a5390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:42.355 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.355+0000 7ff345f7e640 1 --2- 172.21.15.32:0/3422064912 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7ff3280a2ed0 0x7ff3280a5390 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7ff3380098d0 tx=0x7ff338009900 comp rx=0 tx=0).stop 2024-04-25T00:44:42.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.355+0000 7ff345f7e640 1 -- 172.21.15.32:0/3422064912 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff34010dfc0 msgr2=0x7ff3401a6fc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:42.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.355+0000 7ff345f7e640 1 --2- 172.21.15.32:0/3422064912 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff34010dfc0 0x7ff3401a6fc0 secure :-1 s=READY pgs=654 cs=0 l=1 rev1=1 crypto rx=0x7ff334009fd0 tx=0x7ff33400b870 comp rx=0 tx=0).stop 2024-04-25T00:44:42.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.355+0000 7ff345f7e640 1 -- 172.21.15.32:0/3422064912 shutdown_connections 2024-04-25T00:44:42.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.355+0000 7ff345f7e640 1 --2- 172.21.15.32:0/3422064912 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7ff3280a2ed0 0x7ff3280a5390 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:42.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.355+0000 7ff345f7e640 1 --2- 172.21.15.32:0/3422064912 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7ff34011af20 0x7ff3401ab880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:42.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.355+0000 7ff345f7e640 1 --2- 172.21.15.32:0/3422064912 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff34010ff90 0x7ff3401a7500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:42.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.355+0000 7ff345f7e640 1 --2- 172.21.15.32:0/3422064912 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7ff34010dfc0 0x7ff3401a6fc0 unknown :-1 s=CLOSED pgs=654 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:42.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.355+0000 7ff345f7e640 1 -- 172.21.15.32:0/3422064912 >> 172.21.15.32:0/3422064912 conn(0x7ff34006c850 msgr2=0x7ff340119a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:44:42.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.355+0000 7ff345f7e640 1 -- 172.21.15.32:0/3422064912 shutdown_connections 2024-04-25T00:44:42.356 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.355+0000 7ff345f7e640 1 -- 172.21.15.32:0/3422064912 wait complete. 2024-04-25T00:44:42.374 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-04-25T00:44:42.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.557+0000 7f9a50bdf640 1 -- 172.21.15.32:0/1001229304 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9a440957f0 msgr2=0x7f9a440a1ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:42.557 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.557+0000 7f9a50bdf640 1 --2- 172.21.15.32:0/1001229304 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9a440957f0 0x7f9a440a1ac0 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7f9a4c067440 tx=0x7f9a4c0960a0 comp rx=0 tx=0).stop 2024-04-25T00:44:42.561 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.561+0000 7f9a50bdf640 1 -- 172.21.15.32:0/1001229304 shutdown_connections 2024-04-25T00:44:42.561 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.561+0000 7f9a50bdf640 1 --2- 172.21.15.32:0/1001229304 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9a440957f0 0x7f9a440a1ac0 unknown :-1 s=CLOSED pgs=137 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:42.561 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.561+0000 7f9a50bdf640 1 --2- 172.21.15.32:0/1001229304 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9a44094e50 0x7f9a440952b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:42.561 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.561+0000 7f9a50bdf640 1 --2- 172.21.15.32:0/1001229304 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9a44099bc0 0x7f9a44099fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:42.561 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.561+0000 7f9a50bdf640 1 -- 172.21.15.32:0/1001229304 >> 172.21.15.32:0/1001229304 conn(0x7f9a4400a740 msgr2=0x7f9a4400ab50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:44:42.561 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.561+0000 7f9a50bdf640 1 -- 172.21.15.32:0/1001229304 shutdown_connections 2024-04-25T00:44:42.562 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.561+0000 7f9a50bdf640 1 -- 172.21.15.32:0/1001229304 wait complete. 2024-04-25T00:44:42.562 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.562+0000 7f9a50bdf640 1 Processor -- start 2024-04-25T00:44:42.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.566+0000 7f9a50bdf640 1 -- start start 2024-04-25T00:44:42.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.566+0000 7f9a50bdf640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9a44094e50 0x7f9a4413c100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:42.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.566+0000 7f9a50bdf640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9a440957f0 0x7f9a4413c640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:42.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.566+0000 7f9a50bdf640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9a44099bc0 0x7f9a44141150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:42.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.566+0000 7f9a50bdf640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f9a440a4f60 con 0x7f9a44099bc0 2024-04-25T00:44:42.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.566+0000 7f9a50bdf640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f9a440a4de0 con 0x7f9a44094e50 2024-04-25T00:44:42.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.566+0000 7f9a50bdf640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f9a440a50e0 con 0x7f9a440957f0 2024-04-25T00:44:42.567 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.567+0000 7f9a4affd640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9a440957f0 0x7f9a4413c640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:42.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.567+0000 7f9a4affd640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9a440957f0 0x7f9a4413c640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.32:36372/0 (socket says 172.21.15.32:36372) 2024-04-25T00:44:42.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.567+0000 7f9a4affd640 1 -- 172.21.15.32:0/2105643318 learned_addr learned my addr 172.21.15.32:0/2105643318 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:44:42.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.567+0000 7f9a4b7fe640 1 --2- 172.21.15.32:0/2105643318 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9a44094e50 0x7f9a4413c100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:42.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.567+0000 7f9a4bfff640 1 --2- 172.21.15.32:0/2105643318 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9a44099bc0 0x7f9a44141150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:42.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.568+0000 7f9a4bfff640 1 -- 172.21.15.32:0/2105643318 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9a440957f0 msgr2=0x7f9a4413c640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:42.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.568+0000 7f9a4bfff640 1 --2- 172.21.15.32:0/2105643318 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9a440957f0 0x7f9a4413c640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:42.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.568+0000 7f9a4bfff640 1 -- 172.21.15.32:0/2105643318 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9a44094e50 msgr2=0x7f9a4413c100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:42.568 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.568+0000 7f9a4bfff640 1 --2- 172.21.15.32:0/2105643318 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9a44094e50 0x7f9a4413c100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:42.569 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.568+0000 7f9a4bfff640 1 -- 172.21.15.32:0/2105643318 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9a44141690 con 0x7f9a44099bc0 2024-04-25T00:44:42.572 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.572+0000 7f9a4bfff640 1 --2- 172.21.15.32:0/2105643318 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9a44099bc0 0x7f9a44141150 secure :-1 s=READY pgs=655 cs=0 l=1 rev1=1 crypto rx=0x7f9a4c066f10 tx=0x7f9a4c09e620 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:44:42.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.580+0000 7f9a48ff9640 1 -- 172.21.15.32:0/2105643318 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9a4c09e790 con 0x7f9a44099bc0 2024-04-25T00:44:42.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.580+0000 7f9a48ff9640 1 -- 172.21.15.32:0/2105643318 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9a4c09ee10 con 0x7f9a44099bc0 2024-04-25T00:44:42.580 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.580+0000 7f9a48ff9640 1 -- 172.21.15.32:0/2105643318 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9a4c09d390 con 0x7f9a44099bc0 2024-04-25T00:44:42.587 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.581+0000 7f9a50bdf640 1 -- 172.21.15.32:0/2105643318 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9a44141920 con 0x7f9a44099bc0 2024-04-25T00:44:42.587 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.584+0000 7f9a50bdf640 1 -- 172.21.15.32:0/2105643318 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9a44141df0 con 0x7f9a44099bc0 2024-04-25T00:44:42.589 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.587+0000 7f9a50bdf640 1 -- 172.21.15.32:0/2105643318 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9a440962f0 con 0x7f9a44099bc0 2024-04-25T00:44:42.596 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.591+0000 7f9a48ff9640 1 -- 172.21.15.32:0/2105643318 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f9a4c072690 con 0x7f9a44099bc0 2024-04-25T00:44:42.596 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.591+0000 7f9a48ff9640 1 --2- 172.21.15.32:0/2105643318 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f9a3c0a2f00 0x7f9a3c0a53c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:42.596 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.592+0000 7f9a48ff9640 1 -- 172.21.15.32:0/2105643318 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(105..105 src has 1..105) ==== 9510+0+0 (secure 0 0 0) 0x7f9a4c153f70 con 0x7f9a44099bc0 2024-04-25T00:44:42.596 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.593+0000 7f9a4b7fe640 1 --2- 172.21.15.32:0/2105643318 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f9a3c0a2f00 0x7f9a3c0a53c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:42.600 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.600+0000 7f9a4b7fe640 1 --2- 172.21.15.32:0/2105643318 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f9a3c0a2f00 0x7f9a3c0a53c0 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f9a4000bb70 tx=0x7f9a4000b2e0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:44:42.602 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.601+0000 7f9a48ff9640 1 -- 172.21.15.32:0/2105643318 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f9a4c06d070 con 0x7f9a44099bc0 2024-04-25T00:44:42.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:42 smithi130 ceph-mon[32525]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:44:42.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:42 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1191696468' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:44:42.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:42 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3422064912' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:44:42.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:42 smithi032 ceph-mon[29357]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:44:42.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:42 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1191696468' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:44:42.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:42 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3422064912' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:44:42.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:42 smithi121 ceph-mon[32231]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:44:42.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:42 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1191696468' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:44:42.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:42 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3422064912' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:44:42.983 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.982+0000 7f9a50bdf640 1 -- 172.21.15.32:0/2105643318 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) -- 0x7f9a4409d150 con 0x7f9a44099bc0 2024-04-25T00:44:42.984 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.984+0000 7f9a48ff9640 1 -- 172.21.15.32:0/2105643318 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v105) ==== 100+0+11 (secure 0 0 0) 0x7f9a4c120210 con 0x7f9a44099bc0 2024-04-25T00:44:42.987 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 64 2024-04-25T00:44:42.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.992+0000 7f9a50bdf640 1 -- 172.21.15.32:0/2105643318 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f9a3c0a2f00 msgr2=0x7f9a3c0a53c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:42.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.992+0000 7f9a50bdf640 1 --2- 172.21.15.32:0/2105643318 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f9a3c0a2f00 0x7f9a3c0a53c0 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f9a4000bb70 tx=0x7f9a4000b2e0 comp rx=0 tx=0).stop 2024-04-25T00:44:42.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.993+0000 7f9a50bdf640 1 -- 172.21.15.32:0/2105643318 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9a44099bc0 msgr2=0x7f9a44141150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:42.993 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.993+0000 7f9a50bdf640 1 --2- 172.21.15.32:0/2105643318 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9a44099bc0 0x7f9a44141150 secure :-1 s=READY pgs=655 cs=0 l=1 rev1=1 crypto rx=0x7f9a4c066f10 tx=0x7f9a4c09e620 comp rx=0 tx=0).stop 2024-04-25T00:44:42.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.996+0000 7f9a50bdf640 1 -- 172.21.15.32:0/2105643318 shutdown_connections 2024-04-25T00:44:42.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.996+0000 7f9a50bdf640 1 --2- 172.21.15.32:0/2105643318 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f9a3c0a2f00 0x7f9a3c0a53c0 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:42.996 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.996+0000 7f9a50bdf640 1 --2- 172.21.15.32:0/2105643318 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9a44099bc0 0x7f9a44141150 unknown :-1 s=CLOSED pgs=655 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:42.997 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.996+0000 7f9a50bdf640 1 --2- 172.21.15.32:0/2105643318 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9a440957f0 0x7f9a4413c640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:42.997 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.996+0000 7f9a50bdf640 1 --2- 172.21.15.32:0/2105643318 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9a44094e50 0x7f9a4413c100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:42.997 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.996+0000 7f9a50bdf640 1 -- 172.21.15.32:0/2105643318 >> 172.21.15.32:0/2105643318 conn(0x7f9a4400a740 msgr2=0x7f9a44092b60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:44:42.997 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.997+0000 7f9a50bdf640 1 -- 172.21.15.32:0/2105643318 shutdown_connections 2024-04-25T00:44:42.999 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:42.999+0000 7f9a50bdf640 1 -- 172.21.15.32:0/2105643318 wait complete. 2024-04-25T00:44:43.021 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data_ec pg_num 2024-04-25T00:44:43.243 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.241+0000 7f05fc829640 1 -- 172.21.15.32:0/2613181781 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f05f810e3d0 msgr2=0x7f05f811cf30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:43.243 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.241+0000 7f05fc829640 1 --2- 172.21.15.32:0/2613181781 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f05f810e3d0 0x7f05f811cf30 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7f05f0009900 tx=0x7f05f002a420 comp rx=0 tx=0).stop 2024-04-25T00:44:43.243 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.241+0000 7f05fc829640 1 -- 172.21.15.32:0/2613181781 shutdown_connections 2024-04-25T00:44:43.243 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.241+0000 7f05fc829640 1 --2- 172.21.15.32:0/2613181781 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f05f810e3d0 0x7f05f811cf30 unknown :-1 s=CLOSED pgs=137 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:43.243 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.241+0000 7f05fc829640 1 --2- 172.21.15.32:0/2613181781 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f05f810d820 0x7f05f810dca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:43.243 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.241+0000 7f05fc829640 1 --2- 172.21.15.32:0/2613181781 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f05f810fd50 0x7f05f8110130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:43.243 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.241+0000 7f05fc829640 1 -- 172.21.15.32:0/2613181781 >> 172.21.15.32:0/2613181781 conn(0x7f05f806c620 msgr2=0x7f05f806ca30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:44:43.243 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.241+0000 7f05fc829640 1 -- 172.21.15.32:0/2613181781 shutdown_connections 2024-04-25T00:44:43.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.242+0000 7f05fc829640 1 -- 172.21.15.32:0/2613181781 wait complete. 2024-04-25T00:44:43.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.243+0000 7f05fc829640 1 Processor -- start 2024-04-25T00:44:43.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.246+0000 7f05fc829640 1 -- start start 2024-04-25T00:44:43.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.246+0000 7f05fc829640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f05f810d820 0x7f05f811c460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:43.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.246+0000 7f05fc829640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f05f810e3d0 0x7f05f811c9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:43.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.246+0000 7f05fc829640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f05f810fd50 0x7f05f8115680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:43.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.246+0000 7f05fc829640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f05f81207a0 con 0x7f05f810e3d0 2024-04-25T00:44:43.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.246+0000 7f05fc829640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f05f8120620 con 0x7f05f810fd50 2024-04-25T00:44:43.247 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.246+0000 7f05fc829640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f05f8120920 con 0x7f05f810d820 2024-04-25T00:44:43.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.246+0000 7f05f759e640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f05f810fd50 0x7f05f8115680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:43.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.247+0000 7f05f759e640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f05f810fd50 0x7f05f8115680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.121:3300/0 says I am v2:172.21.15.32:39170/0 (socket says 172.21.15.32:39170) 2024-04-25T00:44:43.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.247+0000 7f05f759e640 1 -- 172.21.15.32:0/2786270928 learned_addr learned my addr 172.21.15.32:0/2786270928 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:44:43.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.247+0000 7f05f659c640 1 --2- 172.21.15.32:0/2786270928 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f05f810e3d0 0x7f05f811c9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:43.248 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.248+0000 7f05f6d9d640 1 --2- 172.21.15.32:0/2786270928 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f05f810d820 0x7f05f811c460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:43.249 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.249+0000 7f05f759e640 1 -- 172.21.15.32:0/2786270928 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f05f810d820 msgr2=0x7f05f811c460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:43.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.249+0000 7f05f759e640 1 --2- 172.21.15.32:0/2786270928 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f05f810d820 0x7f05f811c460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:43.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.249+0000 7f05f759e640 1 -- 172.21.15.32:0/2786270928 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f05f810e3d0 msgr2=0x7f05f811c9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:43.250 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.249+0000 7f05f759e640 1 --2- 172.21.15.32:0/2786270928 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f05f810e3d0 0x7f05f811c9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:43.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.249+0000 7f05f759e640 1 -- 172.21.15.32:0/2786270928 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f05f8115f40 con 0x7f05f810fd50 2024-04-25T00:44:43.251 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.249+0000 7f05f759e640 1 --2- 172.21.15.32:0/2786270928 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f05f810fd50 0x7f05f8115680 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f05f002a940 tx=0x7f05f0003b90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:44:43.254 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.254+0000 7f05effff640 1 -- 172.21.15.32:0/2786270928 <== mon.2 v2:172.21.15.121:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f05f003f070 con 0x7f05f810fd50 2024-04-25T00:44:43.254 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.254+0000 7f05effff640 1 -- 172.21.15.32:0/2786270928 <== mon.2 v2:172.21.15.121:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f05f0031c20 con 0x7f05f810fd50 2024-04-25T00:44:43.254 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.254+0000 7f05fc829640 1 -- 172.21.15.32:0/2786270928 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f05f81161d0 con 0x7f05f810fd50 2024-04-25T00:44:43.255 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.254+0000 7f05fc829640 1 -- 172.21.15.32:0/2786270928 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f05f811a160 con 0x7f05f810fd50 2024-04-25T00:44:43.258 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.255+0000 7f05effff640 1 -- 172.21.15.32:0/2786270928 <== mon.2 v2:172.21.15.121:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f05f0035030 con 0x7f05f810fd50 2024-04-25T00:44:43.263 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.260+0000 7f05fc829640 1 -- 172.21.15.32:0/2786270928 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f05d8005290 con 0x7f05f810fd50 2024-04-25T00:44:43.263 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.262+0000 7f05effff640 1 -- 172.21.15.32:0/2786270928 <== mon.2 v2:172.21.15.121:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f05f0042070 con 0x7f05f810fd50 2024-04-25T00:44:43.264 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.263+0000 7f05effff640 1 --2- 172.21.15.32:0/2786270928 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f05e40a2ed0 0x7f05e40a5390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:43.264 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.263+0000 7f05effff640 1 -- 172.21.15.32:0/2786270928 <== mon.2 v2:172.21.15.121:3300/0 5 ==== osd_map(105..105 src has 1..105) ==== 9510+0+0 (secure 0 0 0) 0x7f05f00efd50 con 0x7f05f810fd50 2024-04-25T00:44:43.266 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.264+0000 7f05f6d9d640 1 --2- 172.21.15.32:0/2786270928 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f05e40a2ed0 0x7f05e40a5390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:43.266 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.265+0000 7f05f6d9d640 1 --2- 172.21.15.32:0/2786270928 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f05e40a2ed0 0x7f05e40a5390 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f05e8009f40 tx=0x7f05e80094d0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:44:43.278 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.274+0000 7f05effff640 1 -- 172.21.15.32:0/2786270928 <== mon.2 v2:172.21.15.121:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f05f0049030 con 0x7f05f810fd50 2024-04-25T00:44:43.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:43 smithi130 ceph-mon[32525]: pgmap v356: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 6.4 KiB/s rd, 426 B/s wr, 8 op/s 2024-04-25T00:44:43.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:43 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:44:43.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:43 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2105643318' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:44:43.649 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.648+0000 7f05fc829640 1 -- 172.21.15.32:0/2786270928 --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"} v 0) -- 0x7f05d8005850 con 0x7f05f810fd50 2024-04-25T00:44:43.652 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.652+0000 7f05effff640 1 -- 172.21.15.32:0/2786270928 <== mon.2 v2:172.21.15.121:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]=0 v105) ==== 103+0+11 (secure 0 0 0) 0x7f05f00bc530 con 0x7f05f810fd50 2024-04-25T00:44:43.653 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 32 2024-04-25T00:44:43.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.656+0000 7f05fc829640 1 -- 172.21.15.32:0/2786270928 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f05e40a2ed0 msgr2=0x7f05e40a5390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:43.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.656+0000 7f05fc829640 1 --2- 172.21.15.32:0/2786270928 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f05e40a2ed0 0x7f05e40a5390 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f05e8009f40 tx=0x7f05e80094d0 comp rx=0 tx=0).stop 2024-04-25T00:44:43.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.656+0000 7f05fc829640 1 -- 172.21.15.32:0/2786270928 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f05f810fd50 msgr2=0x7f05f8115680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:43.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.656+0000 7f05fc829640 1 --2- 172.21.15.32:0/2786270928 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f05f810fd50 0x7f05f8115680 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f05f002a940 tx=0x7f05f0003b90 comp rx=0 tx=0).stop 2024-04-25T00:44:43.657 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.657+0000 7f05fc829640 1 -- 172.21.15.32:0/2786270928 shutdown_connections 2024-04-25T00:44:43.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.657+0000 7f05fc829640 1 --2- 172.21.15.32:0/2786270928 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f05e40a2ed0 0x7f05e40a5390 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:43.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.658+0000 7f05fc829640 1 --2- 172.21.15.32:0/2786270928 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f05f810fd50 0x7f05f8115680 unknown :-1 s=CLOSED pgs=138 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:43.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.658+0000 7f05fc829640 1 --2- 172.21.15.32:0/2786270928 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f05f810e3d0 0x7f05f811c9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:43.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.658+0000 7f05fc829640 1 --2- 172.21.15.32:0/2786270928 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f05f810d820 0x7f05f811c460 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:43.658 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.658+0000 7f05fc829640 1 -- 172.21.15.32:0/2786270928 >> 172.21.15.32:0/2786270928 conn(0x7f05f806c620 msgr2=0x7f05f8070a40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:44:43.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.660+0000 7f05fc829640 1 -- 172.21.15.32:0/2786270928 shutdown_connections 2024-04-25T00:44:43.660 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.660+0000 7f05fc829640 1 -- 172.21.15.32:0/2786270928 wait complete. 2024-04-25T00:44:43.677 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:43 smithi032 ceph-mon[29357]: pgmap v356: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 6.4 KiB/s rd, 426 B/s wr, 8 op/s 2024-04-25T00:44:43.677 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:43 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:44:43.677 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:43 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2105643318' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:44:43.679 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mgr dump --format=json-pretty 2024-04-25T00:44:43.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:43 smithi121 ceph-mon[32231]: pgmap v356: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 6.4 KiB/s rd, 426 B/s wr, 8 op/s 2024-04-25T00:44:43.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:43 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T00:44:43.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:43 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2105643318' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:44:43.886 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.885+0000 7efc4029a640 1 -- 172.21.15.32:0/1903806306 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efc38114890 msgr2=0x7efc38116c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:43.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.885+0000 7efc4029a640 1 --2- 172.21.15.32:0/1903806306 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efc38114890 0x7efc38116c80 secure :-1 s=READY pgs=656 cs=0 l=1 rev1=1 crypto rx=0x7efc28009900 tx=0x7efc2802a440 comp rx=0 tx=0).stop 2024-04-25T00:44:43.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.886+0000 7efc4029a640 1 -- 172.21.15.32:0/1903806306 shutdown_connections 2024-04-25T00:44:43.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.886+0000 7efc4029a640 1 --2- 172.21.15.32:0/1903806306 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efc38114890 0x7efc38116c80 unknown :-1 s=CLOSED pgs=656 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:43.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.886+0000 7efc4029a640 1 --2- 172.21.15.32:0/1903806306 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efc38109290 0x7efc38114350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:43.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.886+0000 7efc4029a640 1 --2- 172.21.15.32:0/1903806306 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efc38108970 0x7efc38108d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:43.887 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.886+0000 7efc4029a640 1 -- 172.21.15.32:0/1903806306 >> 172.21.15.32:0/1903806306 conn(0x7efc3806c860 msgr2=0x7efc3806cc70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:44:43.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.887+0000 7efc4029a640 1 -- 172.21.15.32:0/1903806306 shutdown_connections 2024-04-25T00:44:43.888 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.887+0000 7efc4029a640 1 -- 172.21.15.32:0/1903806306 wait complete. 2024-04-25T00:44:43.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.888+0000 7efc4029a640 1 Processor -- start 2024-04-25T00:44:43.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.888+0000 7efc4029a640 1 -- start start 2024-04-25T00:44:43.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.888+0000 7efc4029a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efc38108970 0x7efc381a6520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:43.889 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.888+0000 7efc4029a640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efc38109290 0x7efc381a6a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:43.890 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.888+0000 7efc4029a640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efc38114890 0x7efc381a0670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:43.890 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.888+0000 7efc4029a640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7efc3811a390 con 0x7efc38108970 2024-04-25T00:44:43.890 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.888+0000 7efc4029a640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7efc3811a210 con 0x7efc38114890 2024-04-25T00:44:43.890 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.888+0000 7efc4029a640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7efc3811a510 con 0x7efc38109290 2024-04-25T00:44:43.890 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.889+0000 7efc3e036640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efc38108970 0x7efc381a6520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:43.890 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.889+0000 7efc3e036640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efc38108970 0x7efc381a6520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:39074/0 (socket says 172.21.15.32:39074) 2024-04-25T00:44:43.890 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.889+0000 7efc3e036640 1 -- 172.21.15.32:0/4097369147 learned_addr learned my addr 172.21.15.32:0/4097369147 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:44:43.890 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.889+0000 7efc3d835640 1 --2- 172.21.15.32:0/4097369147 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efc38109290 0x7efc381a6a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:43.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.889+0000 7efc3e036640 1 -- 172.21.15.32:0/4097369147 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efc38109290 msgr2=0x7efc381a6a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:43.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.889+0000 7efc3e036640 1 --2- 172.21.15.32:0/4097369147 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efc38109290 0x7efc381a6a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:43.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.889+0000 7efc3e036640 1 -- 172.21.15.32:0/4097369147 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efc38114890 msgr2=0x7efc381a0670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:43.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.889+0000 7efc3e036640 1 --2- 172.21.15.32:0/4097369147 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efc38114890 0x7efc381a0670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:43.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.889+0000 7efc3e036640 1 -- 172.21.15.32:0/4097369147 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7efc381a0ee0 con 0x7efc38108970 2024-04-25T00:44:43.891 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.890+0000 7efc3d835640 1 --2- 172.21.15.32:0/4097369147 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efc38109290 0x7efc381a6a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:44:43.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.890+0000 7efc3e036640 1 --2- 172.21.15.32:0/4097369147 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efc38108970 0x7efc381a6520 secure :-1 s=READY pgs=657 cs=0 l=1 rev1=1 crypto rx=0x7efc3000ba50 tx=0x7efc3000bf10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:44:43.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.893+0000 7efc2f7fe640 1 -- 172.21.15.32:0/4097369147 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efc30019070 con 0x7efc38108970 2024-04-25T00:44:43.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.893+0000 7efc2f7fe640 1 -- 172.21.15.32:0/4097369147 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7efc3000cc10 con 0x7efc38108970 2024-04-25T00:44:43.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.893+0000 7efc2f7fe640 1 -- 172.21.15.32:0/4097369147 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efc30007770 con 0x7efc38108970 2024-04-25T00:44:43.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.893+0000 7efc4029a640 1 -- 172.21.15.32:0/4097369147 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7efc381a1170 con 0x7efc38108970 2024-04-25T00:44:43.895 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.893+0000 7efc4029a640 1 -- 172.21.15.32:0/4097369147 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7efc3810a0e0 con 0x7efc38108970 2024-04-25T00:44:43.900 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.899+0000 7efc4029a640 1 -- 172.21.15.32:0/4097369147 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7efc3806d460 con 0x7efc38108970 2024-04-25T00:44:43.904 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.903+0000 7efc2f7fe640 1 -- 172.21.15.32:0/4097369147 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7efc30007910 con 0x7efc38108970 2024-04-25T00:44:43.906 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.904+0000 7efc2f7fe640 1 --2- 172.21.15.32:0/4097369147 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7efc240a2f10 0x7efc240a53d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:44:43.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.904+0000 7efc2f7fe640 1 -- 172.21.15.32:0/4097369147 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(105..105 src has 1..105) ==== 9510+0+0 (secure 0 0 0) 0x7efc300cecb0 con 0x7efc38108970 2024-04-25T00:44:43.907 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.906+0000 7efc3d835640 1 --2- 172.21.15.32:0/4097369147 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7efc240a2f10 0x7efc240a53d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:44:43.908 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.907+0000 7efc3d835640 1 --2- 172.21.15.32:0/4097369147 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7efc240a2f10 0x7efc240a53d0 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7efc381162a0 tx=0x7efc3400b390 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:44:43.914 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:43.913+0000 7efc2f7fe640 1 -- 172.21.15.32:0/4097369147 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7efc3004f740 con 0x7efc38108970 2024-04-25T00:44:44.390 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:44.388+0000 7efc4029a640 1 -- 172.21.15.32:0/4097369147 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "mgr dump", "format": "json-pretty"} v 0) -- 0x7efc381a1b80 con 0x7efc38108970 2024-04-25T00:44:44.422 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:44.422+0000 7efc2f7fe640 1 -- 172.21.15.32:0/4097369147 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json-pretty"}]=0 v26) ==== 81+0+740902 (secure 0 0 0) 0x7efc3009b5a0 con 0x7efc38108970 2024-04-25T00:44:44.423 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:44:44.433 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:44:44.434 INFO:teuthology.orchestra.run.smithi032.stdout: "epoch": 26, 2024-04-25T00:44:44.434 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.434 INFO:teuthology.orchestra.run.smithi032.stdout: "active_gid": 15483, 2024-04-25T00:44:44.434 INFO:teuthology.orchestra.run.smithi032.stdout: "active_name": "x", 2024-04-25T00:44:44.434 INFO:teuthology.orchestra.run.smithi032.stdout: "active_addrs": { 2024-04-25T00:44:44.434 INFO:teuthology.orchestra.run.smithi032.stdout: "addrvec": [ 2024-04-25T00:44:44.434 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.434 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "v2", 2024-04-25T00:44:44.434 INFO:teuthology.orchestra.run.smithi032.stdout: "addr": "172.21.15.32:6800", 2024-04-25T00:44:44.434 INFO:teuthology.orchestra.run.smithi032.stdout: "nonce": 3307874911 2024-04-25T00:44:44.434 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.434 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.435 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "v1", 2024-04-25T00:44:44.435 INFO:teuthology.orchestra.run.smithi032.stdout: "addr": "172.21.15.32:6801", 2024-04-25T00:44:44.435 INFO:teuthology.orchestra.run.smithi032.stdout: "nonce": 3307874911 2024-04-25T00:44:44.435 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.435 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-04-25T00:44:44.435 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.435 INFO:teuthology.orchestra.run.smithi032.stdout: "active_addr": "172.21.15.32:6801/3307874911", 2024-04-25T00:44:44.435 INFO:teuthology.orchestra.run.smithi032.stdout: "active_change": "2024-04-25T00:33:17.920133+0000", 2024-04-25T00:44:44.435 INFO:teuthology.orchestra.run.smithi032.stdout: "active_mgr_features": 4540701547738038271, 2024-04-25T00:44:44.435 INFO:teuthology.orchestra.run.smithi032.stdout: "available": true, 2024-04-25T00:44:44.435 INFO:teuthology.orchestra.run.smithi032.stdout: "standbys": [ 2024-04-25T00:44:44.435 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.435 INFO:teuthology.orchestra.run.smithi032.stdout: "gid": 25087, 2024-04-25T00:44:44.436 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "z", 2024-04-25T00:44:44.436 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_features": 4540701547738038271, 2024-04-25T00:44:44.436 INFO:teuthology.orchestra.run.smithi032.stdout: "available_modules": [ 2024-04-25T00:44:44.436 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.436 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "alerts", 2024-04-25T00:44:44.436 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.436 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.436 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.436 INFO:teuthology.orchestra.run.smithi032.stdout: "interval": { 2024-04-25T00:44:44.436 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "interval", 2024-04-25T00:44:44.436 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.436 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.436 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.437 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "60", 2024-04-25T00:44:44.437 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.437 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.437 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.437 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "How frequently to reexamine health status", 2024-04-25T00:44:44.437 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.437 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.437 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.437 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.437 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.437 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.437 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.437 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.437 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.438 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.438 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.438 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.438 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.438 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.438 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.438 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.438 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.438 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.438 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.438 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.438 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.438 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.438 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.439 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.439 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.439 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.439 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.439 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.439 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.439 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.439 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.439 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.439 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.439 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.439 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.439 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.439 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.440 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.440 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.440 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.440 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.440 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.440 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.440 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.440 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.440 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.440 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.440 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.440 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.440 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.440 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.441 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.441 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.441 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.441 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.441 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.441 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.441 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.441 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.441 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.441 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.441 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.441 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.441 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.441 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.441 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.441 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.442 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.442 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.442 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.442 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.442 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.442 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.442 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.442 INFO:teuthology.orchestra.run.smithi032.stdout: "smtp_destination": { 2024-04-25T00:44:44.442 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "smtp_destination", 2024-04-25T00:44:44.442 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.442 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.442 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.442 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.442 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.442 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.443 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.443 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Email address to send alerts to", 2024-04-25T00:44:44.443 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.443 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.443 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.443 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.443 INFO:teuthology.orchestra.run.smithi032.stdout: "smtp_from_name": { 2024-04-25T00:44:44.443 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "smtp_from_name", 2024-04-25T00:44:44.443 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.443 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.443 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.443 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "Ceph", 2024-04-25T00:44:44.443 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.443 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.443 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.444 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Email From: name", 2024-04-25T00:44:44.444 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.444 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.444 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.444 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.444 INFO:teuthology.orchestra.run.smithi032.stdout: "smtp_host": { 2024-04-25T00:44:44.444 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "smtp_host", 2024-04-25T00:44:44.444 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.444 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.444 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.444 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.444 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.444 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.444 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.444 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "SMTP server", 2024-04-25T00:44:44.445 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.445 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.445 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.445 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.445 INFO:teuthology.orchestra.run.smithi032.stdout: "smtp_password": { 2024-04-25T00:44:44.445 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "smtp_password", 2024-04-25T00:44:44.445 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.445 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.445 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.445 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.445 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.445 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.445 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.445 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Password to authenticate with", 2024-04-25T00:44:44.446 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.446 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.446 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.446 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.446 INFO:teuthology.orchestra.run.smithi032.stdout: "smtp_port": { 2024-04-25T00:44:44.446 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "smtp_port", 2024-04-25T00:44:44.446 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.446 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.446 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.446 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "465", 2024-04-25T00:44:44.446 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.446 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.446 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.446 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "SMTP port", 2024-04-25T00:44:44.447 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.447 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.447 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.447 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.447 INFO:teuthology.orchestra.run.smithi032.stdout: "smtp_sender": { 2024-04-25T00:44:44.447 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "smtp_sender", 2024-04-25T00:44:44.447 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.447 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.447 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.447 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.447 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.447 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.447 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.447 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "SMTP envelope sender", 2024-04-25T00:44:44.447 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.448 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.448 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.448 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.448 INFO:teuthology.orchestra.run.smithi032.stdout: "smtp_ssl": { 2024-04-25T00:44:44.448 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "smtp_ssl", 2024-04-25T00:44:44.448 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.448 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.448 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.448 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.448 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.448 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.448 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.448 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Use SSL to connect to SMTP server", 2024-04-25T00:44:44.449 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.449 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.449 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.449 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.449 INFO:teuthology.orchestra.run.smithi032.stdout: "smtp_user": { 2024-04-25T00:44:44.449 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "smtp_user", 2024-04-25T00:44:44.449 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.449 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.449 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.449 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.449 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.449 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.449 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.449 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "User to authenticate as", 2024-04-25T00:44:44.450 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.450 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.450 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.450 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.450 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.450 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.450 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.450 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "balancer", 2024-04-25T00:44:44.450 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.450 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.450 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.450 INFO:teuthology.orchestra.run.smithi032.stdout: "active": { 2024-04-25T00:44:44.450 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "active", 2024-04-25T00:44:44.450 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.451 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.451 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.451 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.451 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.451 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.451 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.451 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "automatically balance PGs across cluster", 2024-04-25T00:44:44.451 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.451 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.451 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.451 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.451 INFO:teuthology.orchestra.run.smithi032.stdout: "begin_time": { 2024-04-25T00:44:44.451 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "begin_time", 2024-04-25T00:44:44.452 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.452 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.452 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.452 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "0000", 2024-04-25T00:44:44.452 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.452 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.452 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.452 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "beginning time of day to automatically balance", 2024-04-25T00:44:44.452 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "This is a time of day in the format HHMM.", 2024-04-25T00:44:44.452 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.452 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.452 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.452 INFO:teuthology.orchestra.run.smithi032.stdout: "begin_weekday": { 2024-04-25T00:44:44.452 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "begin_weekday", 2024-04-25T00:44:44.452 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "uint", 2024-04-25T00:44:44.453 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.453 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.453 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "0", 2024-04-25T00:44:44.453 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "0", 2024-04-25T00:44:44.453 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "6", 2024-04-25T00:44:44.453 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.453 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Restrict automatic balancing to this day of the week or later", 2024-04-25T00:44:44.453 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2024-04-25T00:44:44.453 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.453 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.453 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.453 INFO:teuthology.orchestra.run.smithi032.stdout: "crush_compat_max_iterations": { 2024-04-25T00:44:44.453 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "crush_compat_max_iterations", 2024-04-25T00:44:44.454 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "uint", 2024-04-25T00:44:44.454 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.454 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.454 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "25", 2024-04-25T00:44:44.454 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "1", 2024-04-25T00:44:44.454 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "250", 2024-04-25T00:44:44.454 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.454 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "maximum number of iterations to attempt optimization", 2024-04-25T00:44:44.454 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.454 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.454 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.454 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.454 INFO:teuthology.orchestra.run.smithi032.stdout: "crush_compat_metrics": { 2024-04-25T00:44:44.454 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "crush_compat_metrics", 2024-04-25T00:44:44.455 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.455 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.455 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.455 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "pgs,objects,bytes", 2024-04-25T00:44:44.455 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.455 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.455 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.455 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "metrics with which to calculate OSD utilization", 2024-04-25T00:44:44.455 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.", 2024-04-25T00:44:44.455 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.455 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.455 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.455 INFO:teuthology.orchestra.run.smithi032.stdout: "crush_compat_step": { 2024-04-25T00:44:44.455 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "crush_compat_step", 2024-04-25T00:44:44.455 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "float", 2024-04-25T00:44:44.455 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.456 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.456 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "0.5", 2024-04-25T00:44:44.456 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "0.001", 2024-04-25T00:44:44.456 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "0.999", 2024-04-25T00:44:44.456 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.456 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "aggressiveness of optimization", 2024-04-25T00:44:44.456 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": ".99 is very aggressive, .01 is less aggressive", 2024-04-25T00:44:44.456 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.456 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.456 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.456 INFO:teuthology.orchestra.run.smithi032.stdout: "end_time": { 2024-04-25T00:44:44.456 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "end_time", 2024-04-25T00:44:44.456 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.456 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.456 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.457 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "2359", 2024-04-25T00:44:44.457 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.457 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.457 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.457 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "ending time of day to automatically balance", 2024-04-25T00:44:44.457 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "This is a time of day in the format HHMM.", 2024-04-25T00:44:44.457 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.457 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.457 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.457 INFO:teuthology.orchestra.run.smithi032.stdout: "end_weekday": { 2024-04-25T00:44:44.457 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "end_weekday", 2024-04-25T00:44:44.457 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "uint", 2024-04-25T00:44:44.457 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.457 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.457 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "0", 2024-04-25T00:44:44.457 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "0", 2024-04-25T00:44:44.458 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "6", 2024-04-25T00:44:44.458 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.458 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Restrict automatic balancing to days of the week earlier than this", 2024-04-25T00:44:44.458 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2024-04-25T00:44:44.458 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.458 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.458 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.458 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.458 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.458 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.458 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.458 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.458 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.458 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.459 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.459 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.459 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.459 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.459 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.459 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.459 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.459 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.459 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.459 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.459 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.459 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.459 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.459 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.460 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.460 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.460 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.460 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.460 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.460 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.460 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.460 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.460 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.460 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.460 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.460 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.460 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.461 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.461 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.461 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.461 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.461 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.461 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.461 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.461 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.461 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.461 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.461 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.461 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.461 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.461 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.462 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.462 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.462 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.462 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.462 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.462 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.462 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.462 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.462 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.462 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.462 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.462 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.462 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.463 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.463 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.463 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.463 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.463 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.463 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.463 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.463 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.463 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.463 INFO:teuthology.orchestra.run.smithi032.stdout: "min_score": { 2024-04-25T00:44:44.463 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "min_score", 2024-04-25T00:44:44.463 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "float", 2024-04-25T00:44:44.463 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.463 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.463 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "0", 2024-04-25T00:44:44.464 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.464 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.464 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.464 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "minimum score, below which no optimization is attempted", 2024-04-25T00:44:44.464 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.464 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.464 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.464 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.464 INFO:teuthology.orchestra.run.smithi032.stdout: "mode": { 2024-04-25T00:44:44.464 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "mode", 2024-04-25T00:44:44.464 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.464 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.464 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.464 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "upmap", 2024-04-25T00:44:44.465 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.465 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.465 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.465 INFO:teuthology.orchestra.run.smithi032.stdout: "crush-compat", 2024-04-25T00:44:44.465 INFO:teuthology.orchestra.run.smithi032.stdout: "none", 2024-04-25T00:44:44.465 INFO:teuthology.orchestra.run.smithi032.stdout: "read", 2024-04-25T00:44:44.465 INFO:teuthology.orchestra.run.smithi032.stdout: "upmap", 2024-04-25T00:44:44.465 INFO:teuthology.orchestra.run.smithi032.stdout: "upmap-read" 2024-04-25T00:44:44.465 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.465 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Balancer mode", 2024-04-25T00:44:44.465 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.465 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.465 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.465 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.466 INFO:teuthology.orchestra.run.smithi032.stdout: "pool_ids": { 2024-04-25T00:44:44.466 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "pool_ids", 2024-04-25T00:44:44.466 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.466 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.466 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.466 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.466 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.466 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.466 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.466 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "pools which the automatic balancing will be limited to", 2024-04-25T00:44:44.466 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.466 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.466 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.466 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.467 INFO:teuthology.orchestra.run.smithi032.stdout: "sleep_interval": { 2024-04-25T00:44:44.467 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "sleep_interval", 2024-04-25T00:44:44.467 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.467 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.467 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.467 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "60", 2024-04-25T00:44:44.467 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.467 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.467 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.467 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "how frequently to wake up and attempt optimization", 2024-04-25T00:44:44.467 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.467 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.467 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.467 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.468 INFO:teuthology.orchestra.run.smithi032.stdout: "upmap_max_deviation": { 2024-04-25T00:44:44.468 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "upmap_max_deviation", 2024-04-25T00:44:44.468 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.468 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.468 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.468 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "5", 2024-04-25T00:44:44.468 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "1", 2024-04-25T00:44:44.468 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.468 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.468 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "deviation below which no optimization is attempted", 2024-04-25T00:44:44.468 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "If the number of PGs are within this count then no optimization is attempted", 2024-04-25T00:44:44.468 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.468 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.468 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.468 INFO:teuthology.orchestra.run.smithi032.stdout: "upmap_max_optimizations": { 2024-04-25T00:44:44.468 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "upmap_max_optimizations", 2024-04-25T00:44:44.469 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "uint", 2024-04-25T00:44:44.469 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.469 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.469 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10", 2024-04-25T00:44:44.469 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.469 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.469 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.469 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "maximum upmap optimizations to make per attempt", 2024-04-25T00:44:44.469 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.469 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.469 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.469 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.469 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.469 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.469 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.469 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "cephadm", 2024-04-25T00:44:44.470 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.470 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.470 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.470 INFO:teuthology.orchestra.run.smithi032.stdout: "agent_down_multiplier": { 2024-04-25T00:44:44.470 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "agent_down_multiplier", 2024-04-25T00:44:44.470 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "float", 2024-04-25T00:44:44.470 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.470 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.470 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "3.0", 2024-04-25T00:44:44.470 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.470 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.470 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.470 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Multiplied by agent refresh rate to calculate how long agent must not report before being marked down", 2024-04-25T00:44:44.470 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.471 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.471 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.471 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.471 INFO:teuthology.orchestra.run.smithi032.stdout: "agent_refresh_rate": { 2024-04-25T00:44:44.471 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "agent_refresh_rate", 2024-04-25T00:44:44.471 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.471 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.471 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.471 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "20", 2024-04-25T00:44:44.471 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.471 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.471 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.471 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "How often agent on each host will try to gather and send metadata", 2024-04-25T00:44:44.471 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.471 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.472 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.472 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.472 INFO:teuthology.orchestra.run.smithi032.stdout: "agent_starting_port": { 2024-04-25T00:44:44.472 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "agent_starting_port", 2024-04-25T00:44:44.472 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.472 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.472 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.472 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "4721", 2024-04-25T00:44:44.472 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.472 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.472 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.472 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)", 2024-04-25T00:44:44.472 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.472 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.472 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.473 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.473 INFO:teuthology.orchestra.run.smithi032.stdout: "allow_ptrace": { 2024-04-25T00:44:44.473 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "allow_ptrace", 2024-04-25T00:44:44.473 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.473 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.473 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.473 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.473 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.473 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.473 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.473 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "allow SYS_PTRACE capability on ceph containers", 2024-04-25T00:44:44.473 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.", 2024-04-25T00:44:44.473 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.473 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.473 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.473 INFO:teuthology.orchestra.run.smithi032.stdout: "autotune_interval": { 2024-04-25T00:44:44.474 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "autotune_interval", 2024-04-25T00:44:44.474 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.474 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.474 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.474 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "600", 2024-04-25T00:44:44.474 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.474 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.474 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.474 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "how frequently to autotune daemon memory", 2024-04-25T00:44:44.474 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.474 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.474 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.474 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.474 INFO:teuthology.orchestra.run.smithi032.stdout: "autotune_memory_target_ratio": { 2024-04-25T00:44:44.474 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "autotune_memory_target_ratio", 2024-04-25T00:44:44.475 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "float", 2024-04-25T00:44:44.475 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.475 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.475 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "0.7", 2024-04-25T00:44:44.475 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.475 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.475 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.475 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "ratio of total system memory to divide amongst autotuned daemons", 2024-04-25T00:44:44.475 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.475 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.475 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.475 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.475 INFO:teuthology.orchestra.run.smithi032.stdout: "cephadm_log_destination": { 2024-04-25T00:44:44.475 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "cephadm_log_destination", 2024-04-25T00:44:44.476 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.476 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.476 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.476 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.476 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.476 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.476 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.476 INFO:teuthology.orchestra.run.smithi032.stdout: "file", 2024-04-25T00:44:44.476 INFO:teuthology.orchestra.run.smithi032.stdout: "file,syslog", 2024-04-25T00:44:44.476 INFO:teuthology.orchestra.run.smithi032.stdout: "syslog" 2024-04-25T00:44:44.476 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.476 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Destination for cephadm command's persistent logging", 2024-04-25T00:44:44.476 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.476 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.476 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.476 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.477 INFO:teuthology.orchestra.run.smithi032.stdout: "cgroups_split": { 2024-04-25T00:44:44.477 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "cgroups_split", 2024-04-25T00:44:44.477 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.477 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.477 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.477 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.477 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.477 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.477 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.477 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Pass --cgroups=split when cephadm creates containers (currently podman only)", 2024-04-25T00:44:44.477 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.477 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.477 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.477 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.477 INFO:teuthology.orchestra.run.smithi032.stdout: "config_checks_enabled": { 2024-04-25T00:44:44.477 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "config_checks_enabled", 2024-04-25T00:44:44.478 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.478 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.478 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.478 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.478 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.478 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.478 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.478 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Enable or disable the cephadm configuration analysis", 2024-04-25T00:44:44.478 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.478 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.478 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.478 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.478 INFO:teuthology.orchestra.run.smithi032.stdout: "config_dashboard": { 2024-04-25T00:44:44.479 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "config_dashboard", 2024-04-25T00:44:44.479 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.479 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.479 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.479 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.479 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.479 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.479 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.479 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "manage configs like API endpoints in Dashboard.", 2024-04-25T00:44:44.479 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.479 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.479 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.479 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.479 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_alertmanager": { 2024-04-25T00:44:44.479 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_alertmanager", 2024-04-25T00:44:44.480 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.480 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.480 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.480 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/prometheus/alertmanager:v0.25.0", 2024-04-25T00:44:44.480 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.480 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.480 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.480 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Prometheus container image", 2024-04-25T00:44:44.480 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.480 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.480 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.480 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.480 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_base": { 2024-04-25T00:44:44.480 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_base", 2024-04-25T00:44:44.480 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.481 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.481 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.481 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/ceph/ceph", 2024-04-25T00:44:44.481 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.481 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.481 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.481 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Container image name, without the tag", 2024-04-25T00:44:44.481 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.481 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.481 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.481 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.481 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_elasticsearch": { 2024-04-25T00:44:44.481 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_elasticsearch", 2024-04-25T00:44:44.481 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.482 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.482 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.482 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/omrizeneva/elasticsearch:6.8.23", 2024-04-25T00:44:44.482 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.482 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.482 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.482 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "elasticsearch container image", 2024-04-25T00:44:44.482 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.482 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.482 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.482 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.482 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_grafana": { 2024-04-25T00:44:44.482 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_grafana", 2024-04-25T00:44:44.483 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.483 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.483 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.483 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/ceph/grafana:9.4.12", 2024-04-25T00:44:44.483 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.483 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.483 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.483 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Prometheus container image", 2024-04-25T00:44:44.483 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.483 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.483 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.483 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.483 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_haproxy": { 2024-04-25T00:44:44.483 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_haproxy", 2024-04-25T00:44:44.484 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.484 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.484 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.484 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/ceph/haproxy:2.3", 2024-04-25T00:44:44.484 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.484 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.484 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.484 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "HAproxy container image", 2024-04-25T00:44:44.484 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.484 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.484 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.484 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.484 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_jaeger_agent": { 2024-04-25T00:44:44.484 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_jaeger_agent", 2024-04-25T00:44:44.484 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.485 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.485 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.485 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/jaegertracing/jaeger-agent:1.29", 2024-04-25T00:44:44.485 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.485 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.485 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.485 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Jaeger agent container image", 2024-04-25T00:44:44.485 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.485 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.485 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.485 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.485 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_jaeger_collector": { 2024-04-25T00:44:44.485 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_jaeger_collector", 2024-04-25T00:44:44.485 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.485 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.485 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.485 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/jaegertracing/jaeger-collector:1.29", 2024-04-25T00:44:44.486 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.486 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.486 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.486 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Jaeger collector container image", 2024-04-25T00:44:44.486 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.486 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.486 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.486 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.486 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_jaeger_query": { 2024-04-25T00:44:44.486 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_jaeger_query", 2024-04-25T00:44:44.486 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.486 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.486 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.486 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/jaegertracing/jaeger-query:1.29", 2024-04-25T00:44:44.486 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.486 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.487 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.487 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Jaeger query container image", 2024-04-25T00:44:44.487 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.487 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.487 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.487 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.487 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_keepalived": { 2024-04-25T00:44:44.487 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_keepalived", 2024-04-25T00:44:44.487 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.487 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.487 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.487 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/ceph/keepalived:2.2.4", 2024-04-25T00:44:44.487 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.487 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.487 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.488 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Keepalived container image", 2024-04-25T00:44:44.488 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.488 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.488 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.488 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.488 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_loki": { 2024-04-25T00:44:44.488 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_loki", 2024-04-25T00:44:44.488 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.488 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.488 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.488 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "docker.io/grafana/loki:2.4.0", 2024-04-25T00:44:44.488 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.488 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.488 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.488 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Loki container image", 2024-04-25T00:44:44.488 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.489 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.489 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.489 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.489 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_node_exporter": { 2024-04-25T00:44:44.489 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_node_exporter", 2024-04-25T00:44:44.489 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.489 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.489 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.489 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/prometheus/node-exporter:v1.5.0", 2024-04-25T00:44:44.489 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.489 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.489 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.489 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Prometheus container image", 2024-04-25T00:44:44.489 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.489 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.489 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.490 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.490 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_nvmeof": { 2024-04-25T00:44:44.490 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_nvmeof", 2024-04-25T00:44:44.490 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.490 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.490 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.490 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/ceph/nvmeof:1.2.1", 2024-04-25T00:44:44.490 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.490 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.490 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.490 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Nvme-of container image", 2024-04-25T00:44:44.490 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.490 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.490 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.490 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.491 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_prometheus": { 2024-04-25T00:44:44.491 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_prometheus", 2024-04-25T00:44:44.491 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.491 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.491 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.491 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/prometheus/prometheus:v2.43.0", 2024-04-25T00:44:44.491 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.491 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.491 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.491 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Prometheus container image", 2024-04-25T00:44:44.491 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.491 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.491 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.491 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.491 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_promtail": { 2024-04-25T00:44:44.492 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_promtail", 2024-04-25T00:44:44.492 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.492 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.492 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.492 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "docker.io/grafana/promtail:2.4.0", 2024-04-25T00:44:44.492 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.492 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.492 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.492 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Promtail container image", 2024-04-25T00:44:44.492 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.492 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.492 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.492 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.492 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_samba": { 2024-04-25T00:44:44.492 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_samba", 2024-04-25T00:44:44.493 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.493 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.493 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.493 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/samba.org/samba-server:devbuilds-centos-amd64", 2024-04-25T00:44:44.493 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.493 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.493 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.493 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Samba/SMB container image", 2024-04-25T00:44:44.493 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.493 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.493 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.493 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.493 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_snmp_gateway": { 2024-04-25T00:44:44.493 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_snmp_gateway", 2024-04-25T00:44:44.494 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.494 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.494 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.494 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "docker.io/maxwo/snmp-notifier:v1.2.1", 2024-04-25T00:44:44.494 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.494 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.494 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.494 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "SNMP Gateway container image", 2024-04-25T00:44:44.494 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.494 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.494 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.494 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.494 INFO:teuthology.orchestra.run.smithi032.stdout: "container_init": { 2024-04-25T00:44:44.494 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_init", 2024-04-25T00:44:44.495 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.495 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.495 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.495 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.495 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.495 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.495 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.495 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Run podman/docker with `--init`", 2024-04-25T00:44:44.495 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.495 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.495 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.495 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.495 INFO:teuthology.orchestra.run.smithi032.stdout: "daemon_cache_timeout": { 2024-04-25T00:44:44.495 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "daemon_cache_timeout", 2024-04-25T00:44:44.495 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.496 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.496 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.496 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "600", 2024-04-25T00:44:44.496 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.496 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.496 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.496 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "seconds to cache service (daemon) inventory", 2024-04-25T00:44:44.496 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.496 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.496 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.496 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.496 INFO:teuthology.orchestra.run.smithi032.stdout: "default_cephadm_command_timeout": { 2024-04-25T00:44:44.496 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "default_cephadm_command_timeout", 2024-04-25T00:44:44.496 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.496 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.497 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.497 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "900", 2024-04-25T00:44:44.497 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.497 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.497 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.497 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Default timeout applied to cephadm commands run directly on the host (in seconds)", 2024-04-25T00:44:44.497 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.497 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.497 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.497 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.497 INFO:teuthology.orchestra.run.smithi032.stdout: "default_registry": { 2024-04-25T00:44:44.497 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "default_registry", 2024-04-25T00:44:44.497 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.497 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.497 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.498 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "docker.io", 2024-04-25T00:44:44.498 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.498 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.498 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.498 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Search-registry to which we should normalize unqualified image names. This is not the default registry", 2024-04-25T00:44:44.498 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.498 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.498 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.498 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.498 INFO:teuthology.orchestra.run.smithi032.stdout: "device_cache_timeout": { 2024-04-25T00:44:44.498 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "device_cache_timeout", 2024-04-25T00:44:44.498 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.498 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.499 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.499 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "1800", 2024-04-25T00:44:44.499 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.499 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.499 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.499 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "seconds to cache device inventory", 2024-04-25T00:44:44.499 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.499 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.499 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.499 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.499 INFO:teuthology.orchestra.run.smithi032.stdout: "device_enhanced_scan": { 2024-04-25T00:44:44.499 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "device_enhanced_scan", 2024-04-25T00:44:44.499 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.499 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.500 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.500 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.500 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.500 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.500 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.500 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Use libstoragemgmt during device scans", 2024-04-25T00:44:44.500 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.500 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.500 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.500 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.500 INFO:teuthology.orchestra.run.smithi032.stdout: "facts_cache_timeout": { 2024-04-25T00:44:44.500 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "facts_cache_timeout", 2024-04-25T00:44:44.500 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.500 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.501 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.501 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "60", 2024-04-25T00:44:44.501 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.501 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.501 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.501 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "seconds to cache host facts data", 2024-04-25T00:44:44.501 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.501 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.501 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.501 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.501 INFO:teuthology.orchestra.run.smithi032.stdout: "grafana_dashboards_path": { 2024-04-25T00:44:44.501 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "grafana_dashboards_path", 2024-04-25T00:44:44.501 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.501 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.501 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.502 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "/etc/grafana/dashboards/ceph-dashboard/", 2024-04-25T00:44:44.502 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.502 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.502 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.502 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "location of dashboards to include in grafana deployments", 2024-04-25T00:44:44.502 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.502 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.502 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.502 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.502 INFO:teuthology.orchestra.run.smithi032.stdout: "host_check_interval": { 2024-04-25T00:44:44.502 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "host_check_interval", 2024-04-25T00:44:44.502 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.502 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.503 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.503 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "600", 2024-04-25T00:44:44.503 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.503 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.503 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.503 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "how frequently to perform a host check", 2024-04-25T00:44:44.503 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.503 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.503 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.503 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.503 INFO:teuthology.orchestra.run.smithi032.stdout: "hw_monitoring": { 2024-04-25T00:44:44.503 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "hw_monitoring", 2024-04-25T00:44:44.503 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.503 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.504 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.504 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.504 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.504 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.504 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.504 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Deploy hw monitoring daemon on every host.", 2024-04-25T00:44:44.504 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.504 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.504 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.504 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.504 INFO:teuthology.orchestra.run.smithi032.stdout: "inventory_list_all": { 2024-04-25T00:44:44.504 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "inventory_list_all", 2024-04-25T00:44:44.504 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.504 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.504 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.505 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.505 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.505 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.505 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.505 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)", 2024-04-25T00:44:44.505 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.505 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.505 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.505 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.505 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.505 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.505 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.505 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.505 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.505 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.506 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.506 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.506 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.506 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.506 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.506 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.506 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.506 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.506 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.506 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.506 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.506 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.506 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.506 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.506 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.507 INFO:teuthology.orchestra.run.smithi032.stdout: "log_refresh_metadata": { 2024-04-25T00:44:44.507 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_refresh_metadata", 2024-04-25T00:44:44.507 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.507 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.507 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.507 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.507 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.507 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.507 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.507 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level", 2024-04-25T00:44:44.507 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.507 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.508 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.508 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.508 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.508 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.508 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.508 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.508 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.508 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.508 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.508 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.508 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.508 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "log to the \"cephadm\" cluster log channel\"", 2024-04-25T00:44:44.508 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.508 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.509 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.509 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.509 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.509 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.509 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.509 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.509 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.509 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.509 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.509 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.509 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.509 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.509 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.509 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.510 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.510 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.510 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.510 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.510 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.510 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.510 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.510 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.510 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.510 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.510 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.510 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.510 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.510 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.511 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.511 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.511 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.511 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.511 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.511 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.511 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.511 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.511 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.511 INFO:teuthology.orchestra.run.smithi032.stdout: "manage_etc_ceph_ceph_conf": { 2024-04-25T00:44:44.511 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "manage_etc_ceph_ceph_conf", 2024-04-25T00:44:44.511 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.511 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.511 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.511 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.512 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.512 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.512 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.512 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Manage and own /etc/ceph/ceph.conf on the hosts.", 2024-04-25T00:44:44.512 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.512 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.512 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.512 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.512 INFO:teuthology.orchestra.run.smithi032.stdout: "manage_etc_ceph_ceph_conf_hosts": { 2024-04-25T00:44:44.512 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "manage_etc_ceph_ceph_conf_hosts", 2024-04-25T00:44:44.512 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.512 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.512 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.512 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "*", 2024-04-25T00:44:44.513 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.513 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.513 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.513 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf", 2024-04-25T00:44:44.513 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.513 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.513 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.513 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.513 INFO:teuthology.orchestra.run.smithi032.stdout: "max_count_per_host": { 2024-04-25T00:44:44.513 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "max_count_per_host", 2024-04-25T00:44:44.513 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.513 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.513 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.513 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10", 2024-04-25T00:44:44.514 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.514 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.514 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.514 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "max number of daemons per service per host", 2024-04-25T00:44:44.514 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.514 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.514 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.514 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.514 INFO:teuthology.orchestra.run.smithi032.stdout: "max_osd_draining_count": { 2024-04-25T00:44:44.514 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "max_osd_draining_count", 2024-04-25T00:44:44.514 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.514 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.514 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.514 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10", 2024-04-25T00:44:44.515 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.515 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.515 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.515 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "max number of osds that will be drained simultaneously when osds are removed", 2024-04-25T00:44:44.515 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.515 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.515 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.515 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.515 INFO:teuthology.orchestra.run.smithi032.stdout: "migration_current": { 2024-04-25T00:44:44.515 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "migration_current", 2024-04-25T00:44:44.515 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.515 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.515 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.515 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.515 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.516 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.516 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.516 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "internal - do not modify", 2024-04-25T00:44:44.516 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.516 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.516 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.516 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.516 INFO:teuthology.orchestra.run.smithi032.stdout: "mode": { 2024-04-25T00:44:44.516 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "mode", 2024-04-25T00:44:44.516 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.516 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.516 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.516 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "root", 2024-04-25T00:44:44.516 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.517 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.517 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.517 INFO:teuthology.orchestra.run.smithi032.stdout: "cephadm-package", 2024-04-25T00:44:44.517 INFO:teuthology.orchestra.run.smithi032.stdout: "root" 2024-04-25T00:44:44.517 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.517 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "mode for remote execution of cephadm", 2024-04-25T00:44:44.517 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.517 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.517 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.517 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.517 INFO:teuthology.orchestra.run.smithi032.stdout: "oob_default_addr": { 2024-04-25T00:44:44.517 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "oob_default_addr", 2024-04-25T00:44:44.517 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.517 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.517 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.518 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "169.254.1.1", 2024-04-25T00:44:44.518 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.518 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.518 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.518 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Default address for RedFish API (oob management).", 2024-04-25T00:44:44.518 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.518 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.518 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.518 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.518 INFO:teuthology.orchestra.run.smithi032.stdout: "prometheus_alerts_path": { 2024-04-25T00:44:44.518 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "prometheus_alerts_path", 2024-04-25T00:44:44.518 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.518 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.518 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.519 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "/etc/prometheus/ceph/ceph_default_alerts.yml", 2024-04-25T00:44:44.519 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.519 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.519 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.519 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "location of alerts to include in prometheus deployments", 2024-04-25T00:44:44.519 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.519 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.519 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.519 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.519 INFO:teuthology.orchestra.run.smithi032.stdout: "registry_insecure": { 2024-04-25T00:44:44.519 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "registry_insecure", 2024-04-25T00:44:44.519 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.519 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.520 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.520 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.520 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.520 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.520 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.520 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Registry is to be considered insecure (no TLS available). Only for development purposes.", 2024-04-25T00:44:44.520 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.520 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.520 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.520 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.520 INFO:teuthology.orchestra.run.smithi032.stdout: "registry_password": { 2024-04-25T00:44:44.520 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "registry_password", 2024-04-25T00:44:44.520 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.520 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.521 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.521 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.521 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.521 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.521 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.521 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Custom repository password. Only used for logging into a registry.", 2024-04-25T00:44:44.521 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.521 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.521 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.521 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.521 INFO:teuthology.orchestra.run.smithi032.stdout: "registry_url": { 2024-04-25T00:44:44.521 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "registry_url", 2024-04-25T00:44:44.521 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.521 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.522 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.522 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.522 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.522 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.522 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.522 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Registry url for login purposes. This is not the default registry", 2024-04-25T00:44:44.522 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.522 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.522 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.522 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.522 INFO:teuthology.orchestra.run.smithi032.stdout: "registry_username": { 2024-04-25T00:44:44.522 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "registry_username", 2024-04-25T00:44:44.522 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.523 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.523 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.523 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.523 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.523 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.523 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.523 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Custom repository username. Only used for logging into a registry.", 2024-04-25T00:44:44.523 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.523 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.523 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.523 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.523 INFO:teuthology.orchestra.run.smithi032.stdout: "secure_monitoring_stack": { 2024-04-25T00:44:44.523 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "secure_monitoring_stack", 2024-04-25T00:44:44.523 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.524 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.524 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.524 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.524 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.524 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.524 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.524 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Enable TLS security for all the monitoring stack daemons", 2024-04-25T00:44:44.524 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.524 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.524 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.524 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.524 INFO:teuthology.orchestra.run.smithi032.stdout: "service_discovery_port": { 2024-04-25T00:44:44.524 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "service_discovery_port", 2024-04-25T00:44:44.524 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.524 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.525 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.525 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "8765", 2024-04-25T00:44:44.525 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.525 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.525 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.525 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "cephadm service discovery port", 2024-04-25T00:44:44.525 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.525 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.525 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.525 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.525 INFO:teuthology.orchestra.run.smithi032.stdout: "ssh_config_file": { 2024-04-25T00:44:44.525 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ssh_config_file", 2024-04-25T00:44:44.525 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.525 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.525 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.526 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.526 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.526 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.526 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.526 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "customized SSH config file to connect to managed hosts", 2024-04-25T00:44:44.526 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.526 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.526 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.526 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.526 INFO:teuthology.orchestra.run.smithi032.stdout: "use_agent": { 2024-04-25T00:44:44.526 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "use_agent", 2024-04-25T00:44:44.526 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.526 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.526 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.526 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.527 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.527 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.527 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.527 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Use cephadm agent on each host to gather and send metadata", 2024-04-25T00:44:44.527 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.527 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.527 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.527 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.527 INFO:teuthology.orchestra.run.smithi032.stdout: "use_repo_digest": { 2024-04-25T00:44:44.527 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "use_repo_digest", 2024-04-25T00:44:44.527 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.527 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.527 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.527 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.528 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.528 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.528 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.528 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Automatically convert image tags to image digest. Make sure all daemons use the same image", 2024-04-25T00:44:44.528 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.528 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.528 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.528 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.528 INFO:teuthology.orchestra.run.smithi032.stdout: "warn_on_failed_host_check": { 2024-04-25T00:44:44.528 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "warn_on_failed_host_check", 2024-04-25T00:44:44.528 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.528 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.528 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.528 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.528 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.529 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.529 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.529 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "raise a health warning if the host check fails", 2024-04-25T00:44:44.529 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.529 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.529 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.529 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.529 INFO:teuthology.orchestra.run.smithi032.stdout: "warn_on_stray_daemons": { 2024-04-25T00:44:44.529 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "warn_on_stray_daemons", 2024-04-25T00:44:44.529 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.529 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.529 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.529 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.529 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.529 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.530 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.530 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "raise a health warning if daemons are detected that are not managed by cephadm", 2024-04-25T00:44:44.530 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.530 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.530 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.530 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.530 INFO:teuthology.orchestra.run.smithi032.stdout: "warn_on_stray_hosts": { 2024-04-25T00:44:44.530 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "warn_on_stray_hosts", 2024-04-25T00:44:44.530 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.530 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.530 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.530 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.530 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.530 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.530 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.531 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "raise a health warning if daemons are detected on a host that is not managed by cephadm", 2024-04-25T00:44:44.531 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.531 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.531 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.531 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.531 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.531 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.531 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.531 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "crash", 2024-04-25T00:44:44.531 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.531 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.531 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.531 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.531 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.532 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.532 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.532 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.532 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.532 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.532 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.532 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.532 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.532 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.532 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.532 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.532 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.532 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.532 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.533 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.533 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.533 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.533 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.533 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.533 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.533 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.533 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.534 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.534 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.534 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.534 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.534 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.534 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.534 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.534 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.535 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.535 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.535 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.535 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.535 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.535 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.535 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.535 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.535 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.535 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.535 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.535 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.535 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.535 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.536 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.536 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.536 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.536 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.536 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.536 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.536 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.536 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.536 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.536 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.536 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.536 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.536 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.536 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.537 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.537 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.537 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.537 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.537 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.537 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.537 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.537 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.537 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.537 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.537 INFO:teuthology.orchestra.run.smithi032.stdout: "retain_interval": { 2024-04-25T00:44:44.537 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "retain_interval", 2024-04-25T00:44:44.537 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.537 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.538 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.538 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "31536000", 2024-04-25T00:44:44.538 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.538 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.538 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.538 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "how long to retain crashes before pruning them", 2024-04-25T00:44:44.538 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.538 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.538 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.538 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.538 INFO:teuthology.orchestra.run.smithi032.stdout: "warn_recent_interval": { 2024-04-25T00:44:44.538 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "warn_recent_interval", 2024-04-25T00:44:44.538 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.538 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.538 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.539 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "1209600", 2024-04-25T00:44:44.539 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.539 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.539 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.539 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "time interval in which to warn about recent crashes", 2024-04-25T00:44:44.539 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.539 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.539 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.539 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.539 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.539 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.539 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.539 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "dashboard", 2024-04-25T00:44:44.539 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.539 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.540 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.540 INFO:teuthology.orchestra.run.smithi032.stdout: "ACCOUNT_LOCKOUT_ATTEMPTS": { 2024-04-25T00:44:44.540 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ACCOUNT_LOCKOUT_ATTEMPTS", 2024-04-25T00:44:44.540 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.540 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.540 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.540 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10", 2024-04-25T00:44:44.540 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.540 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.540 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.540 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.540 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.540 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.540 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.540 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.541 INFO:teuthology.orchestra.run.smithi032.stdout: "ALERTMANAGER_API_HOST": { 2024-04-25T00:44:44.541 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ALERTMANAGER_API_HOST", 2024-04-25T00:44:44.541 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.541 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.541 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.541 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.541 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.541 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.541 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.541 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.541 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.541 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.541 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.541 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.541 INFO:teuthology.orchestra.run.smithi032.stdout: "ALERTMANAGER_API_SSL_VERIFY": { 2024-04-25T00:44:44.542 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ALERTMANAGER_API_SSL_VERIFY", 2024-04-25T00:44:44.542 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.542 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.542 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.542 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.542 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.542 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.542 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.542 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.542 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.542 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.542 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.542 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.542 INFO:teuthology.orchestra.run.smithi032.stdout: "AUDIT_API_ENABLED": { 2024-04-25T00:44:44.542 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "AUDIT_API_ENABLED", 2024-04-25T00:44:44.543 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.543 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.543 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.543 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.543 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.543 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.543 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.543 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.543 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.543 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.543 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.543 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.543 INFO:teuthology.orchestra.run.smithi032.stdout: "AUDIT_API_LOG_PAYLOAD": { 2024-04-25T00:44:44.543 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "AUDIT_API_LOG_PAYLOAD", 2024-04-25T00:44:44.544 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.544 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.544 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.544 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.544 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.544 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.544 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.544 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.544 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.544 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.544 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.544 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.544 INFO:teuthology.orchestra.run.smithi032.stdout: "ENABLE_BROWSABLE_API": { 2024-04-25T00:44:44.544 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ENABLE_BROWSABLE_API", 2024-04-25T00:44:44.544 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.545 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.545 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.545 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.545 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.545 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.545 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.545 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.545 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.545 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.545 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.545 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.545 INFO:teuthology.orchestra.run.smithi032.stdout: "FEATURE_TOGGLE_CEPHFS": { 2024-04-25T00:44:44.545 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "FEATURE_TOGGLE_CEPHFS", 2024-04-25T00:44:44.546 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.546 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.546 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.546 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.546 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.546 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.546 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.546 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.546 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.546 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.546 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.546 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.546 INFO:teuthology.orchestra.run.smithi032.stdout: "FEATURE_TOGGLE_DASHBOARD": { 2024-04-25T00:44:44.546 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "FEATURE_TOGGLE_DASHBOARD", 2024-04-25T00:44:44.547 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.547 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.547 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.547 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.547 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.547 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.547 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.547 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.547 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.547 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.547 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.547 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.547 INFO:teuthology.orchestra.run.smithi032.stdout: "FEATURE_TOGGLE_ISCSI": { 2024-04-25T00:44:44.547 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "FEATURE_TOGGLE_ISCSI", 2024-04-25T00:44:44.547 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.548 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.548 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.548 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.548 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.548 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.548 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.548 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.548 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.548 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.548 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.548 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.548 INFO:teuthology.orchestra.run.smithi032.stdout: "FEATURE_TOGGLE_MIRRORING": { 2024-04-25T00:44:44.548 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "FEATURE_TOGGLE_MIRRORING", 2024-04-25T00:44:44.548 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.549 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.549 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.549 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.549 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.549 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.549 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.549 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.549 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.549 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.549 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.549 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.549 INFO:teuthology.orchestra.run.smithi032.stdout: "FEATURE_TOGGLE_NFS": { 2024-04-25T00:44:44.549 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "FEATURE_TOGGLE_NFS", 2024-04-25T00:44:44.549 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.549 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.550 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.550 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.550 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.550 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.550 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.550 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.550 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.550 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.550 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.550 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.550 INFO:teuthology.orchestra.run.smithi032.stdout: "FEATURE_TOGGLE_RBD": { 2024-04-25T00:44:44.550 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "FEATURE_TOGGLE_RBD", 2024-04-25T00:44:44.550 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.550 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.551 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.551 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.551 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.551 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.551 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.551 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.551 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.551 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.551 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.551 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.551 INFO:teuthology.orchestra.run.smithi032.stdout: "FEATURE_TOGGLE_RGW": { 2024-04-25T00:44:44.551 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "FEATURE_TOGGLE_RGW", 2024-04-25T00:44:44.551 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.551 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.551 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.552 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.552 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.552 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.552 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.552 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.552 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.552 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.552 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.552 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.552 INFO:teuthology.orchestra.run.smithi032.stdout: "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE": { 2024-04-25T00:44:44.552 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE", 2024-04-25T00:44:44.552 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.552 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.552 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.553 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.553 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.553 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.553 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.553 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.553 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.553 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.553 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.553 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.553 INFO:teuthology.orchestra.run.smithi032.stdout: "GRAFANA_API_PASSWORD": { 2024-04-25T00:44:44.553 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "GRAFANA_API_PASSWORD", 2024-04-25T00:44:44.553 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.553 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.553 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.553 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "admin", 2024-04-25T00:44:44.554 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.554 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.554 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.554 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.554 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.554 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.554 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.554 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.554 INFO:teuthology.orchestra.run.smithi032.stdout: "GRAFANA_API_SSL_VERIFY": { 2024-04-25T00:44:44.554 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "GRAFANA_API_SSL_VERIFY", 2024-04-25T00:44:44.554 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.554 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.554 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.555 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.555 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.555 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.555 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.555 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.555 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.555 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.555 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.555 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.555 INFO:teuthology.orchestra.run.smithi032.stdout: "GRAFANA_API_URL": { 2024-04-25T00:44:44.555 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "GRAFANA_API_URL", 2024-04-25T00:44:44.555 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.555 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.555 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.555 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.556 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.556 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.557 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.557 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.558 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.558 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.558 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.558 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.558 INFO:teuthology.orchestra.run.smithi032.stdout: "GRAFANA_API_USERNAME": { 2024-04-25T00:44:44.558 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "GRAFANA_API_USERNAME", 2024-04-25T00:44:44.558 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.558 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.558 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.558 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "admin", 2024-04-25T00:44:44.558 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.558 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.559 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.559 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.559 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.559 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.559 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.559 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.559 INFO:teuthology.orchestra.run.smithi032.stdout: "GRAFANA_FRONTEND_API_URL": { 2024-04-25T00:44:44.559 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "GRAFANA_FRONTEND_API_URL", 2024-04-25T00:44:44.559 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.559 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.559 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.559 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.559 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.559 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.559 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.560 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.560 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.560 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.560 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.560 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.560 INFO:teuthology.orchestra.run.smithi032.stdout: "GRAFANA_UPDATE_DASHBOARDS": { 2024-04-25T00:44:44.560 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "GRAFANA_UPDATE_DASHBOARDS", 2024-04-25T00:44:44.560 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.560 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.560 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.560 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.560 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.560 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.561 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.561 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.561 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.561 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.561 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.561 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.561 INFO:teuthology.orchestra.run.smithi032.stdout: "ISCSI_API_SSL_VERIFICATION": { 2024-04-25T00:44:44.561 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ISCSI_API_SSL_VERIFICATION", 2024-04-25T00:44:44.561 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.561 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.561 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.561 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.561 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.561 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.561 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.562 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.562 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.562 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.562 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.562 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.562 INFO:teuthology.orchestra.run.smithi032.stdout: "ISSUE_TRACKER_API_KEY": { 2024-04-25T00:44:44.562 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ISSUE_TRACKER_API_KEY", 2024-04-25T00:44:44.562 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.562 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.562 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.562 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.562 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.562 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.562 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.563 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.563 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.563 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.563 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.563 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.563 INFO:teuthology.orchestra.run.smithi032.stdout: "PROMETHEUS_API_HOST": { 2024-04-25T00:44:44.563 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PROMETHEUS_API_HOST", 2024-04-25T00:44:44.563 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.563 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.563 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.563 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.563 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.563 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.563 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.564 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.564 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.564 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.564 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.564 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.564 INFO:teuthology.orchestra.run.smithi032.stdout: "PROMETHEUS_API_SSL_VERIFY": { 2024-04-25T00:44:44.564 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PROMETHEUS_API_SSL_VERIFY", 2024-04-25T00:44:44.564 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.564 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.564 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.564 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.564 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.564 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.564 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.564 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.565 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.565 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.565 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.565 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.565 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_CHECK_COMPLEXITY_ENABLED": { 2024-04-25T00:44:44.565 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_CHECK_COMPLEXITY_ENABLED", 2024-04-25T00:44:44.565 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.565 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.565 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.565 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.565 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.565 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.565 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.565 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.566 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.566 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.566 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.566 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.566 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED": { 2024-04-25T00:44:44.566 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED", 2024-04-25T00:44:44.566 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.566 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.566 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.566 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.566 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.566 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.566 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.566 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.566 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.567 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.567 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.567 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.567 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_CHECK_LENGTH_ENABLED": { 2024-04-25T00:44:44.567 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_CHECK_LENGTH_ENABLED", 2024-04-25T00:44:44.567 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.567 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.567 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.567 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.567 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.567 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.567 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.567 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.567 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.568 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.568 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.568 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.568 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_CHECK_OLDPWD_ENABLED": { 2024-04-25T00:44:44.568 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_CHECK_OLDPWD_ENABLED", 2024-04-25T00:44:44.568 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.568 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.568 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.568 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.568 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.568 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.568 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.568 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.568 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.568 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.569 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.569 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.569 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED": { 2024-04-25T00:44:44.569 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED", 2024-04-25T00:44:44.569 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.569 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.569 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.569 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.569 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.569 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.569 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.569 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.569 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.569 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.570 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.570 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.570 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED": { 2024-04-25T00:44:44.570 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED", 2024-04-25T00:44:44.570 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.570 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.570 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.570 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.570 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.570 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.570 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.570 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.570 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.570 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.571 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.571 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.571 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_CHECK_USERNAME_ENABLED": { 2024-04-25T00:44:44.571 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_CHECK_USERNAME_ENABLED", 2024-04-25T00:44:44.571 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.571 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.571 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.571 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.571 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.571 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.571 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.571 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.571 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.571 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.572 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.572 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.572 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_ENABLED": { 2024-04-25T00:44:44.572 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_ENABLED", 2024-04-25T00:44:44.572 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.572 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.572 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.572 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.572 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.572 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.572 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.572 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.572 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.572 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.573 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.573 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.573 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_EXCLUSION_LIST": { 2024-04-25T00:44:44.573 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_EXCLUSION_LIST", 2024-04-25T00:44:44.573 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.573 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.573 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.573 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps", 2024-04-25T00:44:44.573 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.573 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.573 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.573 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.573 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.573 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.574 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.574 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.574 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_MIN_COMPLEXITY": { 2024-04-25T00:44:44.574 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_MIN_COMPLEXITY", 2024-04-25T00:44:44.574 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.574 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.574 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.574 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10", 2024-04-25T00:44:44.574 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.574 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.574 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.574 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.574 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.574 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.575 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.575 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.575 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_MIN_LENGTH": { 2024-04-25T00:44:44.575 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_MIN_LENGTH", 2024-04-25T00:44:44.575 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.575 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.575 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.575 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "8", 2024-04-25T00:44:44.575 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.575 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.575 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.575 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.575 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.575 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.576 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.576 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.576 INFO:teuthology.orchestra.run.smithi032.stdout: "REST_REQUESTS_TIMEOUT": { 2024-04-25T00:44:44.576 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "REST_REQUESTS_TIMEOUT", 2024-04-25T00:44:44.576 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.576 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.576 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.576 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "45", 2024-04-25T00:44:44.576 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.576 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.576 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.576 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.576 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.576 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.576 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.577 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.577 INFO:teuthology.orchestra.run.smithi032.stdout: "RGW_API_ACCESS_KEY": { 2024-04-25T00:44:44.577 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "RGW_API_ACCESS_KEY", 2024-04-25T00:44:44.577 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.577 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.577 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.577 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.577 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.577 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.577 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.577 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.577 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.577 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.577 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.578 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.578 INFO:teuthology.orchestra.run.smithi032.stdout: "RGW_API_ADMIN_RESOURCE": { 2024-04-25T00:44:44.578 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "RGW_API_ADMIN_RESOURCE", 2024-04-25T00:44:44.578 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.578 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.578 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.578 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "admin", 2024-04-25T00:44:44.578 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.578 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.578 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.578 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.578 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.578 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.578 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.578 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.579 INFO:teuthology.orchestra.run.smithi032.stdout: "RGW_API_SECRET_KEY": { 2024-04-25T00:44:44.579 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "RGW_API_SECRET_KEY", 2024-04-25T00:44:44.579 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.579 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.579 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.579 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.579 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.579 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.579 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.579 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.579 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.579 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.579 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.579 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.579 INFO:teuthology.orchestra.run.smithi032.stdout: "RGW_API_SSL_VERIFY": { 2024-04-25T00:44:44.580 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "RGW_API_SSL_VERIFY", 2024-04-25T00:44:44.580 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.580 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.580 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.580 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.580 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.580 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.580 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.580 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.580 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.580 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.580 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.580 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.580 INFO:teuthology.orchestra.run.smithi032.stdout: "USER_PWD_EXPIRATION_SPAN": { 2024-04-25T00:44:44.581 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "USER_PWD_EXPIRATION_SPAN", 2024-04-25T00:44:44.581 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.581 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.581 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.581 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "0", 2024-04-25T00:44:44.581 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.581 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.581 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.581 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.581 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.581 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.581 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.581 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.581 INFO:teuthology.orchestra.run.smithi032.stdout: "USER_PWD_EXPIRATION_WARNING_1": { 2024-04-25T00:44:44.581 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "USER_PWD_EXPIRATION_WARNING_1", 2024-04-25T00:44:44.582 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.582 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.582 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.582 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10", 2024-04-25T00:44:44.582 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.582 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.582 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.582 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.582 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.582 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.582 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.582 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.582 INFO:teuthology.orchestra.run.smithi032.stdout: "USER_PWD_EXPIRATION_WARNING_2": { 2024-04-25T00:44:44.582 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "USER_PWD_EXPIRATION_WARNING_2", 2024-04-25T00:44:44.582 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.583 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.583 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.583 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "5", 2024-04-25T00:44:44.583 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.583 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.583 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.583 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.583 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.583 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.583 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.583 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.583 INFO:teuthology.orchestra.run.smithi032.stdout: "cross_origin_url": { 2024-04-25T00:44:44.583 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "cross_origin_url", 2024-04-25T00:44:44.583 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.584 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.584 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.584 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.584 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.584 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.584 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.584 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.584 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.584 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.584 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.584 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.584 INFO:teuthology.orchestra.run.smithi032.stdout: "crt_file": { 2024-04-25T00:44:44.584 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "crt_file", 2024-04-25T00:44:44.584 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.584 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.584 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.585 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.585 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.585 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.585 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.585 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.585 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.585 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.585 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.585 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.585 INFO:teuthology.orchestra.run.smithi032.stdout: "debug": { 2024-04-25T00:44:44.585 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "debug", 2024-04-25T00:44:44.585 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.585 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.585 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.586 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.586 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.586 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.586 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.586 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Enable/disable debug options", 2024-04-25T00:44:44.586 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.586 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.586 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.586 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.586 INFO:teuthology.orchestra.run.smithi032.stdout: "jwt_token_ttl": { 2024-04-25T00:44:44.586 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "jwt_token_ttl", 2024-04-25T00:44:44.586 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.586 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.586 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.587 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "28800", 2024-04-25T00:44:44.587 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.587 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.587 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.587 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.587 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.587 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.587 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.587 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.587 INFO:teuthology.orchestra.run.smithi032.stdout: "key_file": { 2024-04-25T00:44:44.587 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "key_file", 2024-04-25T00:44:44.587 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.587 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.587 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.587 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.587 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.588 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.588 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.588 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.588 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.588 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.588 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.588 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.588 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.588 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.588 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.588 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.588 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.588 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.588 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.588 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.588 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.589 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.589 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.589 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.589 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.589 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.589 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.589 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.589 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.589 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.589 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.589 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.589 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.589 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.589 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.590 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.590 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.590 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.590 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.590 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.590 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.590 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.590 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.590 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.590 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.590 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.590 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.590 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.590 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.591 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.591 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.591 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.591 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.591 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.591 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.591 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.591 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.591 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.591 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.591 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.591 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.591 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.591 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.591 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.592 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.592 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.592 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.592 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.592 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.592 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.592 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.592 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.592 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.592 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.592 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.592 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.592 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.592 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.593 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.593 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.593 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.593 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.593 INFO:teuthology.orchestra.run.smithi032.stdout: "motd": { 2024-04-25T00:44:44.593 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "motd", 2024-04-25T00:44:44.593 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.593 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.593 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.593 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.593 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.593 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.593 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.593 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "The message of the day", 2024-04-25T00:44:44.593 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.594 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.594 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.594 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.594 INFO:teuthology.orchestra.run.smithi032.stdout: "redirect_resolve_ip_addr": { 2024-04-25T00:44:44.594 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "redirect_resolve_ip_addr", 2024-04-25T00:44:44.594 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.594 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.594 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.594 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.594 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.594 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.594 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.594 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.594 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.594 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.595 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.595 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.595 INFO:teuthology.orchestra.run.smithi032.stdout: "server_addr": { 2024-04-25T00:44:44.595 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "server_addr", 2024-04-25T00:44:44.595 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.595 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.595 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.595 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "::", 2024-04-25T00:44:44.595 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.595 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.595 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.595 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.595 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.595 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.595 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.596 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.596 INFO:teuthology.orchestra.run.smithi032.stdout: "server_port": { 2024-04-25T00:44:44.596 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "server_port", 2024-04-25T00:44:44.596 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.596 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.596 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.596 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "8080", 2024-04-25T00:44:44.596 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.596 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.596 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.596 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.596 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.596 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.596 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.597 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.597 INFO:teuthology.orchestra.run.smithi032.stdout: "ssl": { 2024-04-25T00:44:44.597 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ssl", 2024-04-25T00:44:44.597 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.597 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.597 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.597 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.597 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.597 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.597 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.597 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.597 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.597 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.597 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.598 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.598 INFO:teuthology.orchestra.run.smithi032.stdout: "ssl_server_port": { 2024-04-25T00:44:44.598 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ssl_server_port", 2024-04-25T00:44:44.598 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.598 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.598 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.598 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "8443", 2024-04-25T00:44:44.598 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.598 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.598 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.598 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.598 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.598 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.599 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.599 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.599 INFO:teuthology.orchestra.run.smithi032.stdout: "standby_behaviour": { 2024-04-25T00:44:44.599 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "standby_behaviour", 2024-04-25T00:44:44.599 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.599 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.599 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.599 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "redirect", 2024-04-25T00:44:44.599 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.599 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.599 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.599 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.599 INFO:teuthology.orchestra.run.smithi032.stdout: "redirect" 2024-04-25T00:44:44.599 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.600 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.600 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.600 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.600 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.600 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.600 INFO:teuthology.orchestra.run.smithi032.stdout: "standby_error_status_code": { 2024-04-25T00:44:44.600 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "standby_error_status_code", 2024-04-25T00:44:44.600 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.600 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.600 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.600 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "500", 2024-04-25T00:44:44.600 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "400", 2024-04-25T00:44:44.600 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "599", 2024-04-25T00:44:44.600 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.600 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.601 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.601 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.601 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.601 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.601 INFO:teuthology.orchestra.run.smithi032.stdout: "url_prefix": { 2024-04-25T00:44:44.601 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "url_prefix", 2024-04-25T00:44:44.601 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.601 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.601 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.601 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.601 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.601 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.601 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.601 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.601 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.602 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.602 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.602 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.602 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.602 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.602 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.602 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "devicehealth", 2024-04-25T00:44:44.602 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.602 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.602 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.602 INFO:teuthology.orchestra.run.smithi032.stdout: "enable_monitoring": { 2024-04-25T00:44:44.602 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "enable_monitoring", 2024-04-25T00:44:44.602 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.602 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.603 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.603 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.603 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.603 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.603 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.603 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "monitor device health metrics", 2024-04-25T00:44:44.603 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.603 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.603 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.603 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.603 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.603 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.603 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.603 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.604 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.604 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.604 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.604 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.604 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.604 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.604 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.604 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.604 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.604 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.604 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.604 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.604 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.605 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.605 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.605 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.605 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.605 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.605 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.605 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.605 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.605 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.605 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.605 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.605 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.605 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.605 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.605 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.606 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.606 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.606 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.606 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.606 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.606 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.606 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.606 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.606 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.606 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.606 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.606 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.606 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.606 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.606 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.607 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.607 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.607 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.607 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.607 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.607 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.607 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.607 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.607 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.607 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.607 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.607 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.607 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.607 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.607 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.608 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.608 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.608 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.608 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.608 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.608 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.608 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.608 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.608 INFO:teuthology.orchestra.run.smithi032.stdout: "mark_out_threshold": { 2024-04-25T00:44:44.608 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "mark_out_threshold", 2024-04-25T00:44:44.608 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.608 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.608 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.608 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "2419200", 2024-04-25T00:44:44.609 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.609 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.609 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.609 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "automatically mark OSD if it may fail before this long", 2024-04-25T00:44:44.609 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.609 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.609 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.609 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.609 INFO:teuthology.orchestra.run.smithi032.stdout: "pool_name": { 2024-04-25T00:44:44.609 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "pool_name", 2024-04-25T00:44:44.609 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.609 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.609 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.609 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "device_health_metrics", 2024-04-25T00:44:44.610 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.610 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.610 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.610 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "name of pool in which to store device health metrics", 2024-04-25T00:44:44.610 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.610 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.610 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.610 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.610 INFO:teuthology.orchestra.run.smithi032.stdout: "retention_period": { 2024-04-25T00:44:44.610 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "retention_period", 2024-04-25T00:44:44.610 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.610 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.610 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.610 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "15552000", 2024-04-25T00:44:44.611 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.611 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.611 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.611 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "how long to retain device health metrics", 2024-04-25T00:44:44.611 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.611 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.611 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.611 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.611 INFO:teuthology.orchestra.run.smithi032.stdout: "scrape_frequency": { 2024-04-25T00:44:44.611 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "scrape_frequency", 2024-04-25T00:44:44.611 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.611 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.611 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.611 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "86400", 2024-04-25T00:44:44.611 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.612 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.612 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.612 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "how frequently to scrape device health metrics", 2024-04-25T00:44:44.612 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.612 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.612 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.612 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.612 INFO:teuthology.orchestra.run.smithi032.stdout: "self_heal": { 2024-04-25T00:44:44.612 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "self_heal", 2024-04-25T00:44:44.612 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.612 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.612 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.612 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.612 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.613 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.613 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.613 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "preemptively heal cluster around devices that may fail", 2024-04-25T00:44:44.613 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.613 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.613 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.613 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.613 INFO:teuthology.orchestra.run.smithi032.stdout: "sleep_interval": { 2024-04-25T00:44:44.613 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "sleep_interval", 2024-04-25T00:44:44.613 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.613 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.613 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.613 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "600", 2024-04-25T00:44:44.613 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.613 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.614 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.614 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "how frequently to wake up and check device health", 2024-04-25T00:44:44.614 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.614 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.614 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.614 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.614 INFO:teuthology.orchestra.run.smithi032.stdout: "warn_threshold": { 2024-04-25T00:44:44.614 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "warn_threshold", 2024-04-25T00:44:44.614 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.614 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.614 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.614 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "7257600", 2024-04-25T00:44:44.614 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.614 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.615 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.615 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "raise health warning if OSD may fail before this long", 2024-04-25T00:44:44.615 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.615 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.615 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.615 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.615 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.615 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.615 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.615 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "diskprediction_local", 2024-04-25T00:44:44.615 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.615 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.615 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.615 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.615 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.615 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.616 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.616 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.616 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.616 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.616 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.616 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.616 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.616 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.616 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.616 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.616 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.616 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.616 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.616 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.616 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.616 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.617 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.617 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.617 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.617 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.617 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.617 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.617 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.617 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.617 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.617 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.617 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.618 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.618 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.618 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.618 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.618 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.618 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.618 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.618 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.618 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.618 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.619 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.619 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.619 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.619 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.619 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.619 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.619 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.619 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.619 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.619 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.619 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.619 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.619 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.619 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.619 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.620 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.620 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.620 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.620 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.620 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.620 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.620 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.620 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.620 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.620 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.620 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.620 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.620 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.621 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.621 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.621 INFO:teuthology.orchestra.run.smithi032.stdout: "predict_interval": { 2024-04-25T00:44:44.621 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "predict_interval", 2024-04-25T00:44:44.621 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.621 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.621 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.621 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "86400", 2024-04-25T00:44:44.621 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.621 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.621 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.621 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.621 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.621 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.622 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.622 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.622 INFO:teuthology.orchestra.run.smithi032.stdout: "predictor_model": { 2024-04-25T00:44:44.622 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "predictor_model", 2024-04-25T00:44:44.622 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.622 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.622 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.622 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "prophetstor", 2024-04-25T00:44:44.622 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.622 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.622 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.622 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.622 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.622 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.622 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.623 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.623 INFO:teuthology.orchestra.run.smithi032.stdout: "sleep_interval": { 2024-04-25T00:44:44.623 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "sleep_interval", 2024-04-25T00:44:44.623 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.623 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.623 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.623 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "600", 2024-04-25T00:44:44.623 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.623 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.623 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.623 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.623 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.623 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.623 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.624 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.624 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.624 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.624 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.624 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "influx", 2024-04-25T00:44:44.624 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": false, 2024-04-25T00:44:44.624 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "influxdb python module not found", 2024-04-25T00:44:44.624 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.624 INFO:teuthology.orchestra.run.smithi032.stdout: "batch_size": { 2024-04-25T00:44:44.624 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "batch_size", 2024-04-25T00:44:44.624 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.624 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.624 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.624 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "5000", 2024-04-25T00:44:44.624 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.625 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.625 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.625 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "How big batches of data points should be when sending to InfluxDB.", 2024-04-25T00:44:44.625 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.625 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.625 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.625 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.625 INFO:teuthology.orchestra.run.smithi032.stdout: "database": { 2024-04-25T00:44:44.625 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "database", 2024-04-25T00:44:44.625 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.625 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.625 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.625 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "ceph", 2024-04-25T00:44:44.626 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.626 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.626 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.626 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.", 2024-04-25T00:44:44.626 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.626 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.626 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.626 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.626 INFO:teuthology.orchestra.run.smithi032.stdout: "hostname": { 2024-04-25T00:44:44.626 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "hostname", 2024-04-25T00:44:44.626 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.626 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.626 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.626 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.626 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.627 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.627 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.627 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "InfluxDB server hostname", 2024-04-25T00:44:44.627 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.627 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.627 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.627 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.627 INFO:teuthology.orchestra.run.smithi032.stdout: "interval": { 2024-04-25T00:44:44.627 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "interval", 2024-04-25T00:44:44.627 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.627 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.627 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.627 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "30", 2024-04-25T00:44:44.627 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "5", 2024-04-25T00:44:44.627 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.628 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.628 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Time between reports to InfluxDB. Default 30 seconds.", 2024-04-25T00:44:44.628 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.628 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.628 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.628 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.628 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.628 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.628 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.628 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.628 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.628 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.628 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.628 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.629 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.629 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.629 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.629 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.629 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.629 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.629 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.629 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.629 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.629 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.629 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.629 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.629 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.629 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.629 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.630 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.630 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.630 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.630 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.630 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.630 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.630 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.630 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.630 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.630 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.630 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.630 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.630 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.630 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.631 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.631 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.631 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.631 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.631 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.631 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.631 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.631 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.631 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.631 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.631 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.631 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.631 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.631 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.632 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.632 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.632 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.632 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.632 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.632 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.632 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.632 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.632 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.632 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.632 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.632 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.632 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.632 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.632 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.633 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.633 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.633 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.633 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.633 INFO:teuthology.orchestra.run.smithi032.stdout: "password": { 2024-04-25T00:44:44.633 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "password", 2024-04-25T00:44:44.633 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.633 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.633 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.633 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.633 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.633 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.633 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.633 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "password of InfluxDB server user", 2024-04-25T00:44:44.633 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.634 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.634 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.634 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.634 INFO:teuthology.orchestra.run.smithi032.stdout: "port": { 2024-04-25T00:44:44.634 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "port", 2024-04-25T00:44:44.634 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.634 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.634 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.634 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "8086", 2024-04-25T00:44:44.634 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.634 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.634 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.634 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "InfluxDB server port", 2024-04-25T00:44:44.634 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.635 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.635 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.635 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.635 INFO:teuthology.orchestra.run.smithi032.stdout: "ssl": { 2024-04-25T00:44:44.635 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ssl", 2024-04-25T00:44:44.635 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.635 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.635 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.635 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "false", 2024-04-25T00:44:44.635 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.635 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.635 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.635 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Use https connection for InfluxDB server. Use \"true\" or \"false\".", 2024-04-25T00:44:44.635 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.635 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.636 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.636 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.636 INFO:teuthology.orchestra.run.smithi032.stdout: "threads": { 2024-04-25T00:44:44.636 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "threads", 2024-04-25T00:44:44.636 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.636 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.636 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.636 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "5", 2024-04-25T00:44:44.636 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "1", 2024-04-25T00:44:44.636 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "32", 2024-04-25T00:44:44.636 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.636 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "How many worker threads should be spawned for sending data to InfluxDB.", 2024-04-25T00:44:44.636 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.636 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.637 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.637 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.637 INFO:teuthology.orchestra.run.smithi032.stdout: "username": { 2024-04-25T00:44:44.637 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "username", 2024-04-25T00:44:44.637 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.637 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.637 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.637 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.637 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.637 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.637 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.637 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "username of InfluxDB server user", 2024-04-25T00:44:44.637 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.637 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.638 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.638 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.638 INFO:teuthology.orchestra.run.smithi032.stdout: "verify_ssl": { 2024-04-25T00:44:44.638 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "verify_ssl", 2024-04-25T00:44:44.638 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.638 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.638 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.638 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "true", 2024-04-25T00:44:44.638 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.638 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.638 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.638 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Verify https cert for InfluxDB server. Use \"true\" or \"false\".", 2024-04-25T00:44:44.638 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.639 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.639 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.639 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.639 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.639 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.639 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.639 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "insights", 2024-04-25T00:44:44.639 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.639 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.639 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.639 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.639 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.639 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.639 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.640 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.640 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.640 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.640 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.640 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.640 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.640 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.640 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.640 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.640 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.640 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.640 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.640 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.640 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.641 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.641 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.641 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.641 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.641 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.641 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.641 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.641 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.641 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.641 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.641 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.641 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.641 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.641 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.642 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.642 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.642 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.642 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.642 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.642 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.642 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.642 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.642 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.642 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.642 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.642 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.642 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.643 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.643 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.643 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.643 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.643 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.643 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.643 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.643 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.643 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.643 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.643 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.643 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.643 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.643 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.643 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.644 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.644 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.644 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.644 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.644 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.644 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.644 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.644 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.644 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.644 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.644 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.644 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.644 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.644 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "iostat", 2024-04-25T00:44:44.644 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.645 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.645 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.645 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.645 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.645 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.645 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.645 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.645 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.645 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.645 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.645 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.645 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.645 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.645 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.645 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.646 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.646 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.646 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.646 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.646 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.646 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.646 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.646 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.646 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.646 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.646 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.646 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.646 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.646 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.647 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.647 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.647 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.647 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.647 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.647 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.647 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.647 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.647 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.647 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.647 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.647 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.647 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.647 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.647 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.647 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.648 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.648 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.648 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.648 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.648 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.648 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.648 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.648 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.648 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.648 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.648 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.648 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.648 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.648 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.648 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.648 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.649 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.649 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.649 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.649 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.649 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.649 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.649 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.649 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.649 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.649 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.649 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.649 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.649 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.649 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.649 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "k8sevents", 2024-04-25T00:44:44.650 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.650 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.650 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.650 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph_event_retention_days": { 2024-04-25T00:44:44.650 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ceph_event_retention_days", 2024-04-25T00:44:44.650 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.650 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.650 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.650 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "7", 2024-04-25T00:44:44.650 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.650 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.650 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.650 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Days to hold ceph event information within local cache", 2024-04-25T00:44:44.650 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.651 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.651 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.651 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.651 INFO:teuthology.orchestra.run.smithi032.stdout: "config_check_secs": { 2024-04-25T00:44:44.651 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "config_check_secs", 2024-04-25T00:44:44.651 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.651 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.651 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.651 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10", 2024-04-25T00:44:44.651 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "10", 2024-04-25T00:44:44.651 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.651 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.651 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "interval (secs) to check for cluster configuration changes", 2024-04-25T00:44:44.652 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.652 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.652 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.652 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.652 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.652 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.652 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.652 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.652 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.652 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.652 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.652 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.652 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.652 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.653 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.653 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.653 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.653 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.653 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.653 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.653 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.653 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.653 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.653 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.653 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.653 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.653 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.653 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.653 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.653 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.653 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.654 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.654 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.654 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.654 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.654 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.654 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.654 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.654 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.654 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.654 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.654 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.654 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.654 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.654 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.655 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.655 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.655 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.655 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.655 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.655 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.655 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.655 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.655 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.655 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.655 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.655 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.655 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.655 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.655 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.656 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.656 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.656 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.656 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.656 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.656 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.656 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.656 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.656 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.656 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.656 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.656 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.656 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.656 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.656 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.656 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.657 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.657 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "localpool", 2024-04-25T00:44:44.657 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.657 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.657 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.657 INFO:teuthology.orchestra.run.smithi032.stdout: "failure_domain": { 2024-04-25T00:44:44.657 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "failure_domain", 2024-04-25T00:44:44.657 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.657 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.657 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.657 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "host", 2024-04-25T00:44:44.657 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.657 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.657 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.657 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "failure domain for any created local pool", 2024-04-25T00:44:44.657 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "what failure domain we should separate data replicas across.", 2024-04-25T00:44:44.657 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.657 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.658 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.658 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.658 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.658 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.658 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.658 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.658 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.658 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.658 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.658 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.658 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.658 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.658 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.658 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.659 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.659 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.659 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.659 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.659 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.659 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.659 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.659 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.659 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.659 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.659 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.659 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.659 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.659 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.659 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.659 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.659 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.660 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.660 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.660 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.660 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.660 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.660 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.660 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.660 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.660 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.660 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.660 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.660 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.660 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.660 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.661 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.661 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.661 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.661 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.661 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.661 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.661 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.661 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.661 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.661 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.661 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.661 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.661 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.661 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.661 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.662 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.662 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.662 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.662 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.662 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.662 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.662 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.662 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.662 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.662 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.662 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.662 INFO:teuthology.orchestra.run.smithi032.stdout: "min_size": { 2024-04-25T00:44:44.662 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "min_size", 2024-04-25T00:44:44.662 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.662 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.663 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.663 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.663 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.663 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.663 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.663 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "default min_size for any created local pool", 2024-04-25T00:44:44.663 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "value to set min_size to (unchanged from Ceph's default if this option is not set)", 2024-04-25T00:44:44.663 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.663 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.663 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.663 INFO:teuthology.orchestra.run.smithi032.stdout: "num_rep": { 2024-04-25T00:44:44.663 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "num_rep", 2024-04-25T00:44:44.663 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.663 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.663 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.664 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "3", 2024-04-25T00:44:44.664 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.664 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.664 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.664 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "default replica count for any created local pool", 2024-04-25T00:44:44.664 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.664 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.664 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.664 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.664 INFO:teuthology.orchestra.run.smithi032.stdout: "pg_num": { 2024-04-25T00:44:44.664 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "pg_num", 2024-04-25T00:44:44.664 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.664 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.664 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.665 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "128", 2024-04-25T00:44:44.665 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.665 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.665 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.665 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "default pg_num for any created local pool", 2024-04-25T00:44:44.665 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.665 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.665 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.665 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.665 INFO:teuthology.orchestra.run.smithi032.stdout: "prefix": { 2024-04-25T00:44:44.665 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "prefix", 2024-04-25T00:44:44.665 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.665 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.665 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.666 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.666 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.666 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.666 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.666 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "name prefix for any created local pool", 2024-04-25T00:44:44.666 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.666 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.666 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.666 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.666 INFO:teuthology.orchestra.run.smithi032.stdout: "subtree": { 2024-04-25T00:44:44.666 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "subtree", 2024-04-25T00:44:44.666 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.666 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.666 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.667 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "rack", 2024-04-25T00:44:44.667 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.667 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.667 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.667 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "CRUSH level for which to create a local pool", 2024-04-25T00:44:44.667 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "which CRUSH subtree type the module should create a pool for.", 2024-04-25T00:44:44.667 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.667 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.667 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.667 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.667 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.667 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.667 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "mds_autoscaler", 2024-04-25T00:44:44.667 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.667 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.667 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.668 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.668 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.668 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.668 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.668 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.668 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.668 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.668 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.668 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.668 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.668 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.668 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.668 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.668 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.668 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.668 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.669 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.669 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.669 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.669 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.669 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.669 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.669 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.669 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.669 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.669 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.669 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.669 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.669 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.669 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.669 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.669 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.669 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.670 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.670 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.670 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.670 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.670 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.670 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.670 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.670 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.670 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.670 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.670 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.670 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.670 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.670 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.670 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.671 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.671 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.671 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.671 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.671 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.671 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.671 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.671 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.671 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.671 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.671 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.671 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.671 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.671 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.671 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.671 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.672 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.672 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.672 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.672 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.672 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.672 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.672 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.672 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.672 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.672 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "mirroring", 2024-04-25T00:44:44.672 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.672 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.672 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.672 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.672 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.673 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.673 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.673 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.673 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.673 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.673 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.673 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.673 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.673 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.673 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.673 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.673 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.673 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.673 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.673 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.673 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.673 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.674 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.674 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.674 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.674 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.674 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.674 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.674 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.674 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.674 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.674 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.674 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.674 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.674 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.674 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.675 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.675 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.675 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.675 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.675 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.675 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.675 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.675 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.675 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.675 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.675 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.675 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.675 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.676 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.676 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.676 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.676 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.676 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.676 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.676 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.676 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.676 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.676 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.676 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.676 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.676 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.676 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.677 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.677 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.677 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.677 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.677 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.677 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.677 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.677 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.677 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.677 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.677 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.677 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.677 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.677 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "nfs", 2024-04-25T00:44:44.678 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.678 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.678 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.678 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.678 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.678 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.678 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.678 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.678 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.678 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.678 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.678 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.678 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.678 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.679 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.679 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.679 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.679 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.679 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.679 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.679 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.679 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.679 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.679 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.679 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.679 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.679 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.679 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.679 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.680 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.680 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.680 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.680 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.680 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.680 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.680 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.680 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.680 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.680 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.680 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.680 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.680 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.680 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.681 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.681 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.681 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.681 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.681 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.681 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.681 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.681 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.681 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.681 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.681 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.681 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.681 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.681 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.681 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.682 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.682 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.682 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.682 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.682 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.682 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.682 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.682 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.682 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.682 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.682 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.682 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.682 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.682 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.683 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.683 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.683 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.683 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.683 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "orchestrator", 2024-04-25T00:44:44.683 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.683 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.683 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.683 INFO:teuthology.orchestra.run.smithi032.stdout: "fail_fs": { 2024-04-25T00:44:44.683 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "fail_fs", 2024-04-25T00:44:44.683 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.683 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.683 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.683 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.684 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.684 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.684 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.684 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Fail filesystem for rapid multi-rank mds upgrade", 2024-04-25T00:44:44.684 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.684 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.684 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.684 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.684 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.684 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.684 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.684 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.684 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.684 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.684 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.685 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.685 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.685 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.685 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.685 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.685 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.685 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.685 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.685 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.685 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.685 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.685 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.685 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.685 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.685 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.685 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.686 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.686 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.686 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.686 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.686 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.686 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.686 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.686 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.686 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.686 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.686 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.686 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.686 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.686 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.686 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.687 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.687 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.687 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.687 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.687 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.687 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.687 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.687 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.687 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.687 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.687 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.687 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.687 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.687 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.688 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.688 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.688 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.688 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.688 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.688 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.688 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.688 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.688 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.688 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.688 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.688 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.688 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.688 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.688 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.689 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.689 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.689 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.689 INFO:teuthology.orchestra.run.smithi032.stdout: "orchestrator": { 2024-04-25T00:44:44.689 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "orchestrator", 2024-04-25T00:44:44.689 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.689 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.689 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.689 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.689 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.689 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.689 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.689 INFO:teuthology.orchestra.run.smithi032.stdout: "cephadm", 2024-04-25T00:44:44.689 INFO:teuthology.orchestra.run.smithi032.stdout: "rook", 2024-04-25T00:44:44.689 INFO:teuthology.orchestra.run.smithi032.stdout: "test_orchestrator" 2024-04-25T00:44:44.689 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.690 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Orchestrator backend", 2024-04-25T00:44:44.690 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.690 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.690 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.690 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.690 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.690 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.690 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.690 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "osd_perf_query", 2024-04-25T00:44:44.690 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.690 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.690 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.690 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.690 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.690 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.691 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.691 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.691 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.691 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.691 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.691 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.691 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.691 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.691 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.691 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.691 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.691 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.691 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.691 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.691 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.691 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.692 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.692 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.692 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.692 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.692 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.692 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.692 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.692 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.692 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.692 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.692 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.692 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.692 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.692 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.692 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.692 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.693 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.693 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.693 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.693 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.693 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.693 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.693 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.693 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.693 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.693 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.693 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.693 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.693 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.693 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.693 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.693 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.693 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.694 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.694 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.694 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.694 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.694 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.694 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.694 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.694 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.694 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.694 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.694 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.694 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.694 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.694 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.695 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.695 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.695 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.695 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.695 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.695 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.695 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.695 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "osd_support", 2024-04-25T00:44:44.695 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.695 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.695 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.695 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.695 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.695 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.695 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.695 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.696 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.696 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.696 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.696 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.696 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.696 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.696 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.696 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.696 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.696 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.696 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.696 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.696 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.696 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.696 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.696 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.696 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.697 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.697 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.697 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.697 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.697 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.697 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.697 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.697 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.697 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.697 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.697 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.697 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.697 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.697 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.697 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.697 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.697 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.698 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.698 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.698 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.698 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.698 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.698 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.698 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.698 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.698 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.698 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.698 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.698 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.698 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.698 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.698 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.699 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.699 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.699 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.699 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.699 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.699 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.699 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.699 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.699 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.699 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.699 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.699 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.699 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.699 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.699 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.699 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.699 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.700 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.700 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.700 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "pg_autoscaler", 2024-04-25T00:44:44.700 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.700 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.700 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.700 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.700 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.700 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.700 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.700 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.700 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.700 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.700 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.700 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.700 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.700 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.700 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.701 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.701 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.701 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.701 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.701 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.701 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.701 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.701 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.701 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.701 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.701 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.701 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.701 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.701 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.701 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.701 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.701 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.702 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.702 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.702 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.702 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.702 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.702 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.702 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.702 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.702 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.702 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.702 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.702 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.702 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.703 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.703 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.703 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.703 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.703 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.703 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.703 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.703 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.703 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.703 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.703 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.703 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.703 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.703 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.703 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.703 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.704 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.704 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.704 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.704 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.704 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.704 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.704 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.704 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.704 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.704 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.704 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.704 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.704 INFO:teuthology.orchestra.run.smithi032.stdout: "sleep_interval": { 2024-04-25T00:44:44.704 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "sleep_interval", 2024-04-25T00:44:44.704 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.705 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.705 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.705 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "60", 2024-04-25T00:44:44.705 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.705 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.705 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.705 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.705 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.705 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.705 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.705 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.705 INFO:teuthology.orchestra.run.smithi032.stdout: "threshold": { 2024-04-25T00:44:44.705 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "threshold", 2024-04-25T00:44:44.705 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "float", 2024-04-25T00:44:44.705 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.705 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.706 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "3.0", 2024-04-25T00:44:44.706 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "1.0", 2024-04-25T00:44:44.706 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.706 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.706 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "scaling threshold", 2024-04-25T00:44:44.706 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0", 2024-04-25T00:44:44.706 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.706 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.706 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.706 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.706 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.706 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.706 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "progress", 2024-04-25T00:44:44.706 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.707 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.707 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.707 INFO:teuthology.orchestra.run.smithi032.stdout: "allow_pg_recovery_event": { 2024-04-25T00:44:44.707 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "allow_pg_recovery_event", 2024-04-25T00:44:44.707 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.707 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.707 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.707 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.707 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.707 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.707 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.707 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "allow the module to show pg recovery progress", 2024-04-25T00:44:44.707 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.707 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.708 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.708 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.708 INFO:teuthology.orchestra.run.smithi032.stdout: "enabled": { 2024-04-25T00:44:44.708 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "enabled", 2024-04-25T00:44:44.708 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.708 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.708 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.708 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.708 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.708 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.708 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.708 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.708 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.708 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.709 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.709 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.709 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.709 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.709 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.709 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.709 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.709 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.709 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.709 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.709 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.709 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.709 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.709 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.709 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.710 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.710 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.710 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.710 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.710 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.710 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.710 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.710 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.710 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.710 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.710 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.710 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.710 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.710 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.711 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.711 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.711 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.711 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.711 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.711 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.711 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.711 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.711 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.711 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.711 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.711 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.711 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.711 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.711 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.712 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.712 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.712 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.712 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.712 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.712 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.712 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.712 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.712 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.712 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.712 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.712 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.712 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.712 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.712 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.713 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.713 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.713 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.713 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.713 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.713 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.713 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.713 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.713 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.713 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.713 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.713 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.713 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.713 INFO:teuthology.orchestra.run.smithi032.stdout: "max_completed_events": { 2024-04-25T00:44:44.713 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "max_completed_events", 2024-04-25T00:44:44.713 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.714 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.714 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.714 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "50", 2024-04-25T00:44:44.714 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.714 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.714 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.714 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "number of past completed events to remember", 2024-04-25T00:44:44.714 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.714 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.714 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.714 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.714 INFO:teuthology.orchestra.run.smithi032.stdout: "sleep_interval": { 2024-04-25T00:44:44.714 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "sleep_interval", 2024-04-25T00:44:44.714 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.715 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.715 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.715 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "5", 2024-04-25T00:44:44.715 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.715 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.715 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.715 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "how long the module is going to sleep", 2024-04-25T00:44:44.715 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.715 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.715 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.715 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.715 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.715 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.716 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.716 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "prometheus", 2024-04-25T00:44:44.716 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.716 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.716 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.716 INFO:teuthology.orchestra.run.smithi032.stdout: "cache": { 2024-04-25T00:44:44.716 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "cache", 2024-04-25T00:44:44.716 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.716 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.716 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.716 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.716 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.716 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.716 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.716 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.717 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.717 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.717 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.717 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.717 INFO:teuthology.orchestra.run.smithi032.stdout: "exclude_perf_counters": { 2024-04-25T00:44:44.717 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "exclude_perf_counters", 2024-04-25T00:44:44.717 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.717 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.717 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.717 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.717 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.717 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.717 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.717 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Do not include perf-counters in the metrics output", 2024-04-25T00:44:44.717 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.", 2024-04-25T00:44:44.718 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.718 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.718 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.718 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.718 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.718 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.718 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.718 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.718 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.718 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.719 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.719 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.719 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.719 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.719 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.719 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.719 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.719 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.719 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.719 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.719 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.719 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.719 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.719 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.720 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.720 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.720 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.720 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.720 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.720 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.720 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.720 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.720 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.720 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.720 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.720 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.721 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.721 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.721 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.721 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.721 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.721 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.721 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.721 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.721 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.721 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.721 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.721 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.721 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.721 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.722 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.722 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.722 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.722 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.722 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.722 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.722 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.722 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.722 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.722 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.722 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.722 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.722 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.722 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.722 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.723 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.723 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.723 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.723 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.723 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.723 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.723 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.723 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.723 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_stats_pools": { 2024-04-25T00:44:44.723 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rbd_stats_pools", 2024-04-25T00:44:44.723 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.723 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.723 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.723 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.723 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.724 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.724 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.724 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.724 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.724 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.724 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.724 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.724 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_stats_pools_refresh_interval": { 2024-04-25T00:44:44.724 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rbd_stats_pools_refresh_interval", 2024-04-25T00:44:44.724 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.724 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.724 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.724 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "300", 2024-04-25T00:44:44.724 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.724 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.724 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.725 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.725 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.725 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.725 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.725 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.725 INFO:teuthology.orchestra.run.smithi032.stdout: "scrape_interval": { 2024-04-25T00:44:44.725 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "scrape_interval", 2024-04-25T00:44:44.725 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "float", 2024-04-25T00:44:44.725 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.725 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.725 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "15.0", 2024-04-25T00:44:44.725 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.725 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.725 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.725 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.726 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.726 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.726 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.726 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.726 INFO:teuthology.orchestra.run.smithi032.stdout: "server_addr": { 2024-04-25T00:44:44.726 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "server_addr", 2024-04-25T00:44:44.726 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.726 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.726 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.726 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "::", 2024-04-25T00:44:44.726 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.726 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.726 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.726 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "the IPv4 or IPv6 address on which the module listens for HTTP requests", 2024-04-25T00:44:44.726 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.727 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.727 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.727 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.727 INFO:teuthology.orchestra.run.smithi032.stdout: "server_port": { 2024-04-25T00:44:44.727 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "server_port", 2024-04-25T00:44:44.727 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.727 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.727 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.727 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "9283", 2024-04-25T00:44:44.727 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.727 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.727 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.727 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "the port on which the module listens for HTTP requests", 2024-04-25T00:44:44.727 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.727 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.728 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.728 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.728 INFO:teuthology.orchestra.run.smithi032.stdout: "stale_cache_strategy": { 2024-04-25T00:44:44.728 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "stale_cache_strategy", 2024-04-25T00:44:44.728 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.728 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.728 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.728 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "log", 2024-04-25T00:44:44.728 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.728 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.729 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.729 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.729 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.729 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.729 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.729 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.729 INFO:teuthology.orchestra.run.smithi032.stdout: "standby_behaviour": { 2024-04-25T00:44:44.729 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "standby_behaviour", 2024-04-25T00:44:44.729 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.729 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.729 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.729 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "default", 2024-04-25T00:44:44.729 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.730 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.730 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.730 INFO:teuthology.orchestra.run.smithi032.stdout: "default", 2024-04-25T00:44:44.730 INFO:teuthology.orchestra.run.smithi032.stdout: "error" 2024-04-25T00:44:44.730 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.730 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.730 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.730 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.730 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.730 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.730 INFO:teuthology.orchestra.run.smithi032.stdout: "standby_error_status_code": { 2024-04-25T00:44:44.730 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "standby_error_status_code", 2024-04-25T00:44:44.730 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.730 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.731 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.731 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "500", 2024-04-25T00:44:44.731 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "400", 2024-04-25T00:44:44.731 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "599", 2024-04-25T00:44:44.731 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.731 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.731 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.731 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.731 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.731 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.731 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.731 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.731 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.731 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rbd_support", 2024-04-25T00:44:44.731 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.732 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.732 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.732 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.732 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.732 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.732 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.732 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.732 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.732 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.732 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.732 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.732 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.732 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.732 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.732 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.733 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.733 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.733 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.733 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.733 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.733 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.733 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.733 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.733 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.733 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.733 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.733 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.733 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.733 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.733 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.734 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.734 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.734 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.734 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.734 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.734 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.734 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.734 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.734 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.734 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.734 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.734 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.734 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.734 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.735 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.735 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.735 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.735 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.735 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.735 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.735 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.735 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.735 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.735 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.735 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.735 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.735 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.735 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.736 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.736 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.736 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.736 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.736 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.736 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.736 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.736 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.736 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.736 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.736 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.736 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.736 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.736 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.736 INFO:teuthology.orchestra.run.smithi032.stdout: "max_concurrent_snap_create": { 2024-04-25T00:44:44.737 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "max_concurrent_snap_create", 2024-04-25T00:44:44.737 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.737 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.737 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.737 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10", 2024-04-25T00:44:44.737 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.737 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.737 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.737 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.737 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.737 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.737 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.737 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.737 INFO:teuthology.orchestra.run.smithi032.stdout: "mirror_snapshot_schedule": { 2024-04-25T00:44:44.737 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "mirror_snapshot_schedule", 2024-04-25T00:44:44.738 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.738 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.738 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.738 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.738 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.738 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.738 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.738 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.738 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.738 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.738 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.738 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.738 INFO:teuthology.orchestra.run.smithi032.stdout: "trash_purge_schedule": { 2024-04-25T00:44:44.738 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "trash_purge_schedule", 2024-04-25T00:44:44.739 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.739 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.739 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.739 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.739 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.739 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.739 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.739 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.739 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.739 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.739 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.739 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.739 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.739 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.740 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.740 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "restful", 2024-04-25T00:44:44.740 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.740 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.740 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.740 INFO:teuthology.orchestra.run.smithi032.stdout: "enable_auth": { 2024-04-25T00:44:44.740 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "enable_auth", 2024-04-25T00:44:44.740 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.740 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.740 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.740 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.740 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.740 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.740 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.740 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.741 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.741 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.741 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.741 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.741 INFO:teuthology.orchestra.run.smithi032.stdout: "key_file": { 2024-04-25T00:44:44.741 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "key_file", 2024-04-25T00:44:44.741 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.741 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.741 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.741 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.741 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.741 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.741 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.741 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.741 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.742 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.742 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.742 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.742 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.742 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.742 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.742 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.742 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.742 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.742 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.743 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.743 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.743 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.743 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.743 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.743 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.743 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.743 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.743 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.743 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.743 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.743 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.743 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.743 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.744 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.744 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.744 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.744 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.744 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.744 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.744 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.744 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.744 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.744 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.744 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.744 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.744 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.744 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.745 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.745 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.745 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.745 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.745 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.745 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.745 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.745 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.745 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.745 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.745 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.745 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.745 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.745 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.745 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.745 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.746 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.746 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.746 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.746 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.746 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.746 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.746 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.746 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.746 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.746 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.746 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.746 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.746 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.747 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.747 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.747 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.747 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.747 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.747 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.747 INFO:teuthology.orchestra.run.smithi032.stdout: "server_addr": { 2024-04-25T00:44:44.747 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "server_addr", 2024-04-25T00:44:44.747 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.747 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.747 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.747 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.747 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.747 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.748 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.748 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.748 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.748 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.748 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.748 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.748 INFO:teuthology.orchestra.run.smithi032.stdout: "server_port": { 2024-04-25T00:44:44.748 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "server_port", 2024-04-25T00:44:44.748 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.748 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.748 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.748 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.748 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.749 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.749 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.749 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.749 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.749 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.749 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.749 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.749 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.749 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.749 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.749 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rgw", 2024-04-25T00:44:44.749 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.749 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.749 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.750 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.750 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.750 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.750 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.750 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.750 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.750 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.750 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.750 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.750 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.750 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.750 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.750 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.750 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.750 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.751 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.751 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.751 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.751 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.751 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.751 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.751 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.751 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.751 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.751 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.751 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.751 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.751 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.751 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.752 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.752 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.752 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.752 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.752 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.752 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.752 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.752 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.752 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.752 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.752 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.752 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.752 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.752 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.752 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.753 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.753 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.753 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.753 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.753 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.753 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.753 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.753 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.753 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.753 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.753 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.753 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.753 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.753 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.753 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.754 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.754 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.754 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.754 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.754 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.754 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.754 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.754 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.754 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.754 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.754 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.754 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.754 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.754 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.755 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rook", 2024-04-25T00:44:44.755 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.755 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.755 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.755 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.755 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.755 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.755 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.755 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.755 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.755 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.755 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.755 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.755 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.755 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.756 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.756 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.756 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.756 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.756 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.756 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.756 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.756 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.756 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.756 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.756 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.756 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.756 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.756 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.757 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.757 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.757 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.757 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.757 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.757 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.757 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.757 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.757 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.757 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.757 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.757 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.757 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.757 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.758 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.758 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.758 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.758 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.758 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.758 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.758 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.758 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.758 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.758 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.758 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.758 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.758 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.758 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.758 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.759 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.759 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.759 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.759 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.759 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.759 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.759 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.759 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.759 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.759 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.759 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.759 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.759 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.759 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.760 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.760 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.760 INFO:teuthology.orchestra.run.smithi032.stdout: "storage_class": { 2024-04-25T00:44:44.760 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "storage_class", 2024-04-25T00:44:44.760 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.760 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.760 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.760 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "local", 2024-04-25T00:44:44.760 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.760 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.760 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.760 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "storage class name for LSO-discovered PVs", 2024-04-25T00:44:44.760 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.760 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.761 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.761 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.761 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.761 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.761 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.761 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "selftest", 2024-04-25T00:44:44.761 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.761 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.761 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.761 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.761 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.761 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.761 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.761 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.761 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.762 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.762 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.762 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.762 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.762 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.762 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.762 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.762 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.762 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.762 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.762 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.762 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.762 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.763 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.763 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.763 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.763 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.763 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.763 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.763 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.763 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.763 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.763 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.763 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.763 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.763 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.763 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.763 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.765 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.765 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.765 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.766 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.766 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.766 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.766 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.766 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.766 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.766 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.766 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.766 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.766 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.766 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.766 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.766 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.766 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.767 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.767 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.767 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.767 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.767 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.767 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.767 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.767 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.767 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.767 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.767 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.767 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.767 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.767 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.768 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.768 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.768 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.768 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.768 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.768 INFO:teuthology.orchestra.run.smithi032.stdout: "roption1": { 2024-04-25T00:44:44.768 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "roption1", 2024-04-25T00:44:44.768 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.768 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.768 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.768 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.768 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.768 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.768 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.768 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.769 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.769 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.769 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.769 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.769 INFO:teuthology.orchestra.run.smithi032.stdout: "roption2": { 2024-04-25T00:44:44.769 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "roption2", 2024-04-25T00:44:44.769 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.769 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.769 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.769 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "xyz", 2024-04-25T00:44:44.769 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.769 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.769 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.769 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.769 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.770 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.770 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.770 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.770 INFO:teuthology.orchestra.run.smithi032.stdout: "rwoption1": { 2024-04-25T00:44:44.770 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rwoption1", 2024-04-25T00:44:44.770 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.770 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.770 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.770 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.770 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.770 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.770 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.770 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.770 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.770 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.771 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.771 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.771 INFO:teuthology.orchestra.run.smithi032.stdout: "rwoption2": { 2024-04-25T00:44:44.771 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rwoption2", 2024-04-25T00:44:44.771 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.771 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.771 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.771 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.771 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.771 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.771 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.771 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.771 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.772 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.772 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.772 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.772 INFO:teuthology.orchestra.run.smithi032.stdout: "rwoption3": { 2024-04-25T00:44:44.772 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rwoption3", 2024-04-25T00:44:44.772 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "float", 2024-04-25T00:44:44.772 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.772 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.772 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.772 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.772 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.772 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.772 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.772 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.772 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.773 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.773 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.773 INFO:teuthology.orchestra.run.smithi032.stdout: "rwoption4": { 2024-04-25T00:44:44.773 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rwoption4", 2024-04-25T00:44:44.773 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.773 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.773 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.773 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.773 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.773 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.773 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.773 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.773 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.773 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.774 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.774 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.774 INFO:teuthology.orchestra.run.smithi032.stdout: "rwoption5": { 2024-04-25T00:44:44.774 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rwoption5", 2024-04-25T00:44:44.774 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.774 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.774 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.774 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.774 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.774 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.774 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.774 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.774 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.774 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.775 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.775 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.775 INFO:teuthology.orchestra.run.smithi032.stdout: "rwoption6": { 2024-04-25T00:44:44.775 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rwoption6", 2024-04-25T00:44:44.775 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.775 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.775 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.775 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.775 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.775 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.775 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.775 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.775 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.775 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.776 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.776 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.776 INFO:teuthology.orchestra.run.smithi032.stdout: "rwoption7": { 2024-04-25T00:44:44.776 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rwoption7", 2024-04-25T00:44:44.776 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.776 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.776 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.776 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.776 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "1", 2024-04-25T00:44:44.776 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "42", 2024-04-25T00:44:44.776 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.776 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.776 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.776 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.776 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.777 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.777 INFO:teuthology.orchestra.run.smithi032.stdout: "testkey": { 2024-04-25T00:44:44.777 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "testkey", 2024-04-25T00:44:44.777 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.777 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.777 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.777 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.777 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.777 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.777 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.777 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.777 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.777 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.777 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.778 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.778 INFO:teuthology.orchestra.run.smithi032.stdout: "testlkey": { 2024-04-25T00:44:44.778 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "testlkey", 2024-04-25T00:44:44.778 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.778 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.778 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.778 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.778 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.778 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.778 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.778 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.778 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.778 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.778 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.779 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.779 INFO:teuthology.orchestra.run.smithi032.stdout: "testnewline": { 2024-04-25T00:44:44.779 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "testnewline", 2024-04-25T00:44:44.779 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.779 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.779 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.779 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.779 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.779 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.779 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.779 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.779 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.779 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.779 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.779 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.780 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.780 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.780 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.780 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "snap_schedule", 2024-04-25T00:44:44.780 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.780 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.780 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.780 INFO:teuthology.orchestra.run.smithi032.stdout: "allow_m_granularity": { 2024-04-25T00:44:44.780 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "allow_m_granularity", 2024-04-25T00:44:44.780 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.780 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.780 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.780 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.780 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.780 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.781 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.781 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "allow minute scheduled snapshots", 2024-04-25T00:44:44.781 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.781 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.781 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.781 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.781 INFO:teuthology.orchestra.run.smithi032.stdout: "dump_on_update": { 2024-04-25T00:44:44.781 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "dump_on_update", 2024-04-25T00:44:44.781 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.781 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.781 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.781 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.781 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.781 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.781 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.782 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "dump database to debug log on update", 2024-04-25T00:44:44.782 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.782 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.782 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.782 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.782 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.782 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.782 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.782 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.782 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.782 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.782 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.782 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.782 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.783 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.783 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.783 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.783 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.783 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.783 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.783 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.783 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.783 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.783 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.783 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.783 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.783 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.783 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.783 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.784 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.784 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.784 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.784 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.784 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.784 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.784 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.784 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.784 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.784 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.784 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.784 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.784 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.784 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.785 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.785 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.785 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.785 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.785 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.785 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.785 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.785 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.785 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.785 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.785 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.785 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.785 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.785 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.785 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.786 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.786 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.786 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.786 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.786 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.786 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.786 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.786 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.786 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.786 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.786 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.786 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.786 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.786 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.786 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.787 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.787 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.787 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.787 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.787 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.787 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "stats", 2024-04-25T00:44:44.787 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.787 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.787 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.787 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.787 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.787 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.787 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.787 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.787 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.788 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.788 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.788 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.788 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.788 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.788 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.788 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.788 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.788 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.788 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.788 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.788 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.788 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.788 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.789 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.789 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.789 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.789 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.789 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.789 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.789 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.789 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.789 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.789 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.789 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.789 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.789 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.789 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.789 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.790 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.790 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.790 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.790 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.790 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.790 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.790 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.790 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.790 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.790 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.790 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.790 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.790 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.790 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.791 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.791 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.791 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.791 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.791 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.791 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.791 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.791 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.791 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.791 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.791 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.791 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.791 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.791 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.791 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.792 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.792 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.792 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.792 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.792 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.792 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.792 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.792 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.792 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.792 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "status", 2024-04-25T00:44:44.792 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.792 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.792 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.792 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.792 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.793 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.793 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.793 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.793 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.793 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.793 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.793 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.793 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.793 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.793 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.793 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.793 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.793 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.793 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.793 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.794 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.794 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.794 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.794 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.794 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.794 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.794 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.794 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.794 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.794 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.794 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.794 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.794 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.794 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.795 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.795 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.795 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.795 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.795 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.795 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.795 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.795 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.795 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.795 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.795 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.795 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.795 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.795 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.795 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.795 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.796 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.796 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.796 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.796 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.796 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.796 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.796 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.796 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.796 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.796 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.796 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.796 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.796 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.796 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.796 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.797 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.797 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.797 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.797 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.797 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.797 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.797 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.797 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.797 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.797 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.797 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.797 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "telegraf", 2024-04-25T00:44:44.797 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.797 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.798 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.798 INFO:teuthology.orchestra.run.smithi032.stdout: "address": { 2024-04-25T00:44:44.798 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "address", 2024-04-25T00:44:44.798 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.798 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.798 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.798 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "unixgram:///tmp/telegraf.sock", 2024-04-25T00:44:44.798 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.798 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.798 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.798 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.798 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.798 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.799 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.799 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.799 INFO:teuthology.orchestra.run.smithi032.stdout: "interval": { 2024-04-25T00:44:44.799 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "interval", 2024-04-25T00:44:44.799 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.799 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.799 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.799 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "15", 2024-04-25T00:44:44.799 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.799 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.799 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.799 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.799 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.799 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.799 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.800 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.800 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.800 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.800 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.800 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.800 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.800 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.800 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.800 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.800 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.800 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.800 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.800 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.800 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.801 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.801 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.801 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.801 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.801 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.801 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.801 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.801 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.801 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.801 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.801 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.801 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.801 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.801 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.801 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.802 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.802 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.802 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.802 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.802 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.802 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.802 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.802 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.802 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.802 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.802 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.802 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.802 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.802 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.803 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.803 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.803 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.803 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.803 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.803 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.803 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.803 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.803 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.803 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.803 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.803 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.803 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.803 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.804 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.804 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.804 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.804 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.804 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.804 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.804 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.804 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.804 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.804 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.804 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.804 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.804 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.804 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.805 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.805 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.805 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.805 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "telemetry", 2024-04-25T00:44:44.805 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.805 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.805 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.805 INFO:teuthology.orchestra.run.smithi032.stdout: "channel_basic": { 2024-04-25T00:44:44.805 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "channel_basic", 2024-04-25T00:44:44.805 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.805 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.805 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.805 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.805 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.805 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.806 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.806 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Share basic cluster information (size, version)", 2024-04-25T00:44:44.806 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.806 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.806 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.806 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.806 INFO:teuthology.orchestra.run.smithi032.stdout: "channel_crash": { 2024-04-25T00:44:44.806 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "channel_crash", 2024-04-25T00:44:44.806 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.806 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.806 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.806 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.806 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.806 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.807 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.807 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Share metadata about Ceph daemon crashes (version, stack straces, etc)", 2024-04-25T00:44:44.807 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.807 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.807 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.807 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.807 INFO:teuthology.orchestra.run.smithi032.stdout: "channel_device": { 2024-04-25T00:44:44.807 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "channel_device", 2024-04-25T00:44:44.807 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.807 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.807 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.807 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.807 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.807 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.807 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.807 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)", 2024-04-25T00:44:44.808 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.808 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.808 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.808 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.808 INFO:teuthology.orchestra.run.smithi032.stdout: "channel_ident": { 2024-04-25T00:44:44.808 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "channel_ident", 2024-04-25T00:44:44.808 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.808 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.808 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.808 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.808 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.808 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.808 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.808 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Share a user-provided description and/or contact email for the cluster", 2024-04-25T00:44:44.809 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.809 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.809 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.809 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.809 INFO:teuthology.orchestra.run.smithi032.stdout: "channel_perf": { 2024-04-25T00:44:44.809 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "channel_perf", 2024-04-25T00:44:44.809 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.809 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.809 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.809 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.809 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.809 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.809 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.809 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Share various performance metrics of a cluster", 2024-04-25T00:44:44.809 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.810 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.810 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.810 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.810 INFO:teuthology.orchestra.run.smithi032.stdout: "contact": { 2024-04-25T00:44:44.810 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "contact", 2024-04-25T00:44:44.810 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.810 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.810 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.810 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.810 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.810 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.810 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.810 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.810 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.811 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.811 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.811 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.811 INFO:teuthology.orchestra.run.smithi032.stdout: "description": { 2024-04-25T00:44:44.811 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "description", 2024-04-25T00:44:44.811 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.811 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.811 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.811 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.811 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.811 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.811 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.811 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.811 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.812 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.812 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.812 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.812 INFO:teuthology.orchestra.run.smithi032.stdout: "device_url": { 2024-04-25T00:44:44.812 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "device_url", 2024-04-25T00:44:44.812 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.812 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.812 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.812 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "https://telemetry.ceph.com/device", 2024-04-25T00:44:44.812 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.812 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.812 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.812 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.812 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.812 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.813 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.813 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.813 INFO:teuthology.orchestra.run.smithi032.stdout: "enabled": { 2024-04-25T00:44:44.813 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "enabled", 2024-04-25T00:44:44.813 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.813 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.813 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.813 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.813 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.813 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.813 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.813 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.813 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.813 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.814 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.814 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.814 INFO:teuthology.orchestra.run.smithi032.stdout: "interval": { 2024-04-25T00:44:44.814 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "interval", 2024-04-25T00:44:44.814 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.814 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.814 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.814 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "24", 2024-04-25T00:44:44.814 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "8", 2024-04-25T00:44:44.814 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.814 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.814 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.814 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.814 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.815 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.815 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.815 INFO:teuthology.orchestra.run.smithi032.stdout: "last_opt_revision": { 2024-04-25T00:44:44.815 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "last_opt_revision", 2024-04-25T00:44:44.815 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.815 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.815 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.815 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "1", 2024-04-25T00:44:44.815 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.815 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.815 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.815 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.815 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.815 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.815 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.816 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.816 INFO:teuthology.orchestra.run.smithi032.stdout: "leaderboard": { 2024-04-25T00:44:44.816 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "leaderboard", 2024-04-25T00:44:44.816 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.816 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.816 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.816 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.816 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.816 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.816 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.816 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.816 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.816 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.816 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.817 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.817 INFO:teuthology.orchestra.run.smithi032.stdout: "leaderboard_description": { 2024-04-25T00:44:44.817 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "leaderboard_description", 2024-04-25T00:44:44.817 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.817 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.817 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.817 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.817 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.817 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.817 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.817 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.817 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.817 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.817 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.817 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.818 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.818 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.818 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.818 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.818 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.818 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.818 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.818 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.818 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.818 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.819 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.819 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.819 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.819 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.819 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.819 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.819 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.819 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.819 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.819 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.819 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.819 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.819 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.819 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.819 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.820 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.820 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.820 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.820 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.820 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.820 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.820 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.820 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.820 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.820 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.820 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.820 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.820 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.820 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.821 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.821 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.821 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.821 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.821 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.821 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.821 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.821 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.821 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.821 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.821 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.821 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.821 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.821 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.821 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.821 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.822 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.822 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.822 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.822 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.822 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.822 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.822 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.822 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.822 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.822 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.822 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.822 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.822 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.822 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.823 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.823 INFO:teuthology.orchestra.run.smithi032.stdout: "organization": { 2024-04-25T00:44:44.823 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "organization", 2024-04-25T00:44:44.823 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.823 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.823 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.823 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.823 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.823 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.823 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.823 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.823 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.823 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.823 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.824 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.824 INFO:teuthology.orchestra.run.smithi032.stdout: "proxy": { 2024-04-25T00:44:44.824 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "proxy", 2024-04-25T00:44:44.824 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.824 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.824 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.824 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.824 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.824 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.824 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.824 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.824 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.824 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.824 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.825 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.825 INFO:teuthology.orchestra.run.smithi032.stdout: "url": { 2024-04-25T00:44:44.825 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "url", 2024-04-25T00:44:44.825 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.825 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.825 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.825 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "https://telemetry.ceph.com/report", 2024-04-25T00:44:44.825 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.825 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.825 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.825 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.825 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.825 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.826 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.826 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.826 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.826 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.826 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.826 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "test_orchestrator", 2024-04-25T00:44:44.826 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.826 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.826 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.826 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.826 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.826 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.826 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.826 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.827 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.827 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.827 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.827 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.827 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.827 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.827 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.827 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.827 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.827 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.827 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.827 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.827 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.827 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.828 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.828 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.828 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.828 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.828 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.828 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.828 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.828 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.828 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.828 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.828 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.828 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.828 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.828 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.829 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.829 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.829 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.829 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.829 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.829 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.829 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.829 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.829 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.829 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.829 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.829 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.829 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.829 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.830 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.830 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.830 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.830 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.830 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.830 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.830 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.830 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.830 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.830 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.830 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.830 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.830 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.831 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.831 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.831 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.831 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.831 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.831 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.831 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.831 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.831 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.831 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.831 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.831 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.831 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.831 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "volumes", 2024-04-25T00:44:44.831 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.831 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.832 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.832 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.832 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.832 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.832 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.832 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.832 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.832 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.832 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.832 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.832 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.832 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.832 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.832 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.832 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.833 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.833 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.833 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.833 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.833 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.833 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.833 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.833 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.833 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.833 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.833 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.833 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.833 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.833 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.833 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.833 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.834 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.834 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.834 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.834 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.834 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.834 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.834 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.834 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.834 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.834 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.834 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.834 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.834 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.834 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.834 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.835 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.835 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.835 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.835 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.835 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.835 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.835 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.835 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.835 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.835 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.835 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.835 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.835 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.835 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.835 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.835 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.835 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.836 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.836 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.836 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.836 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.836 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.836 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.836 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.836 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.836 INFO:teuthology.orchestra.run.smithi032.stdout: "max_concurrent_clones": { 2024-04-25T00:44:44.836 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "max_concurrent_clones", 2024-04-25T00:44:44.836 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.836 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.836 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.836 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "4", 2024-04-25T00:44:44.836 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.837 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.837 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.837 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Number of asynchronous cloner threads", 2024-04-25T00:44:44.837 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.837 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.837 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.837 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.837 INFO:teuthology.orchestra.run.smithi032.stdout: "periodic_async_work": { 2024-04-25T00:44:44.837 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "periodic_async_work", 2024-04-25T00:44:44.837 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.837 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.837 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.837 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.837 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.837 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.837 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.837 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Periodically check for async work", 2024-04-25T00:44:44.838 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.838 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.838 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.838 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.838 INFO:teuthology.orchestra.run.smithi032.stdout: "snapshot_clone_delay": { 2024-04-25T00:44:44.838 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "snapshot_clone_delay", 2024-04-25T00:44:44.838 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.838 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.838 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.838 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "0", 2024-04-25T00:44:44.838 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.838 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.838 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.838 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Delay clone begin operation by snapshot_clone_delay seconds", 2024-04-25T00:44:44.839 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.839 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.839 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.839 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.839 INFO:teuthology.orchestra.run.smithi032.stdout: "snapshot_clone_no_wait": { 2024-04-25T00:44:44.839 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "snapshot_clone_no_wait", 2024-04-25T00:44:44.839 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.839 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.839 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.839 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.839 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.839 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.839 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.839 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Reject subvolume clone request when cloner threads are busy", 2024-04-25T00:44:44.839 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.840 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.840 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.840 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.840 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.840 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.840 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.840 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "zabbix", 2024-04-25T00:44:44.840 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.840 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.840 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.840 INFO:teuthology.orchestra.run.smithi032.stdout: "discovery_interval": { 2024-04-25T00:44:44.840 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "discovery_interval", 2024-04-25T00:44:44.840 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "uint", 2024-04-25T00:44:44.840 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.840 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.841 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "100", 2024-04-25T00:44:44.841 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.841 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.841 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.841 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.841 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.841 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.841 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.841 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.841 INFO:teuthology.orchestra.run.smithi032.stdout: "identifier": { 2024-04-25T00:44:44.841 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "identifier", 2024-04-25T00:44:44.841 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.841 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.841 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.841 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.841 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.842 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.842 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.842 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.842 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.842 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.842 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.842 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.842 INFO:teuthology.orchestra.run.smithi032.stdout: "interval": { 2024-04-25T00:44:44.842 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "interval", 2024-04-25T00:44:44.842 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.842 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.842 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.842 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "60", 2024-04-25T00:44:44.842 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.843 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.843 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.843 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.843 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.843 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.843 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.843 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.843 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.843 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.843 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.843 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.843 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.843 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.843 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.844 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.844 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.844 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.844 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.844 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.844 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.844 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.844 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.844 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.844 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.844 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.844 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.844 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.845 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.845 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.845 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.845 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.845 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.845 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.845 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.845 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.845 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.845 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.845 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.845 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.845 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.845 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.845 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.846 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.846 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.846 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.846 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.846 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.846 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.846 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.846 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.846 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.846 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.846 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.846 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.847 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.847 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.847 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.847 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.847 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.847 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.847 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.847 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.847 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.847 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.847 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.847 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.847 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.847 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.847 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.847 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.848 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.848 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.848 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.848 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.848 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.848 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.848 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.848 INFO:teuthology.orchestra.run.smithi032.stdout: "zabbix_host": { 2024-04-25T00:44:44.848 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "zabbix_host", 2024-04-25T00:44:44.848 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.848 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.848 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.848 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.848 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.848 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.849 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.849 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.849 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.849 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.849 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.849 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.849 INFO:teuthology.orchestra.run.smithi032.stdout: "zabbix_port": { 2024-04-25T00:44:44.849 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "zabbix_port", 2024-04-25T00:44:44.849 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.849 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.849 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.849 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10051", 2024-04-25T00:44:44.849 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.849 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.849 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.850 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.850 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.850 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.850 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.850 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.850 INFO:teuthology.orchestra.run.smithi032.stdout: "zabbix_sender": { 2024-04-25T00:44:44.850 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "zabbix_sender", 2024-04-25T00:44:44.850 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.850 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.850 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.850 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "/usr/bin/zabbix_sender", 2024-04-25T00:44:44.850 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.850 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.850 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.851 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.851 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.851 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.851 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.851 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.851 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.851 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.851 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-04-25T00:44:44.851 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.851 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.851 INFO:teuthology.orchestra.run.smithi032.stdout: "gid": 25106, 2024-04-25T00:44:44.851 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "y", 2024-04-25T00:44:44.851 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_features": 4540701547738038271, 2024-04-25T00:44:44.852 INFO:teuthology.orchestra.run.smithi032.stdout: "available_modules": [ 2024-04-25T00:44:44.852 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.852 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "alerts", 2024-04-25T00:44:44.852 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.852 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.852 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.852 INFO:teuthology.orchestra.run.smithi032.stdout: "interval": { 2024-04-25T00:44:44.852 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "interval", 2024-04-25T00:44:44.852 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.852 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.852 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.852 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "60", 2024-04-25T00:44:44.852 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.852 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.852 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.852 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "How frequently to reexamine health status", 2024-04-25T00:44:44.853 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.853 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.853 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.853 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.853 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.853 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.853 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.853 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.853 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.853 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.853 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.853 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.853 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.853 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.854 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.854 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.854 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.854 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.854 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.854 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.854 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.854 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.854 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.854 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.854 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.854 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.854 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.854 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.855 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.855 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.855 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.855 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.855 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.855 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.855 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.855 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.855 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.855 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.855 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.855 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.855 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.855 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.855 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.856 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.856 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.856 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.856 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.856 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.856 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.856 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.856 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.856 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.856 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.856 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.856 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.856 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.856 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.856 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.857 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.857 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.857 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.857 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.857 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.857 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.857 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.857 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.857 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.857 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.857 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.857 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.857 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.857 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.858 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.858 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.858 INFO:teuthology.orchestra.run.smithi032.stdout: "smtp_destination": { 2024-04-25T00:44:44.858 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "smtp_destination", 2024-04-25T00:44:44.858 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.858 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.858 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.858 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.858 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.858 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.858 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.858 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Email address to send alerts to", 2024-04-25T00:44:44.858 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.858 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.859 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.859 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.859 INFO:teuthology.orchestra.run.smithi032.stdout: "smtp_from_name": { 2024-04-25T00:44:44.859 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "smtp_from_name", 2024-04-25T00:44:44.859 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.859 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.859 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.859 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "Ceph", 2024-04-25T00:44:44.859 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.859 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.859 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.859 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Email From: name", 2024-04-25T00:44:44.859 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.859 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.859 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.859 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.860 INFO:teuthology.orchestra.run.smithi032.stdout: "smtp_host": { 2024-04-25T00:44:44.860 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "smtp_host", 2024-04-25T00:44:44.860 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.860 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.860 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.860 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.860 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.860 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.860 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.860 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "SMTP server", 2024-04-25T00:44:44.860 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.860 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.860 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.861 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.861 INFO:teuthology.orchestra.run.smithi032.stdout: "smtp_password": { 2024-04-25T00:44:44.861 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "smtp_password", 2024-04-25T00:44:44.861 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.861 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.861 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.861 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.861 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.861 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.861 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.861 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Password to authenticate with", 2024-04-25T00:44:44.861 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.861 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.862 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.862 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.862 INFO:teuthology.orchestra.run.smithi032.stdout: "smtp_port": { 2024-04-25T00:44:44.862 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "smtp_port", 2024-04-25T00:44:44.862 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.862 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.862 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.862 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "465", 2024-04-25T00:44:44.862 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.862 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.862 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.862 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "SMTP port", 2024-04-25T00:44:44.862 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.862 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.863 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.863 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.863 INFO:teuthology.orchestra.run.smithi032.stdout: "smtp_sender": { 2024-04-25T00:44:44.863 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "smtp_sender", 2024-04-25T00:44:44.863 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.863 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.863 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.863 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.863 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.863 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.863 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.863 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "SMTP envelope sender", 2024-04-25T00:44:44.863 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.863 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.863 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.864 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.864 INFO:teuthology.orchestra.run.smithi032.stdout: "smtp_ssl": { 2024-04-25T00:44:44.864 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "smtp_ssl", 2024-04-25T00:44:44.864 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.864 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.864 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.864 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.864 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.864 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.864 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.864 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Use SSL to connect to SMTP server", 2024-04-25T00:44:44.864 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.864 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.865 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.865 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.865 INFO:teuthology.orchestra.run.smithi032.stdout: "smtp_user": { 2024-04-25T00:44:44.865 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "smtp_user", 2024-04-25T00:44:44.865 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.865 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.865 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.865 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.865 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.865 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.865 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.865 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "User to authenticate as", 2024-04-25T00:44:44.865 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.865 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.865 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.866 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.866 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.866 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.866 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.866 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "balancer", 2024-04-25T00:44:44.866 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.866 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.866 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.866 INFO:teuthology.orchestra.run.smithi032.stdout: "active": { 2024-04-25T00:44:44.866 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "active", 2024-04-25T00:44:44.866 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.866 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.866 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.867 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.867 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.867 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.867 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.867 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "automatically balance PGs across cluster", 2024-04-25T00:44:44.867 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.867 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.867 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.867 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.867 INFO:teuthology.orchestra.run.smithi032.stdout: "begin_time": { 2024-04-25T00:44:44.867 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "begin_time", 2024-04-25T00:44:44.867 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.867 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.867 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.867 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "0000", 2024-04-25T00:44:44.868 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.868 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.868 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.868 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "beginning time of day to automatically balance", 2024-04-25T00:44:44.868 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "This is a time of day in the format HHMM.", 2024-04-25T00:44:44.868 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.868 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.868 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.868 INFO:teuthology.orchestra.run.smithi032.stdout: "begin_weekday": { 2024-04-25T00:44:44.868 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "begin_weekday", 2024-04-25T00:44:44.868 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "uint", 2024-04-25T00:44:44.868 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.868 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.868 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "0", 2024-04-25T00:44:44.869 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "0", 2024-04-25T00:44:44.869 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "6", 2024-04-25T00:44:44.869 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.869 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Restrict automatic balancing to this day of the week or later", 2024-04-25T00:44:44.869 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2024-04-25T00:44:44.869 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.869 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.869 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.869 INFO:teuthology.orchestra.run.smithi032.stdout: "crush_compat_max_iterations": { 2024-04-25T00:44:44.869 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "crush_compat_max_iterations", 2024-04-25T00:44:44.869 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "uint", 2024-04-25T00:44:44.869 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.869 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.869 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "25", 2024-04-25T00:44:44.870 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "1", 2024-04-25T00:44:44.870 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "250", 2024-04-25T00:44:44.870 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.870 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "maximum number of iterations to attempt optimization", 2024-04-25T00:44:44.870 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.870 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.870 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.870 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.870 INFO:teuthology.orchestra.run.smithi032.stdout: "crush_compat_metrics": { 2024-04-25T00:44:44.870 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "crush_compat_metrics", 2024-04-25T00:44:44.870 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.870 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.870 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.870 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "pgs,objects,bytes", 2024-04-25T00:44:44.870 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.871 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.871 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.871 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "metrics with which to calculate OSD utilization", 2024-04-25T00:44:44.871 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.", 2024-04-25T00:44:44.871 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.871 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.871 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.871 INFO:teuthology.orchestra.run.smithi032.stdout: "crush_compat_step": { 2024-04-25T00:44:44.871 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "crush_compat_step", 2024-04-25T00:44:44.871 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "float", 2024-04-25T00:44:44.871 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.871 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.871 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "0.5", 2024-04-25T00:44:44.871 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "0.001", 2024-04-25T00:44:44.872 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "0.999", 2024-04-25T00:44:44.872 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.872 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "aggressiveness of optimization", 2024-04-25T00:44:44.872 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": ".99 is very aggressive, .01 is less aggressive", 2024-04-25T00:44:44.872 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.872 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.872 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.872 INFO:teuthology.orchestra.run.smithi032.stdout: "end_time": { 2024-04-25T00:44:44.872 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "end_time", 2024-04-25T00:44:44.872 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.872 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.872 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.872 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "2359", 2024-04-25T00:44:44.872 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.872 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.873 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.873 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "ending time of day to automatically balance", 2024-04-25T00:44:44.873 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "This is a time of day in the format HHMM.", 2024-04-25T00:44:44.873 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.873 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.873 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.873 INFO:teuthology.orchestra.run.smithi032.stdout: "end_weekday": { 2024-04-25T00:44:44.873 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "end_weekday", 2024-04-25T00:44:44.873 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "uint", 2024-04-25T00:44:44.873 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.873 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.873 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "0", 2024-04-25T00:44:44.873 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "0", 2024-04-25T00:44:44.873 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "6", 2024-04-25T00:44:44.873 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.874 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Restrict automatic balancing to days of the week earlier than this", 2024-04-25T00:44:44.874 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2024-04-25T00:44:44.874 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.874 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.874 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.874 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.874 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.874 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.874 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.874 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.874 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.874 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.874 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.874 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.875 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.875 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.875 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.875 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.875 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.875 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.875 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.875 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.875 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.875 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.875 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.875 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.875 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.875 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.875 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.876 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.876 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.876 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.876 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.876 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.876 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.876 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.876 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.876 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.876 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.876 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.876 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.876 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.876 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.877 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.877 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.877 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.877 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.877 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.877 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.877 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.877 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.877 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.877 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.877 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.877 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.877 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.878 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.878 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.878 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.878 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.878 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.878 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.878 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.878 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.878 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.878 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.878 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.878 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.878 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.878 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.879 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.879 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.879 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.879 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.879 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.879 INFO:teuthology.orchestra.run.smithi032.stdout: "min_score": { 2024-04-25T00:44:44.879 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "min_score", 2024-04-25T00:44:44.879 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "float", 2024-04-25T00:44:44.879 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.879 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.879 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "0", 2024-04-25T00:44:44.879 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.879 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.879 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.879 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "minimum score, below which no optimization is attempted", 2024-04-25T00:44:44.879 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.880 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.880 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.880 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.880 INFO:teuthology.orchestra.run.smithi032.stdout: "mode": { 2024-04-25T00:44:44.880 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "mode", 2024-04-25T00:44:44.880 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.880 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.880 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.880 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "upmap", 2024-04-25T00:44:44.880 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.880 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.880 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.880 INFO:teuthology.orchestra.run.smithi032.stdout: "crush-compat", 2024-04-25T00:44:44.880 INFO:teuthology.orchestra.run.smithi032.stdout: "none", 2024-04-25T00:44:44.880 INFO:teuthology.orchestra.run.smithi032.stdout: "read", 2024-04-25T00:44:44.881 INFO:teuthology.orchestra.run.smithi032.stdout: "upmap", 2024-04-25T00:44:44.881 INFO:teuthology.orchestra.run.smithi032.stdout: "upmap-read" 2024-04-25T00:44:44.881 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.881 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Balancer mode", 2024-04-25T00:44:44.881 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.881 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.881 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.881 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.881 INFO:teuthology.orchestra.run.smithi032.stdout: "pool_ids": { 2024-04-25T00:44:44.881 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "pool_ids", 2024-04-25T00:44:44.881 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.881 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.881 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.881 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.882 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.882 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.882 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.882 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "pools which the automatic balancing will be limited to", 2024-04-25T00:44:44.882 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.882 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.882 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.882 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.882 INFO:teuthology.orchestra.run.smithi032.stdout: "sleep_interval": { 2024-04-25T00:44:44.882 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "sleep_interval", 2024-04-25T00:44:44.882 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.882 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.882 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.882 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "60", 2024-04-25T00:44:44.883 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.883 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.883 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.883 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "how frequently to wake up and attempt optimization", 2024-04-25T00:44:44.883 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.883 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.883 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.883 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.883 INFO:teuthology.orchestra.run.smithi032.stdout: "upmap_max_deviation": { 2024-04-25T00:44:44.883 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "upmap_max_deviation", 2024-04-25T00:44:44.883 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.883 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.883 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.883 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "5", 2024-04-25T00:44:44.883 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "1", 2024-04-25T00:44:44.884 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.884 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.884 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "deviation below which no optimization is attempted", 2024-04-25T00:44:44.884 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "If the number of PGs are within this count then no optimization is attempted", 2024-04-25T00:44:44.884 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.884 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.884 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.884 INFO:teuthology.orchestra.run.smithi032.stdout: "upmap_max_optimizations": { 2024-04-25T00:44:44.884 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "upmap_max_optimizations", 2024-04-25T00:44:44.884 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "uint", 2024-04-25T00:44:44.884 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.884 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.884 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10", 2024-04-25T00:44:44.884 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.884 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.885 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.885 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "maximum upmap optimizations to make per attempt", 2024-04-25T00:44:44.885 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.885 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.885 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.885 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.885 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.885 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.885 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.885 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "cephadm", 2024-04-25T00:44:44.885 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.885 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.885 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.885 INFO:teuthology.orchestra.run.smithi032.stdout: "agent_down_multiplier": { 2024-04-25T00:44:44.885 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "agent_down_multiplier", 2024-04-25T00:44:44.885 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "float", 2024-04-25T00:44:44.886 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.886 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.886 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "3.0", 2024-04-25T00:44:44.886 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.886 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.886 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.886 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Multiplied by agent refresh rate to calculate how long agent must not report before being marked down", 2024-04-25T00:44:44.886 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.886 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.886 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.886 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.886 INFO:teuthology.orchestra.run.smithi032.stdout: "agent_refresh_rate": { 2024-04-25T00:44:44.886 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "agent_refresh_rate", 2024-04-25T00:44:44.886 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.887 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.887 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.887 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "20", 2024-04-25T00:44:44.887 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.887 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.887 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.887 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "How often agent on each host will try to gather and send metadata", 2024-04-25T00:44:44.887 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.887 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.887 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.887 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.887 INFO:teuthology.orchestra.run.smithi032.stdout: "agent_starting_port": { 2024-04-25T00:44:44.887 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "agent_starting_port", 2024-04-25T00:44:44.887 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.887 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.887 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.888 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "4721", 2024-04-25T00:44:44.888 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.888 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.888 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.888 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)", 2024-04-25T00:44:44.888 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.888 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.888 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.888 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.888 INFO:teuthology.orchestra.run.smithi032.stdout: "allow_ptrace": { 2024-04-25T00:44:44.888 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "allow_ptrace", 2024-04-25T00:44:44.888 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.888 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.888 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.888 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.888 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.889 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.889 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.889 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "allow SYS_PTRACE capability on ceph containers", 2024-04-25T00:44:44.889 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.", 2024-04-25T00:44:44.889 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.889 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.889 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.889 INFO:teuthology.orchestra.run.smithi032.stdout: "autotune_interval": { 2024-04-25T00:44:44.889 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "autotune_interval", 2024-04-25T00:44:44.889 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.889 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.889 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.889 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "600", 2024-04-25T00:44:44.889 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.890 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.890 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.890 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "how frequently to autotune daemon memory", 2024-04-25T00:44:44.890 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.890 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.890 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.890 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.890 INFO:teuthology.orchestra.run.smithi032.stdout: "autotune_memory_target_ratio": { 2024-04-25T00:44:44.890 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "autotune_memory_target_ratio", 2024-04-25T00:44:44.890 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "float", 2024-04-25T00:44:44.890 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.890 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.890 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "0.7", 2024-04-25T00:44:44.890 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.890 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.891 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.891 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "ratio of total system memory to divide amongst autotuned daemons", 2024-04-25T00:44:44.891 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.891 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.891 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.891 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.891 INFO:teuthology.orchestra.run.smithi032.stdout: "cephadm_log_destination": { 2024-04-25T00:44:44.891 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "cephadm_log_destination", 2024-04-25T00:44:44.891 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.891 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.891 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.891 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.891 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.891 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.892 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.892 INFO:teuthology.orchestra.run.smithi032.stdout: "file", 2024-04-25T00:44:44.892 INFO:teuthology.orchestra.run.smithi032.stdout: "file,syslog", 2024-04-25T00:44:44.892 INFO:teuthology.orchestra.run.smithi032.stdout: "syslog" 2024-04-25T00:44:44.892 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.892 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Destination for cephadm command's persistent logging", 2024-04-25T00:44:44.892 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.892 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.892 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.892 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.892 INFO:teuthology.orchestra.run.smithi032.stdout: "cgroups_split": { 2024-04-25T00:44:44.892 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "cgroups_split", 2024-04-25T00:44:44.892 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.892 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.892 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.893 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.893 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.893 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.893 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.893 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Pass --cgroups=split when cephadm creates containers (currently podman only)", 2024-04-25T00:44:44.893 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.893 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.893 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.893 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.893 INFO:teuthology.orchestra.run.smithi032.stdout: "config_checks_enabled": { 2024-04-25T00:44:44.893 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "config_checks_enabled", 2024-04-25T00:44:44.893 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.893 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.893 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.894 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.894 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.894 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.894 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.894 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Enable or disable the cephadm configuration analysis", 2024-04-25T00:44:44.894 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.894 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.894 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.894 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.894 INFO:teuthology.orchestra.run.smithi032.stdout: "config_dashboard": { 2024-04-25T00:44:44.894 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "config_dashboard", 2024-04-25T00:44:44.894 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.894 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.894 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.895 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.895 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.895 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.895 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.895 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "manage configs like API endpoints in Dashboard.", 2024-04-25T00:44:44.895 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.895 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.895 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.895 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.895 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_alertmanager": { 2024-04-25T00:44:44.895 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_alertmanager", 2024-04-25T00:44:44.895 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.895 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.895 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.896 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/prometheus/alertmanager:v0.25.0", 2024-04-25T00:44:44.896 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.896 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.896 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.896 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Prometheus container image", 2024-04-25T00:44:44.896 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.896 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.896 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.896 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.896 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_base": { 2024-04-25T00:44:44.896 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_base", 2024-04-25T00:44:44.896 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.896 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.896 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.896 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/ceph/ceph", 2024-04-25T00:44:44.897 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.897 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.897 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.897 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Container image name, without the tag", 2024-04-25T00:44:44.897 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.897 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.897 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.897 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.897 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_elasticsearch": { 2024-04-25T00:44:44.897 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_elasticsearch", 2024-04-25T00:44:44.897 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.897 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.897 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.897 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/omrizeneva/elasticsearch:6.8.23", 2024-04-25T00:44:44.897 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.898 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.898 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.898 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "elasticsearch container image", 2024-04-25T00:44:44.898 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.898 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.898 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.898 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.898 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_grafana": { 2024-04-25T00:44:44.898 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_grafana", 2024-04-25T00:44:44.898 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.898 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.898 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.898 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/ceph/grafana:9.4.12", 2024-04-25T00:44:44.898 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.898 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.899 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.899 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Prometheus container image", 2024-04-25T00:44:44.899 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.899 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.899 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.899 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.899 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_haproxy": { 2024-04-25T00:44:44.899 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_haproxy", 2024-04-25T00:44:44.899 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.899 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.899 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.899 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/ceph/haproxy:2.3", 2024-04-25T00:44:44.899 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.899 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.899 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.900 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "HAproxy container image", 2024-04-25T00:44:44.900 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.900 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.900 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.900 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.900 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_jaeger_agent": { 2024-04-25T00:44:44.900 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_jaeger_agent", 2024-04-25T00:44:44.900 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.900 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.900 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.900 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/jaegertracing/jaeger-agent:1.29", 2024-04-25T00:44:44.900 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.900 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.900 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.901 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Jaeger agent container image", 2024-04-25T00:44:44.901 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.901 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.901 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.901 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.901 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_jaeger_collector": { 2024-04-25T00:44:44.901 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_jaeger_collector", 2024-04-25T00:44:44.901 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.901 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.901 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.901 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/jaegertracing/jaeger-collector:1.29", 2024-04-25T00:44:44.901 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.901 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.901 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.901 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Jaeger collector container image", 2024-04-25T00:44:44.901 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.902 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.902 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.902 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.902 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_jaeger_query": { 2024-04-25T00:44:44.902 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_jaeger_query", 2024-04-25T00:44:44.902 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.902 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.902 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.902 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/jaegertracing/jaeger-query:1.29", 2024-04-25T00:44:44.902 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.902 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.902 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.902 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Jaeger query container image", 2024-04-25T00:44:44.902 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.903 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.903 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.903 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.903 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_keepalived": { 2024-04-25T00:44:44.903 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_keepalived", 2024-04-25T00:44:44.903 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.903 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.903 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.903 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/ceph/keepalived:2.2.4", 2024-04-25T00:44:44.903 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.903 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.903 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.903 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Keepalived container image", 2024-04-25T00:44:44.903 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.903 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.904 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.904 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.904 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_loki": { 2024-04-25T00:44:44.904 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_loki", 2024-04-25T00:44:44.904 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.904 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.904 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.904 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "docker.io/grafana/loki:2.4.0", 2024-04-25T00:44:44.904 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.904 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.904 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.904 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Loki container image", 2024-04-25T00:44:44.904 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.904 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.904 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.905 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.905 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_node_exporter": { 2024-04-25T00:44:44.905 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_node_exporter", 2024-04-25T00:44:44.905 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.905 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.905 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.905 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/prometheus/node-exporter:v1.5.0", 2024-04-25T00:44:44.905 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.905 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.905 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.905 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Prometheus container image", 2024-04-25T00:44:44.905 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.905 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.905 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.905 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.906 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_nvmeof": { 2024-04-25T00:44:44.906 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_nvmeof", 2024-04-25T00:44:44.906 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.906 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.906 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.906 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/ceph/nvmeof:1.2.1", 2024-04-25T00:44:44.906 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.906 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.906 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.906 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Nvme-of container image", 2024-04-25T00:44:44.906 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.906 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.906 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.906 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.907 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_prometheus": { 2024-04-25T00:44:44.907 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_prometheus", 2024-04-25T00:44:44.907 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.907 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.907 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.907 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/prometheus/prometheus:v2.43.0", 2024-04-25T00:44:44.907 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.907 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.907 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.907 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Prometheus container image", 2024-04-25T00:44:44.907 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.907 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.907 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.907 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.907 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_promtail": { 2024-04-25T00:44:44.907 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_promtail", 2024-04-25T00:44:44.908 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.908 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.908 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.908 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "docker.io/grafana/promtail:2.4.0", 2024-04-25T00:44:44.908 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.908 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.908 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.908 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Promtail container image", 2024-04-25T00:44:44.908 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.908 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.908 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.908 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.908 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_samba": { 2024-04-25T00:44:44.908 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_samba", 2024-04-25T00:44:44.909 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.909 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.909 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.909 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/samba.org/samba-server:devbuilds-centos-amd64", 2024-04-25T00:44:44.909 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.909 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.909 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.909 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Samba/SMB container image", 2024-04-25T00:44:44.909 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.909 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.909 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.909 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.909 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_snmp_gateway": { 2024-04-25T00:44:44.909 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_snmp_gateway", 2024-04-25T00:44:44.909 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.910 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.910 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.910 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "docker.io/maxwo/snmp-notifier:v1.2.1", 2024-04-25T00:44:44.910 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.910 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.910 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.910 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "SNMP Gateway container image", 2024-04-25T00:44:44.910 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.910 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.910 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.910 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.910 INFO:teuthology.orchestra.run.smithi032.stdout: "container_init": { 2024-04-25T00:44:44.910 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_init", 2024-04-25T00:44:44.910 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.910 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.911 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.911 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.911 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.911 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.911 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.911 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Run podman/docker with `--init`", 2024-04-25T00:44:44.911 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.911 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.911 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.911 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.911 INFO:teuthology.orchestra.run.smithi032.stdout: "daemon_cache_timeout": { 2024-04-25T00:44:44.911 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "daemon_cache_timeout", 2024-04-25T00:44:44.911 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.911 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.912 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.912 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "600", 2024-04-25T00:44:44.912 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.912 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.912 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.912 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "seconds to cache service (daemon) inventory", 2024-04-25T00:44:44.912 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.912 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.912 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.912 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.912 INFO:teuthology.orchestra.run.smithi032.stdout: "default_cephadm_command_timeout": { 2024-04-25T00:44:44.912 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "default_cephadm_command_timeout", 2024-04-25T00:44:44.912 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.912 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.912 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.913 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "900", 2024-04-25T00:44:44.913 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.913 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.913 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.913 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Default timeout applied to cephadm commands run directly on the host (in seconds)", 2024-04-25T00:44:44.913 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.913 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.913 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.913 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.913 INFO:teuthology.orchestra.run.smithi032.stdout: "default_registry": { 2024-04-25T00:44:44.913 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "default_registry", 2024-04-25T00:44:44.913 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.913 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.913 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.914 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "docker.io", 2024-04-25T00:44:44.914 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.914 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.914 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.914 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Search-registry to which we should normalize unqualified image names. This is not the default registry", 2024-04-25T00:44:44.914 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.914 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.914 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.914 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.914 INFO:teuthology.orchestra.run.smithi032.stdout: "device_cache_timeout": { 2024-04-25T00:44:44.914 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "device_cache_timeout", 2024-04-25T00:44:44.914 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.914 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.914 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.915 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "1800", 2024-04-25T00:44:44.915 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.915 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.915 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.915 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "seconds to cache device inventory", 2024-04-25T00:44:44.915 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.915 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.915 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.915 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.915 INFO:teuthology.orchestra.run.smithi032.stdout: "device_enhanced_scan": { 2024-04-25T00:44:44.915 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "device_enhanced_scan", 2024-04-25T00:44:44.915 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.915 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.915 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.915 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.916 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.916 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.916 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.916 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Use libstoragemgmt during device scans", 2024-04-25T00:44:44.916 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.916 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.916 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.916 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.916 INFO:teuthology.orchestra.run.smithi032.stdout: "facts_cache_timeout": { 2024-04-25T00:44:44.916 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "facts_cache_timeout", 2024-04-25T00:44:44.916 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.916 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.916 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.916 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "60", 2024-04-25T00:44:44.917 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.917 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.917 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.917 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "seconds to cache host facts data", 2024-04-25T00:44:44.917 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.917 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.917 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.917 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.917 INFO:teuthology.orchestra.run.smithi032.stdout: "grafana_dashboards_path": { 2024-04-25T00:44:44.917 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "grafana_dashboards_path", 2024-04-25T00:44:44.917 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.917 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.917 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.917 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "/etc/grafana/dashboards/ceph-dashboard/", 2024-04-25T00:44:44.917 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.918 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.918 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.918 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "location of dashboards to include in grafana deployments", 2024-04-25T00:44:44.918 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.918 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.918 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.918 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.918 INFO:teuthology.orchestra.run.smithi032.stdout: "host_check_interval": { 2024-04-25T00:44:44.918 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "host_check_interval", 2024-04-25T00:44:44.918 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.918 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.918 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.918 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "600", 2024-04-25T00:44:44.918 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.919 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.919 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.919 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "how frequently to perform a host check", 2024-04-25T00:44:44.919 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.919 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.919 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.919 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.919 INFO:teuthology.orchestra.run.smithi032.stdout: "hw_monitoring": { 2024-04-25T00:44:44.919 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "hw_monitoring", 2024-04-25T00:44:44.919 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.919 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.919 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.919 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.919 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.919 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.920 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.920 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Deploy hw monitoring daemon on every host.", 2024-04-25T00:44:44.920 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.920 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.920 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.920 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.920 INFO:teuthology.orchestra.run.smithi032.stdout: "inventory_list_all": { 2024-04-25T00:44:44.920 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "inventory_list_all", 2024-04-25T00:44:44.920 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.920 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.920 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.920 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.920 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.920 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.921 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.921 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)", 2024-04-25T00:44:44.921 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.921 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.921 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.921 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.921 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.921 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.921 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.921 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.921 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.921 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.921 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.921 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.921 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.922 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.922 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.922 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.922 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.922 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.922 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.922 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.922 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.922 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.922 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.922 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.922 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.922 INFO:teuthology.orchestra.run.smithi032.stdout: "log_refresh_metadata": { 2024-04-25T00:44:44.923 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_refresh_metadata", 2024-04-25T00:44:44.923 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.923 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.923 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.923 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.923 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.923 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.923 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.923 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level", 2024-04-25T00:44:44.923 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.923 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.923 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.923 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.923 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.924 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.924 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.924 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.924 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.924 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.924 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.924 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.924 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.924 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "log to the \"cephadm\" cluster log channel\"", 2024-04-25T00:44:44.924 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.924 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.924 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.924 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.925 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.925 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.925 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.925 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.925 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.925 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.925 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.925 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.925 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.925 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.925 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.925 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.925 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.925 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.925 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.926 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.926 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.926 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.926 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.926 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.926 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.926 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.926 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.926 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.926 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.926 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.926 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.926 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.926 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.927 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.927 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.927 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.927 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.927 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.927 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.927 INFO:teuthology.orchestra.run.smithi032.stdout: "manage_etc_ceph_ceph_conf": { 2024-04-25T00:44:44.927 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "manage_etc_ceph_ceph_conf", 2024-04-25T00:44:44.927 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.927 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.927 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.927 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.927 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.927 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.928 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.928 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Manage and own /etc/ceph/ceph.conf on the hosts.", 2024-04-25T00:44:44.928 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.928 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.928 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.928 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.928 INFO:teuthology.orchestra.run.smithi032.stdout: "manage_etc_ceph_ceph_conf_hosts": { 2024-04-25T00:44:44.928 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "manage_etc_ceph_ceph_conf_hosts", 2024-04-25T00:44:44.928 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.928 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.928 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.928 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "*", 2024-04-25T00:44:44.928 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.928 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.929 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.929 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf", 2024-04-25T00:44:44.929 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.929 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.929 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.929 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.929 INFO:teuthology.orchestra.run.smithi032.stdout: "max_count_per_host": { 2024-04-25T00:44:44.929 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "max_count_per_host", 2024-04-25T00:44:44.929 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.929 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.929 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.929 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10", 2024-04-25T00:44:44.929 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.929 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.930 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.930 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "max number of daemons per service per host", 2024-04-25T00:44:44.930 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.930 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.930 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.930 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.930 INFO:teuthology.orchestra.run.smithi032.stdout: "max_osd_draining_count": { 2024-04-25T00:44:44.930 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "max_osd_draining_count", 2024-04-25T00:44:44.930 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.930 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.930 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.930 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10", 2024-04-25T00:44:44.930 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.930 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.931 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.931 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "max number of osds that will be drained simultaneously when osds are removed", 2024-04-25T00:44:44.931 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.931 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.931 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.931 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.931 INFO:teuthology.orchestra.run.smithi032.stdout: "migration_current": { 2024-04-25T00:44:44.931 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "migration_current", 2024-04-25T00:44:44.931 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.931 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.931 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.931 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.931 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.931 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.932 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.932 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "internal - do not modify", 2024-04-25T00:44:44.932 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.932 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.932 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.932 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.932 INFO:teuthology.orchestra.run.smithi032.stdout: "mode": { 2024-04-25T00:44:44.932 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "mode", 2024-04-25T00:44:44.932 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.932 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.932 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.932 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "root", 2024-04-25T00:44:44.932 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.932 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.932 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.933 INFO:teuthology.orchestra.run.smithi032.stdout: "cephadm-package", 2024-04-25T00:44:44.933 INFO:teuthology.orchestra.run.smithi032.stdout: "root" 2024-04-25T00:44:44.933 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.933 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "mode for remote execution of cephadm", 2024-04-25T00:44:44.933 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.933 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.933 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.933 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.933 INFO:teuthology.orchestra.run.smithi032.stdout: "oob_default_addr": { 2024-04-25T00:44:44.933 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "oob_default_addr", 2024-04-25T00:44:44.933 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.933 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.933 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.933 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "169.254.1.1", 2024-04-25T00:44:44.933 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.934 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.934 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.934 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Default address for RedFish API (oob management).", 2024-04-25T00:44:44.934 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.934 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.934 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.934 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.934 INFO:teuthology.orchestra.run.smithi032.stdout: "prometheus_alerts_path": { 2024-04-25T00:44:44.934 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "prometheus_alerts_path", 2024-04-25T00:44:44.934 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.934 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.934 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.934 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "/etc/prometheus/ceph/ceph_default_alerts.yml", 2024-04-25T00:44:44.934 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.935 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.935 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.935 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "location of alerts to include in prometheus deployments", 2024-04-25T00:44:44.935 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.935 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.935 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.935 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.935 INFO:teuthology.orchestra.run.smithi032.stdout: "registry_insecure": { 2024-04-25T00:44:44.935 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "registry_insecure", 2024-04-25T00:44:44.935 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.935 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.935 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.935 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.935 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.936 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.936 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.936 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Registry is to be considered insecure (no TLS available). Only for development purposes.", 2024-04-25T00:44:44.936 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.936 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.936 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.936 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.936 INFO:teuthology.orchestra.run.smithi032.stdout: "registry_password": { 2024-04-25T00:44:44.936 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "registry_password", 2024-04-25T00:44:44.936 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.936 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.936 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.936 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.936 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.936 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.937 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.937 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Custom repository password. Only used for logging into a registry.", 2024-04-25T00:44:44.937 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.937 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.937 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.937 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.937 INFO:teuthology.orchestra.run.smithi032.stdout: "registry_url": { 2024-04-25T00:44:44.937 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "registry_url", 2024-04-25T00:44:44.937 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.937 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.937 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.937 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.937 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.937 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.938 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.938 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Registry url for login purposes. This is not the default registry", 2024-04-25T00:44:44.938 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.938 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.938 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.938 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.938 INFO:teuthology.orchestra.run.smithi032.stdout: "registry_username": { 2024-04-25T00:44:44.938 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "registry_username", 2024-04-25T00:44:44.938 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.938 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.938 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.938 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.938 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.938 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.938 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.939 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Custom repository username. Only used for logging into a registry.", 2024-04-25T00:44:44.939 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.939 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.939 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.939 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.939 INFO:teuthology.orchestra.run.smithi032.stdout: "secure_monitoring_stack": { 2024-04-25T00:44:44.939 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "secure_monitoring_stack", 2024-04-25T00:44:44.939 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.939 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.939 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.939 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.939 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.939 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.939 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.940 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Enable TLS security for all the monitoring stack daemons", 2024-04-25T00:44:44.940 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.940 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.940 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.940 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.940 INFO:teuthology.orchestra.run.smithi032.stdout: "service_discovery_port": { 2024-04-25T00:44:44.940 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "service_discovery_port", 2024-04-25T00:44:44.940 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.940 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.940 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.940 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "8765", 2024-04-25T00:44:44.940 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.940 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.940 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.940 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "cephadm service discovery port", 2024-04-25T00:44:44.941 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.941 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.941 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.941 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.941 INFO:teuthology.orchestra.run.smithi032.stdout: "ssh_config_file": { 2024-04-25T00:44:44.941 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ssh_config_file", 2024-04-25T00:44:44.941 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.941 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.941 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.941 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.941 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.941 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.941 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.941 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "customized SSH config file to connect to managed hosts", 2024-04-25T00:44:44.942 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.942 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.942 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.942 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.942 INFO:teuthology.orchestra.run.smithi032.stdout: "use_agent": { 2024-04-25T00:44:44.942 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "use_agent", 2024-04-25T00:44:44.942 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.942 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.942 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.942 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.942 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.942 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.942 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.942 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Use cephadm agent on each host to gather and send metadata", 2024-04-25T00:44:44.942 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.943 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.943 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.943 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.943 INFO:teuthology.orchestra.run.smithi032.stdout: "use_repo_digest": { 2024-04-25T00:44:44.943 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "use_repo_digest", 2024-04-25T00:44:44.943 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.943 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.943 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.943 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.943 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.943 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.943 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.943 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Automatically convert image tags to image digest. Make sure all daemons use the same image", 2024-04-25T00:44:44.943 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.943 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.944 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.944 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.944 INFO:teuthology.orchestra.run.smithi032.stdout: "warn_on_failed_host_check": { 2024-04-25T00:44:44.944 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "warn_on_failed_host_check", 2024-04-25T00:44:44.944 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.944 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.944 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.944 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.944 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.944 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.944 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.944 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "raise a health warning if the host check fails", 2024-04-25T00:44:44.944 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.944 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.944 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.945 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.945 INFO:teuthology.orchestra.run.smithi032.stdout: "warn_on_stray_daemons": { 2024-04-25T00:44:44.945 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "warn_on_stray_daemons", 2024-04-25T00:44:44.945 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.945 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.945 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.945 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.945 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.945 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.945 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.945 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "raise a health warning if daemons are detected that are not managed by cephadm", 2024-04-25T00:44:44.945 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.945 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.945 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.945 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.946 INFO:teuthology.orchestra.run.smithi032.stdout: "warn_on_stray_hosts": { 2024-04-25T00:44:44.946 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "warn_on_stray_hosts", 2024-04-25T00:44:44.946 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.946 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.946 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.946 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.946 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.946 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.946 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.946 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "raise a health warning if daemons are detected on a host that is not managed by cephadm", 2024-04-25T00:44:44.946 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.946 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.946 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.946 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.946 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.947 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.947 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.947 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "crash", 2024-04-25T00:44:44.947 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.947 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.947 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.947 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:44.947 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:44.947 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.947 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.947 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.947 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.947 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.947 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.947 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.948 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.948 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.948 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.948 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.948 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.948 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.948 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.948 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.948 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.948 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.948 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.948 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.948 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:44.948 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:44.949 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.949 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.949 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.949 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.949 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.949 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.949 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.949 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.949 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.949 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.949 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.949 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.949 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:44.949 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:44.949 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.950 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.950 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.950 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:44.950 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.950 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.950 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:44.950 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:44.950 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:44.950 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:44.950 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:44.950 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:44.950 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:44.950 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:44.950 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.951 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.951 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.951 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.951 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.951 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:44.951 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:44.951 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.951 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.951 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.951 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.951 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.951 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.951 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.951 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.952 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.952 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.952 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.952 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.952 INFO:teuthology.orchestra.run.smithi032.stdout: "retain_interval": { 2024-04-25T00:44:44.952 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "retain_interval", 2024-04-25T00:44:44.952 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.952 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.952 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.952 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "31536000", 2024-04-25T00:44:44.952 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.952 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.952 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.952 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "how long to retain crashes before pruning them", 2024-04-25T00:44:44.953 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.953 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.953 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.953 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.953 INFO:teuthology.orchestra.run.smithi032.stdout: "warn_recent_interval": { 2024-04-25T00:44:44.953 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "warn_recent_interval", 2024-04-25T00:44:44.953 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:44.953 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.953 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:44.953 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "1209600", 2024-04-25T00:44:44.953 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.953 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.953 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.953 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "time interval in which to warn about recent crashes", 2024-04-25T00:44:44.954 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.954 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.954 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.954 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.954 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:44.954 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.954 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:44.954 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "dashboard", 2024-04-25T00:44:44.954 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:44.954 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:44.954 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:44.954 INFO:teuthology.orchestra.run.smithi032.stdout: "ACCOUNT_LOCKOUT_ATTEMPTS": { 2024-04-25T00:44:44.954 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ACCOUNT_LOCKOUT_ATTEMPTS", 2024-04-25T00:44:44.955 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.955 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.955 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.955 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10", 2024-04-25T00:44:44.955 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.955 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.955 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.955 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.955 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.955 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.955 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.955 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.955 INFO:teuthology.orchestra.run.smithi032.stdout: "ALERTMANAGER_API_HOST": { 2024-04-25T00:44:44.955 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ALERTMANAGER_API_HOST", 2024-04-25T00:44:44.955 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.956 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.956 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.956 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.956 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.956 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.956 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.956 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.956 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.956 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.956 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.956 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.956 INFO:teuthology.orchestra.run.smithi032.stdout: "ALERTMANAGER_API_SSL_VERIFY": { 2024-04-25T00:44:44.956 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ALERTMANAGER_API_SSL_VERIFY", 2024-04-25T00:44:44.956 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.956 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.957 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.957 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.957 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.957 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.957 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.957 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.957 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.957 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.957 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.957 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.957 INFO:teuthology.orchestra.run.smithi032.stdout: "AUDIT_API_ENABLED": { 2024-04-25T00:44:44.957 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "AUDIT_API_ENABLED", 2024-04-25T00:44:44.957 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.957 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.957 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.958 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.958 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.958 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.958 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.958 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.958 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.958 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.958 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.958 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.958 INFO:teuthology.orchestra.run.smithi032.stdout: "AUDIT_API_LOG_PAYLOAD": { 2024-04-25T00:44:44.958 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "AUDIT_API_LOG_PAYLOAD", 2024-04-25T00:44:44.958 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.958 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.959 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.959 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.959 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.959 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.959 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.959 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.959 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.959 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.959 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.959 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.959 INFO:teuthology.orchestra.run.smithi032.stdout: "ENABLE_BROWSABLE_API": { 2024-04-25T00:44:44.959 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ENABLE_BROWSABLE_API", 2024-04-25T00:44:44.959 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.959 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.959 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.959 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.960 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.960 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.960 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.960 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.960 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.960 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.960 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.960 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.960 INFO:teuthology.orchestra.run.smithi032.stdout: "FEATURE_TOGGLE_CEPHFS": { 2024-04-25T00:44:44.960 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "FEATURE_TOGGLE_CEPHFS", 2024-04-25T00:44:44.960 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.960 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.960 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.960 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.960 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.961 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.961 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.961 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.961 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.961 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.961 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.961 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.961 INFO:teuthology.orchestra.run.smithi032.stdout: "FEATURE_TOGGLE_DASHBOARD": { 2024-04-25T00:44:44.961 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "FEATURE_TOGGLE_DASHBOARD", 2024-04-25T00:44:44.961 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.961 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.961 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.961 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.961 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.962 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.962 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.962 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.962 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.962 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.962 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.962 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.962 INFO:teuthology.orchestra.run.smithi032.stdout: "FEATURE_TOGGLE_ISCSI": { 2024-04-25T00:44:44.962 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "FEATURE_TOGGLE_ISCSI", 2024-04-25T00:44:44.962 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.962 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.962 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.962 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.962 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.962 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.963 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.963 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.963 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.963 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.963 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.963 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.963 INFO:teuthology.orchestra.run.smithi032.stdout: "FEATURE_TOGGLE_MIRRORING": { 2024-04-25T00:44:44.963 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "FEATURE_TOGGLE_MIRRORING", 2024-04-25T00:44:44.963 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.963 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.963 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.963 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.963 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.963 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.964 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.964 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.964 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.964 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.964 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.964 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.964 INFO:teuthology.orchestra.run.smithi032.stdout: "FEATURE_TOGGLE_NFS": { 2024-04-25T00:44:44.964 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "FEATURE_TOGGLE_NFS", 2024-04-25T00:44:44.964 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.964 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.964 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.964 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.964 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.964 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.964 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.965 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.965 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.965 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.965 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.965 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.965 INFO:teuthology.orchestra.run.smithi032.stdout: "FEATURE_TOGGLE_RBD": { 2024-04-25T00:44:44.965 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "FEATURE_TOGGLE_RBD", 2024-04-25T00:44:44.965 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.965 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.965 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.965 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.965 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.965 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.965 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.966 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.966 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.966 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.966 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.966 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.966 INFO:teuthology.orchestra.run.smithi032.stdout: "FEATURE_TOGGLE_RGW": { 2024-04-25T00:44:44.966 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "FEATURE_TOGGLE_RGW", 2024-04-25T00:44:44.966 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.966 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.966 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.966 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.966 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.966 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.966 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.967 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.967 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.967 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.967 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.967 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.967 INFO:teuthology.orchestra.run.smithi032.stdout: "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE": { 2024-04-25T00:44:44.967 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE", 2024-04-25T00:44:44.967 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.967 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.967 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.967 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.967 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.967 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.967 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.967 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.968 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.969 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.969 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.969 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.970 INFO:teuthology.orchestra.run.smithi032.stdout: "GRAFANA_API_PASSWORD": { 2024-04-25T00:44:44.970 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "GRAFANA_API_PASSWORD", 2024-04-25T00:44:44.970 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.970 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.970 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.970 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "admin", 2024-04-25T00:44:44.970 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.970 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.970 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.970 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.970 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.970 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.970 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.970 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.970 INFO:teuthology.orchestra.run.smithi032.stdout: "GRAFANA_API_SSL_VERIFY": { 2024-04-25T00:44:44.971 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "GRAFANA_API_SSL_VERIFY", 2024-04-25T00:44:44.971 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.971 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.971 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.971 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.971 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.971 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.971 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.971 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.971 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.971 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.971 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.971 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.971 INFO:teuthology.orchestra.run.smithi032.stdout: "GRAFANA_API_URL": { 2024-04-25T00:44:44.972 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "GRAFANA_API_URL", 2024-04-25T00:44:44.972 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.972 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.972 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.972 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.972 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.972 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.972 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.972 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.972 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.972 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.972 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.972 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.972 INFO:teuthology.orchestra.run.smithi032.stdout: "GRAFANA_API_USERNAME": { 2024-04-25T00:44:44.973 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "GRAFANA_API_USERNAME", 2024-04-25T00:44:44.973 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.973 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.973 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.973 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "admin", 2024-04-25T00:44:44.973 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.973 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.973 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.973 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.973 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.973 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.973 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.973 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.973 INFO:teuthology.orchestra.run.smithi032.stdout: "GRAFANA_FRONTEND_API_URL": { 2024-04-25T00:44:44.973 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "GRAFANA_FRONTEND_API_URL", 2024-04-25T00:44:44.974 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.974 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.974 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.974 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.974 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.974 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.974 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.974 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.974 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.974 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.974 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.974 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.974 INFO:teuthology.orchestra.run.smithi032.stdout: "GRAFANA_UPDATE_DASHBOARDS": { 2024-04-25T00:44:44.974 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "GRAFANA_UPDATE_DASHBOARDS", 2024-04-25T00:44:44.975 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.975 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.975 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.975 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.975 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.975 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.975 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.975 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.975 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.975 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.975 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.975 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.975 INFO:teuthology.orchestra.run.smithi032.stdout: "ISCSI_API_SSL_VERIFICATION": { 2024-04-25T00:44:44.976 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ISCSI_API_SSL_VERIFICATION", 2024-04-25T00:44:44.976 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.976 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.976 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.976 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.976 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.976 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.976 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.976 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.976 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.976 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.976 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.976 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.976 INFO:teuthology.orchestra.run.smithi032.stdout: "ISSUE_TRACKER_API_KEY": { 2024-04-25T00:44:44.977 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ISSUE_TRACKER_API_KEY", 2024-04-25T00:44:44.977 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.977 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.977 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.977 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.977 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.977 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.977 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.977 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.977 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.977 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.977 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.977 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.977 INFO:teuthology.orchestra.run.smithi032.stdout: "PROMETHEUS_API_HOST": { 2024-04-25T00:44:44.977 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PROMETHEUS_API_HOST", 2024-04-25T00:44:44.978 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.978 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.978 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.978 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.978 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.978 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.978 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.978 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.978 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.978 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.978 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.978 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.978 INFO:teuthology.orchestra.run.smithi032.stdout: "PROMETHEUS_API_SSL_VERIFY": { 2024-04-25T00:44:44.978 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PROMETHEUS_API_SSL_VERIFY", 2024-04-25T00:44:44.979 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.979 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.979 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.979 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.979 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.979 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.979 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.979 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.979 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.979 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.979 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.979 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.979 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_CHECK_COMPLEXITY_ENABLED": { 2024-04-25T00:44:44.979 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_CHECK_COMPLEXITY_ENABLED", 2024-04-25T00:44:44.980 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.980 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.980 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.980 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.980 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.980 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.980 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.980 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.980 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.980 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.980 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.980 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.980 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED": { 2024-04-25T00:44:44.980 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED", 2024-04-25T00:44:44.981 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.981 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.981 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.981 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.981 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.981 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.981 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.981 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.981 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.981 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.981 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.981 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.981 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_CHECK_LENGTH_ENABLED": { 2024-04-25T00:44:44.981 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_CHECK_LENGTH_ENABLED", 2024-04-25T00:44:44.981 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.981 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.982 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.982 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.982 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.982 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.982 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.982 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.982 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.982 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.982 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.982 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.982 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_CHECK_OLDPWD_ENABLED": { 2024-04-25T00:44:44.982 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_CHECK_OLDPWD_ENABLED", 2024-04-25T00:44:44.982 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.982 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.982 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.983 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.983 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.983 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.983 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.983 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.983 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.983 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.983 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.983 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.983 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED": { 2024-04-25T00:44:44.983 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED", 2024-04-25T00:44:44.983 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.983 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.983 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.984 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.984 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.984 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.984 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.984 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.984 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.984 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.984 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.984 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.984 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED": { 2024-04-25T00:44:44.984 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED", 2024-04-25T00:44:44.984 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.984 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.984 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.985 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.985 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.985 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.985 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.985 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.985 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.985 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.985 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.985 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.985 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_CHECK_USERNAME_ENABLED": { 2024-04-25T00:44:44.985 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_CHECK_USERNAME_ENABLED", 2024-04-25T00:44:44.985 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.985 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.985 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.986 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:44.986 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.986 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.986 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.986 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.986 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.986 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.986 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.986 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.986 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_ENABLED": { 2024-04-25T00:44:44.986 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_ENABLED", 2024-04-25T00:44:44.986 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.986 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.986 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.987 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.987 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.987 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.987 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.987 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.987 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.987 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.987 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.987 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.987 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_EXCLUSION_LIST": { 2024-04-25T00:44:44.987 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_EXCLUSION_LIST", 2024-04-25T00:44:44.987 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.987 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.987 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.987 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps", 2024-04-25T00:44:44.988 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.988 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.988 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.988 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.988 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.988 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.988 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.988 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.988 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_MIN_COMPLEXITY": { 2024-04-25T00:44:44.988 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_MIN_COMPLEXITY", 2024-04-25T00:44:44.988 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.988 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.988 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.989 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10", 2024-04-25T00:44:44.989 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.989 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.989 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.989 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.989 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.989 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.989 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.989 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.989 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_MIN_LENGTH": { 2024-04-25T00:44:44.989 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_MIN_LENGTH", 2024-04-25T00:44:44.989 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.989 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.989 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.990 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "8", 2024-04-25T00:44:44.990 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.990 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.990 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.990 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.990 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.990 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.990 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.990 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.990 INFO:teuthology.orchestra.run.smithi032.stdout: "REST_REQUESTS_TIMEOUT": { 2024-04-25T00:44:44.990 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "REST_REQUESTS_TIMEOUT", 2024-04-25T00:44:44.990 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.990 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.990 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.991 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "45", 2024-04-25T00:44:44.991 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.991 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.991 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.991 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.991 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.991 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.991 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.991 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.991 INFO:teuthology.orchestra.run.smithi032.stdout: "RGW_API_ACCESS_KEY": { 2024-04-25T00:44:44.991 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "RGW_API_ACCESS_KEY", 2024-04-25T00:44:44.991 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.991 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.991 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.991 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.991 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.992 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.992 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.992 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.992 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.992 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.992 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.992 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.992 INFO:teuthology.orchestra.run.smithi032.stdout: "RGW_API_ADMIN_RESOURCE": { 2024-04-25T00:44:44.992 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "RGW_API_ADMIN_RESOURCE", 2024-04-25T00:44:44.992 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.992 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.992 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.992 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "admin", 2024-04-25T00:44:44.993 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.993 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.993 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.993 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.993 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.993 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.993 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.993 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.993 INFO:teuthology.orchestra.run.smithi032.stdout: "RGW_API_SECRET_KEY": { 2024-04-25T00:44:44.993 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "RGW_API_SECRET_KEY", 2024-04-25T00:44:44.993 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.993 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.993 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.993 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.993 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.994 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.994 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.994 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.994 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.994 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.994 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.994 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.994 INFO:teuthology.orchestra.run.smithi032.stdout: "RGW_API_SSL_VERIFY": { 2024-04-25T00:44:44.994 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "RGW_API_SSL_VERIFY", 2024-04-25T00:44:44.994 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:44.994 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.994 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.994 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:44.994 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.995 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.995 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.995 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.995 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.995 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.995 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.995 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.995 INFO:teuthology.orchestra.run.smithi032.stdout: "USER_PWD_EXPIRATION_SPAN": { 2024-04-25T00:44:44.995 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "USER_PWD_EXPIRATION_SPAN", 2024-04-25T00:44:44.995 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.995 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.995 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.995 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "0", 2024-04-25T00:44:44.995 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.995 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.996 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.996 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.996 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.996 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.996 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.996 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.996 INFO:teuthology.orchestra.run.smithi032.stdout: "USER_PWD_EXPIRATION_WARNING_1": { 2024-04-25T00:44:44.996 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "USER_PWD_EXPIRATION_WARNING_1", 2024-04-25T00:44:44.996 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.996 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.996 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.996 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10", 2024-04-25T00:44:44.996 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.996 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.996 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.996 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.997 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.997 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.997 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.997 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.997 INFO:teuthology.orchestra.run.smithi032.stdout: "USER_PWD_EXPIRATION_WARNING_2": { 2024-04-25T00:44:44.997 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "USER_PWD_EXPIRATION_WARNING_2", 2024-04-25T00:44:44.997 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:44.997 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.997 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.997 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "5", 2024-04-25T00:44:44.997 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.997 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.997 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.997 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.997 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.997 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.998 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.998 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.998 INFO:teuthology.orchestra.run.smithi032.stdout: "cross_origin_url": { 2024-04-25T00:44:44.998 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "cross_origin_url", 2024-04-25T00:44:44.998 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.998 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.998 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.998 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.998 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.998 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.998 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.998 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.998 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.998 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.998 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.999 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.999 INFO:teuthology.orchestra.run.smithi032.stdout: "crt_file": { 2024-04-25T00:44:44.999 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "crt_file", 2024-04-25T00:44:44.999 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:44.999 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:44.999 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:44.999 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:44.999 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:44.999 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:44.999 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:44.999 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:44.999 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:44.999 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:44.999 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:44.999 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:44.999 INFO:teuthology.orchestra.run.smithi032.stdout: "debug": { 2024-04-25T00:44:45.000 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "debug", 2024-04-25T00:44:45.000 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.000 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.000 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.000 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.000 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.000 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.000 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.000 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Enable/disable debug options", 2024-04-25T00:44:45.000 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.000 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.000 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.000 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.000 INFO:teuthology.orchestra.run.smithi032.stdout: "jwt_token_ttl": { 2024-04-25T00:44:45.001 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "jwt_token_ttl", 2024-04-25T00:44:45.001 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.001 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.001 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.001 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "28800", 2024-04-25T00:44:45.001 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.001 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.001 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.001 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.001 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.001 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.001 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.001 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.001 INFO:teuthology.orchestra.run.smithi032.stdout: "key_file": { 2024-04-25T00:44:45.002 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "key_file", 2024-04-25T00:44:45.002 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.002 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.002 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.002 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.002 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.002 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.002 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.002 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.002 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.002 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.002 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.002 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.002 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.003 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.003 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.003 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.003 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.003 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.003 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.003 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.003 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.003 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.003 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.003 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.003 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.003 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.003 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.003 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.003 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.003 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.004 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.004 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.004 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.004 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.004 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.004 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.004 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.004 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.004 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.004 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.004 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.004 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.004 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.004 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.004 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.004 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.005 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.005 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.005 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.005 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.005 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.005 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.005 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.005 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.005 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.005 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.005 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.005 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.005 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.005 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.005 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.005 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.005 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.006 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.006 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.006 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.006 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.006 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.006 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.006 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.006 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.006 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.006 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.006 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.006 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.006 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.006 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.006 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.007 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.007 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.007 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.007 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.007 INFO:teuthology.orchestra.run.smithi032.stdout: "motd": { 2024-04-25T00:44:45.007 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "motd", 2024-04-25T00:44:45.007 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.007 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.007 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.007 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.007 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.007 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.007 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.007 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "The message of the day", 2024-04-25T00:44:45.007 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.008 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.008 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.008 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.008 INFO:teuthology.orchestra.run.smithi032.stdout: "redirect_resolve_ip_addr": { 2024-04-25T00:44:45.008 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "redirect_resolve_ip_addr", 2024-04-25T00:44:45.008 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.008 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.008 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.008 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.008 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.008 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.008 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.008 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.008 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.008 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.008 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.009 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.009 INFO:teuthology.orchestra.run.smithi032.stdout: "server_addr": { 2024-04-25T00:44:45.009 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "server_addr", 2024-04-25T00:44:45.009 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.009 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.009 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.009 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "::", 2024-04-25T00:44:45.009 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.009 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.009 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.009 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.009 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.009 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.009 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.009 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.009 INFO:teuthology.orchestra.run.smithi032.stdout: "server_port": { 2024-04-25T00:44:45.009 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "server_port", 2024-04-25T00:44:45.010 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.010 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.010 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.010 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "8080", 2024-04-25T00:44:45.010 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.010 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.010 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.010 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.010 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.010 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.010 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.010 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.010 INFO:teuthology.orchestra.run.smithi032.stdout: "ssl": { 2024-04-25T00:44:45.010 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ssl", 2024-04-25T00:44:45.010 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.011 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.011 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.011 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.011 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.011 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.011 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.011 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.011 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.011 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.011 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.011 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.011 INFO:teuthology.orchestra.run.smithi032.stdout: "ssl_server_port": { 2024-04-25T00:44:45.011 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ssl_server_port", 2024-04-25T00:44:45.011 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.011 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.012 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.012 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "8443", 2024-04-25T00:44:45.012 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.012 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.012 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.012 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.012 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.012 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.012 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.012 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.012 INFO:teuthology.orchestra.run.smithi032.stdout: "standby_behaviour": { 2024-04-25T00:44:45.012 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "standby_behaviour", 2024-04-25T00:44:45.012 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.012 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.012 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.012 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "redirect", 2024-04-25T00:44:45.012 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.013 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.013 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.013 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.013 INFO:teuthology.orchestra.run.smithi032.stdout: "redirect" 2024-04-25T00:44:45.013 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.013 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.013 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.013 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.013 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.013 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.013 INFO:teuthology.orchestra.run.smithi032.stdout: "standby_error_status_code": { 2024-04-25T00:44:45.013 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "standby_error_status_code", 2024-04-25T00:44:45.013 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.014 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.014 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.014 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "500", 2024-04-25T00:44:45.014 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "400", 2024-04-25T00:44:45.014 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "599", 2024-04-25T00:44:45.014 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.014 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.014 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.014 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.014 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.014 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.014 INFO:teuthology.orchestra.run.smithi032.stdout: "url_prefix": { 2024-04-25T00:44:45.014 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "url_prefix", 2024-04-25T00:44:45.014 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.014 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.015 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.015 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.015 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.015 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.015 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.015 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.015 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.015 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.015 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.015 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.015 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.015 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.015 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.015 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "devicehealth", 2024-04-25T00:44:45.015 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.016 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.016 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.016 INFO:teuthology.orchestra.run.smithi032.stdout: "enable_monitoring": { 2024-04-25T00:44:45.016 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "enable_monitoring", 2024-04-25T00:44:45.016 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.016 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.016 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.016 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.016 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.016 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.016 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.016 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "monitor device health metrics", 2024-04-25T00:44:45.016 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.016 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.017 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.017 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.017 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.017 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.017 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.017 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.017 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.017 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.017 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.017 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.017 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.017 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.017 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.017 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.017 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.018 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.018 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.018 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.018 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.018 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.018 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.018 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.018 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.018 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.018 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.018 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.018 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.018 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.019 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.019 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.019 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.019 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.019 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.019 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.019 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.019 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.019 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.019 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.019 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.019 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.019 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.019 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.020 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.020 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.020 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.020 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.020 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.020 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.020 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.020 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.020 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.020 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.020 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.020 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.020 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.021 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.021 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.021 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.021 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.021 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.021 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.021 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.021 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.021 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.021 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.021 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.021 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.021 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.021 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.021 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.021 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.022 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.022 INFO:teuthology.orchestra.run.smithi032.stdout: "mark_out_threshold": { 2024-04-25T00:44:45.022 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "mark_out_threshold", 2024-04-25T00:44:45.022 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.022 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.022 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.022 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "2419200", 2024-04-25T00:44:45.022 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.022 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.022 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.022 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "automatically mark OSD if it may fail before this long", 2024-04-25T00:44:45.022 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.022 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.023 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.023 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.023 INFO:teuthology.orchestra.run.smithi032.stdout: "pool_name": { 2024-04-25T00:44:45.023 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "pool_name", 2024-04-25T00:44:45.023 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.023 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.023 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.023 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "device_health_metrics", 2024-04-25T00:44:45.023 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.023 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.023 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.023 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "name of pool in which to store device health metrics", 2024-04-25T00:44:45.023 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.023 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.023 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.024 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.024 INFO:teuthology.orchestra.run.smithi032.stdout: "retention_period": { 2024-04-25T00:44:45.024 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "retention_period", 2024-04-25T00:44:45.024 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.024 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.024 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.024 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "15552000", 2024-04-25T00:44:45.024 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.024 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.024 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.024 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "how long to retain device health metrics", 2024-04-25T00:44:45.024 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.024 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.024 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.024 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.025 INFO:teuthology.orchestra.run.smithi032.stdout: "scrape_frequency": { 2024-04-25T00:44:45.025 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "scrape_frequency", 2024-04-25T00:44:45.025 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.025 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.025 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.025 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "86400", 2024-04-25T00:44:45.025 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.025 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.025 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.025 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "how frequently to scrape device health metrics", 2024-04-25T00:44:45.025 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.025 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.025 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.025 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.025 INFO:teuthology.orchestra.run.smithi032.stdout: "self_heal": { 2024-04-25T00:44:45.025 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "self_heal", 2024-04-25T00:44:45.026 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.026 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.026 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.026 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.026 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.026 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.026 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.026 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "preemptively heal cluster around devices that may fail", 2024-04-25T00:44:45.026 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.026 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.026 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.026 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.026 INFO:teuthology.orchestra.run.smithi032.stdout: "sleep_interval": { 2024-04-25T00:44:45.026 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "sleep_interval", 2024-04-25T00:44:45.027 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.027 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.027 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.027 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "600", 2024-04-25T00:44:45.027 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.027 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.027 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.027 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "how frequently to wake up and check device health", 2024-04-25T00:44:45.027 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.027 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.027 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.027 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.027 INFO:teuthology.orchestra.run.smithi032.stdout: "warn_threshold": { 2024-04-25T00:44:45.027 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "warn_threshold", 2024-04-25T00:44:45.028 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.028 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.028 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.028 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "7257600", 2024-04-25T00:44:45.028 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.028 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.028 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.028 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "raise health warning if OSD may fail before this long", 2024-04-25T00:44:45.028 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.028 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.028 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.028 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.028 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.028 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.028 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.029 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "diskprediction_local", 2024-04-25T00:44:45.029 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.029 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.029 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.029 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.029 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.029 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.029 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.029 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.029 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.029 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.029 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.029 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.029 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.029 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.030 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.030 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.030 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.030 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.030 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.030 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.030 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.030 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.030 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.030 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.030 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.030 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.030 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.030 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.031 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.031 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.031 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.031 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.031 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.031 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.031 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.031 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.031 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.031 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.031 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.031 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.031 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.031 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.031 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.032 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.032 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.032 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.032 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.032 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.032 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.032 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.032 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.032 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.032 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.032 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.032 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.032 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.032 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.032 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.033 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.033 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.033 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.033 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.033 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.033 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.033 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.033 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.033 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.033 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.033 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.033 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.033 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.033 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.033 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.034 INFO:teuthology.orchestra.run.smithi032.stdout: "predict_interval": { 2024-04-25T00:44:45.034 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "predict_interval", 2024-04-25T00:44:45.034 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.034 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.034 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.034 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "86400", 2024-04-25T00:44:45.034 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.034 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.034 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.034 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.034 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.034 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.034 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.034 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.035 INFO:teuthology.orchestra.run.smithi032.stdout: "predictor_model": { 2024-04-25T00:44:45.035 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "predictor_model", 2024-04-25T00:44:45.035 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.035 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.035 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.035 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "prophetstor", 2024-04-25T00:44:45.035 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.035 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.035 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.035 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.035 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.035 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.035 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.035 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.035 INFO:teuthology.orchestra.run.smithi032.stdout: "sleep_interval": { 2024-04-25T00:44:45.036 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "sleep_interval", 2024-04-25T00:44:45.036 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.036 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.036 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.036 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "600", 2024-04-25T00:44:45.036 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.036 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.036 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.036 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.036 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.036 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.036 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.036 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.036 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.037 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.037 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.037 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "influx", 2024-04-25T00:44:45.037 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": false, 2024-04-25T00:44:45.037 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "influxdb python module not found", 2024-04-25T00:44:45.037 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.037 INFO:teuthology.orchestra.run.smithi032.stdout: "batch_size": { 2024-04-25T00:44:45.037 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "batch_size", 2024-04-25T00:44:45.037 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.037 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.037 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.037 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "5000", 2024-04-25T00:44:45.037 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.037 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.037 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.038 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "How big batches of data points should be when sending to InfluxDB.", 2024-04-25T00:44:45.038 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.038 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.038 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.038 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.038 INFO:teuthology.orchestra.run.smithi032.stdout: "database": { 2024-04-25T00:44:45.038 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "database", 2024-04-25T00:44:45.038 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.038 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.038 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.038 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "ceph", 2024-04-25T00:44:45.038 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.038 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.038 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.039 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.", 2024-04-25T00:44:45.039 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.039 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.039 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.039 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.039 INFO:teuthology.orchestra.run.smithi032.stdout: "hostname": { 2024-04-25T00:44:45.039 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "hostname", 2024-04-25T00:44:45.039 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.039 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.039 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.039 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.039 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.039 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.039 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.040 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "InfluxDB server hostname", 2024-04-25T00:44:45.040 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.040 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.040 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.040 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.040 INFO:teuthology.orchestra.run.smithi032.stdout: "interval": { 2024-04-25T00:44:45.040 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "interval", 2024-04-25T00:44:45.040 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.040 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.040 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.040 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "30", 2024-04-25T00:44:45.040 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "5", 2024-04-25T00:44:45.040 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.040 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.040 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Time between reports to InfluxDB. Default 30 seconds.", 2024-04-25T00:44:45.041 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.041 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.041 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.041 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.041 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.041 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.041 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.041 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.041 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.041 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.041 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.041 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.041 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.042 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.042 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.042 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.042 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.042 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.042 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.042 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.042 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.042 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.042 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.042 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.042 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.042 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.042 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.043 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.043 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.043 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.043 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.043 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.043 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.043 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.043 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.043 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.043 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.043 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.043 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.043 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.044 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.044 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.044 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.044 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.044 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.044 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.044 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.044 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.044 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.044 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.044 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.044 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.044 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.044 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.044 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.045 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.045 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.045 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.045 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.045 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.045 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.045 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.045 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.045 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.045 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.045 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.045 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.045 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.045 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.045 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.046 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.046 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.046 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.046 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.046 INFO:teuthology.orchestra.run.smithi032.stdout: "password": { 2024-04-25T00:44:45.046 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "password", 2024-04-25T00:44:45.046 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.046 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.046 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.046 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.046 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.046 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.046 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.046 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "password of InfluxDB server user", 2024-04-25T00:44:45.046 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.047 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.047 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.047 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.047 INFO:teuthology.orchestra.run.smithi032.stdout: "port": { 2024-04-25T00:44:45.047 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "port", 2024-04-25T00:44:45.047 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.047 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.047 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.047 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "8086", 2024-04-25T00:44:45.047 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.047 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.047 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.047 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "InfluxDB server port", 2024-04-25T00:44:45.047 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.047 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.047 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.048 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.048 INFO:teuthology.orchestra.run.smithi032.stdout: "ssl": { 2024-04-25T00:44:45.048 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ssl", 2024-04-25T00:44:45.048 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.048 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.048 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.048 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "false", 2024-04-25T00:44:45.048 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.048 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.048 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.048 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Use https connection for InfluxDB server. Use \"true\" or \"false\".", 2024-04-25T00:44:45.048 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.048 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.048 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.048 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.049 INFO:teuthology.orchestra.run.smithi032.stdout: "threads": { 2024-04-25T00:44:45.049 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "threads", 2024-04-25T00:44:45.049 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.049 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.049 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.049 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "5", 2024-04-25T00:44:45.049 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "1", 2024-04-25T00:44:45.049 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "32", 2024-04-25T00:44:45.049 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.049 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "How many worker threads should be spawned for sending data to InfluxDB.", 2024-04-25T00:44:45.049 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.049 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.049 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.049 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.049 INFO:teuthology.orchestra.run.smithi032.stdout: "username": { 2024-04-25T00:44:45.049 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "username", 2024-04-25T00:44:45.050 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.050 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.050 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.050 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.050 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.050 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.050 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.050 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "username of InfluxDB server user", 2024-04-25T00:44:45.050 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.050 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.050 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.050 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.050 INFO:teuthology.orchestra.run.smithi032.stdout: "verify_ssl": { 2024-04-25T00:44:45.050 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "verify_ssl", 2024-04-25T00:44:45.051 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.051 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.051 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.051 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "true", 2024-04-25T00:44:45.051 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.051 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.051 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.051 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Verify https cert for InfluxDB server. Use \"true\" or \"false\".", 2024-04-25T00:44:45.051 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.051 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.051 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.051 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.051 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.051 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.052 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.052 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "insights", 2024-04-25T00:44:45.052 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.052 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.052 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.052 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.052 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.052 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.052 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.052 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.052 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.052 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.052 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.053 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.053 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.053 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.053 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.053 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.053 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.053 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.053 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.053 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.053 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.053 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.053 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.053 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.053 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.054 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.054 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.054 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.054 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.054 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.054 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.054 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.054 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.054 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.054 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.054 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.054 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.054 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.054 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.055 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.055 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.055 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.055 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.055 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.055 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.055 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.055 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.055 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.055 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.055 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.055 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.055 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.055 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.055 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.056 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.056 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.056 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.056 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.056 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.056 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.056 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.056 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.056 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.056 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.056 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.056 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.056 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.056 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.057 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.057 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.057 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.057 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.057 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.057 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.057 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.057 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.057 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "iostat", 2024-04-25T00:44:45.057 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.057 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.057 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.057 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.057 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.057 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.058 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.058 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.058 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.058 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.058 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.058 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.058 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.058 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.058 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.058 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.058 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.058 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.058 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.058 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.059 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.059 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.059 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.059 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.059 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.059 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.059 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.059 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.059 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.059 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.059 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.059 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.059 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.060 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.060 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.060 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.060 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.060 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.060 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.060 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.060 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.060 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.060 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.060 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.060 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.060 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.060 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.060 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.061 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.061 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.061 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.061 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.061 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.061 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.061 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.061 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.061 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.061 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.061 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.061 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.061 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.061 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.061 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.062 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.062 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.062 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.062 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.062 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.062 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.062 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.062 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.062 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.062 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.062 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.062 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.062 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.062 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "k8sevents", 2024-04-25T00:44:45.063 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.063 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.063 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.063 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph_event_retention_days": { 2024-04-25T00:44:45.063 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ceph_event_retention_days", 2024-04-25T00:44:45.063 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.063 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.063 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.063 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "7", 2024-04-25T00:44:45.063 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.063 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.063 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.063 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Days to hold ceph event information within local cache", 2024-04-25T00:44:45.063 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.063 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.064 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.064 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.064 INFO:teuthology.orchestra.run.smithi032.stdout: "config_check_secs": { 2024-04-25T00:44:45.064 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "config_check_secs", 2024-04-25T00:44:45.064 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.064 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.064 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.064 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10", 2024-04-25T00:44:45.064 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "10", 2024-04-25T00:44:45.064 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.064 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.064 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "interval (secs) to check for cluster configuration changes", 2024-04-25T00:44:45.064 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.064 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.064 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.065 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.065 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.065 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.065 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.065 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.065 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.065 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.065 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.065 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.065 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.065 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.065 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.065 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.065 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.066 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.066 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.066 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.066 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.066 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.066 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.066 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.066 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.066 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.066 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.066 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.066 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.066 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.066 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.067 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.067 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.067 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.067 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.067 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.067 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.067 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.067 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.067 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.067 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.067 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.067 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.067 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.067 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.067 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.068 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.068 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.068 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.068 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.068 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.068 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.068 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.068 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.068 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.068 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.068 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.068 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.068 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.068 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.068 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.069 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.069 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.069 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.069 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.069 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.069 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.069 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.069 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.069 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.069 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.069 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.069 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.069 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.069 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.069 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.070 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.070 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "localpool", 2024-04-25T00:44:45.070 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.070 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.070 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.070 INFO:teuthology.orchestra.run.smithi032.stdout: "failure_domain": { 2024-04-25T00:44:45.070 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "failure_domain", 2024-04-25T00:44:45.070 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.070 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.070 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.070 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "host", 2024-04-25T00:44:45.070 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.070 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.070 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.070 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "failure domain for any created local pool", 2024-04-25T00:44:45.071 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "what failure domain we should separate data replicas across.", 2024-04-25T00:44:45.071 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.071 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.071 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.071 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.071 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.071 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.071 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.071 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.071 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.071 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.071 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.071 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.071 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.071 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.072 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.072 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.072 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.072 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.072 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.072 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.072 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.072 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.072 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.072 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.072 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.072 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.072 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.072 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.072 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.073 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.073 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.073 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.073 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.073 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.073 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.073 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.073 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.073 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.073 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.073 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.073 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.073 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.073 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.073 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.074 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.074 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.074 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.074 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.074 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.074 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.074 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.074 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.074 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.074 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.074 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.074 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.074 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.074 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.075 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.075 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.075 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.075 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.075 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.075 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.075 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.075 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.075 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.075 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.075 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.075 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.075 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.075 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.075 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.075 INFO:teuthology.orchestra.run.smithi032.stdout: "min_size": { 2024-04-25T00:44:45.076 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "min_size", 2024-04-25T00:44:45.076 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.076 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.076 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.076 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.076 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.076 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.076 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.076 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "default min_size for any created local pool", 2024-04-25T00:44:45.076 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "value to set min_size to (unchanged from Ceph's default if this option is not set)", 2024-04-25T00:44:45.076 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.076 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.076 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.076 INFO:teuthology.orchestra.run.smithi032.stdout: "num_rep": { 2024-04-25T00:44:45.076 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "num_rep", 2024-04-25T00:44:45.077 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.077 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.077 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.077 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "3", 2024-04-25T00:44:45.077 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.077 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.077 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.077 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "default replica count for any created local pool", 2024-04-25T00:44:45.077 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.077 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.077 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.077 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.077 INFO:teuthology.orchestra.run.smithi032.stdout: "pg_num": { 2024-04-25T00:44:45.077 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "pg_num", 2024-04-25T00:44:45.077 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.078 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.078 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.078 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "128", 2024-04-25T00:44:45.078 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.078 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.078 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.078 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "default pg_num for any created local pool", 2024-04-25T00:44:45.078 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.078 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.078 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.078 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.079 INFO:teuthology.orchestra.run.smithi032.stdout: "prefix": { 2024-04-25T00:44:45.079 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "prefix", 2024-04-25T00:44:45.079 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.079 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.079 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.079 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.079 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.079 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.079 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.079 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "name prefix for any created local pool", 2024-04-25T00:44:45.079 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.079 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.079 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.079 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.079 INFO:teuthology.orchestra.run.smithi032.stdout: "subtree": { 2024-04-25T00:44:45.080 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "subtree", 2024-04-25T00:44:45.080 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.080 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.080 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.080 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "rack", 2024-04-25T00:44:45.080 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.080 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.080 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.080 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "CRUSH level for which to create a local pool", 2024-04-25T00:44:45.080 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "which CRUSH subtree type the module should create a pool for.", 2024-04-25T00:44:45.080 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.080 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.080 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.080 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.081 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.081 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.081 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "mds_autoscaler", 2024-04-25T00:44:45.081 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.081 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.081 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.081 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.081 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.081 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.081 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.081 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.081 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.081 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.081 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.081 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.082 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.082 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.082 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.082 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.082 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.082 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.082 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.082 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.082 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.082 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.082 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.082 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.082 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.082 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.083 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.083 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.083 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.083 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.083 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.083 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.083 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.083 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.083 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.083 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.083 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.083 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.083 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.083 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.083 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.084 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.084 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.084 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.084 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.084 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.084 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.084 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.084 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.084 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.084 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.084 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.084 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.084 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.084 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.084 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.085 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.085 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.085 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.085 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.085 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.085 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.085 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.085 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.085 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.085 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.085 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.085 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.085 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.085 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.085 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.086 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.086 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.086 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.086 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.086 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.086 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "mirroring", 2024-04-25T00:44:45.086 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.086 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.086 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.086 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.086 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.086 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.086 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.086 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.087 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.087 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.087 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.087 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.087 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.087 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.087 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.087 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.087 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.087 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.087 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.087 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.087 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.087 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.087 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.087 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.088 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.088 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.088 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.088 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.088 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.088 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.088 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.088 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.088 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.088 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.088 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.088 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.088 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.088 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.088 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.089 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.089 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.089 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.089 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.089 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.089 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.089 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.089 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.089 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.089 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.089 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.089 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.089 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.089 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.090 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.090 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.090 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.090 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.090 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.090 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.090 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.090 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.090 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.090 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.090 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.090 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.090 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.090 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.091 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.091 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.091 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.091 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.091 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.091 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.091 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.091 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.091 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.091 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "nfs", 2024-04-25T00:44:45.091 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.091 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.091 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.091 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.092 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.092 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.092 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.092 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.092 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.092 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.092 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.092 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.092 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.092 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.092 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.092 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.092 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.092 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.093 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.093 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.093 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.093 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.093 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.093 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.093 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.093 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.093 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.093 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.093 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.093 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.093 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.093 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.093 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.094 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.094 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.094 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.094 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.094 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.094 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.094 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.094 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.094 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.094 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.094 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.094 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.094 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.094 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.095 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.095 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.095 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.095 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.095 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.095 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.095 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.095 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.095 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.095 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.095 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.095 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.095 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.095 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.095 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.096 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.096 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.096 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.096 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.096 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.096 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.096 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.096 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.096 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.096 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.096 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.096 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.096 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.096 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.096 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "orchestrator", 2024-04-25T00:44:45.097 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.097 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.097 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.097 INFO:teuthology.orchestra.run.smithi032.stdout: "fail_fs": { 2024-04-25T00:44:45.097 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "fail_fs", 2024-04-25T00:44:45.097 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.097 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.097 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.097 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.097 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.097 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.097 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.097 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Fail filesystem for rapid multi-rank mds upgrade", 2024-04-25T00:44:45.097 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.097 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.098 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.098 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.098 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.098 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.098 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.098 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.098 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.098 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.098 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.098 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.098 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.098 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.098 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.098 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.099 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.099 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.099 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.099 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.099 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.099 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.099 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.099 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.099 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.099 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.099 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.099 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.099 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.099 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.099 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.099 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.100 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.100 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.100 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.100 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.100 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.100 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.100 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.100 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.100 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.100 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.100 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.100 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.100 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.100 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.100 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.101 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.101 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.101 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.101 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.101 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.101 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.101 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.101 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.101 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.101 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.101 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.101 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.101 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.101 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.101 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.102 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.102 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.102 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.102 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.102 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.102 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.102 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.102 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.102 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.102 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.102 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.102 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.102 INFO:teuthology.orchestra.run.smithi032.stdout: "orchestrator": { 2024-04-25T00:44:45.102 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "orchestrator", 2024-04-25T00:44:45.102 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.103 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.103 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.103 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.103 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.103 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.103 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.103 INFO:teuthology.orchestra.run.smithi032.stdout: "cephadm", 2024-04-25T00:44:45.103 INFO:teuthology.orchestra.run.smithi032.stdout: "rook", 2024-04-25T00:44:45.103 INFO:teuthology.orchestra.run.smithi032.stdout: "test_orchestrator" 2024-04-25T00:44:45.103 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.103 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Orchestrator backend", 2024-04-25T00:44:45.103 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.103 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.103 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.103 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.104 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.104 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.104 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.104 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "osd_perf_query", 2024-04-25T00:44:45.104 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.104 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.104 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.104 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.104 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.104 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.104 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.104 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.104 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.104 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.105 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.105 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.105 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.105 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.105 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.105 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.105 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.105 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.105 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.105 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.105 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.105 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.105 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.105 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.105 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.106 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.106 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.106 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.106 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.106 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.106 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.106 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.106 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.106 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.106 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.106 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.106 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.106 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.106 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.107 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.107 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.107 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.107 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.107 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.107 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.107 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.107 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.107 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.107 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.107 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.107 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.107 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.107 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.107 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.108 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.108 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.108 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.108 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.108 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.108 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.108 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.108 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.108 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.108 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.108 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.108 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.108 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.108 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.108 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.109 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.109 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.109 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.109 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.109 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.109 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.109 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.109 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "osd_support", 2024-04-25T00:44:45.109 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.109 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.109 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.109 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.109 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.109 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.110 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.110 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.110 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.110 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.110 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.110 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.110 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.110 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.110 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.110 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.110 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.110 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.110 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.110 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.111 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.111 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.111 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.111 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.111 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.111 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.111 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.111 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.111 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.111 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.111 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.111 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.111 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.111 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.111 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.112 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.112 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.112 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.112 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.112 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.112 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.112 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.112 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.112 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.112 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.112 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.112 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.112 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.112 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.112 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.113 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.113 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.113 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.113 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.113 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.113 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.113 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.113 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.113 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.113 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.113 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.113 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.113 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.113 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.113 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.113 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.114 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.114 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.114 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.114 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.114 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.114 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.114 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.114 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.114 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.114 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.114 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "pg_autoscaler", 2024-04-25T00:44:45.114 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.114 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.114 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.115 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.115 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.115 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.115 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.115 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.115 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.115 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.115 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.115 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.115 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.115 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.115 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.115 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.115 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.115 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.116 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.116 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.116 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.116 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.116 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.116 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.116 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.116 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.116 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.116 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.116 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.116 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.116 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.116 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.116 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.117 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.117 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.117 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.117 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.117 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.117 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.117 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.117 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.117 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.117 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.117 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.117 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.117 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.117 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.117 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.118 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.118 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.118 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.118 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.118 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.118 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.118 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.118 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.118 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.118 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.118 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.118 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.118 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.119 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.119 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.119 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.119 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.119 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.119 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.119 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.119 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.119 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.119 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.119 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.119 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.119 INFO:teuthology.orchestra.run.smithi032.stdout: "sleep_interval": { 2024-04-25T00:44:45.119 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "sleep_interval", 2024-04-25T00:44:45.120 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.120 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.120 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.120 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "60", 2024-04-25T00:44:45.120 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.120 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.120 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.120 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.120 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.120 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.120 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.120 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.120 INFO:teuthology.orchestra.run.smithi032.stdout: "threshold": { 2024-04-25T00:44:45.120 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "threshold", 2024-04-25T00:44:45.120 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "float", 2024-04-25T00:44:45.121 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.121 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.121 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "3.0", 2024-04-25T00:44:45.121 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "1.0", 2024-04-25T00:44:45.121 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.121 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.121 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "scaling threshold", 2024-04-25T00:44:45.121 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0", 2024-04-25T00:44:45.121 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.121 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.121 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.121 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.121 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.121 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.122 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "progress", 2024-04-25T00:44:45.122 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.122 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.122 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.122 INFO:teuthology.orchestra.run.smithi032.stdout: "allow_pg_recovery_event": { 2024-04-25T00:44:45.122 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "allow_pg_recovery_event", 2024-04-25T00:44:45.122 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.122 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.122 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.122 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.122 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.122 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.122 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.122 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "allow the module to show pg recovery progress", 2024-04-25T00:44:45.122 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.123 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.123 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.123 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.123 INFO:teuthology.orchestra.run.smithi032.stdout: "enabled": { 2024-04-25T00:44:45.123 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "enabled", 2024-04-25T00:44:45.123 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.123 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.123 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.123 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.123 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.123 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.123 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.123 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.124 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.124 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.124 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.124 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.124 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.124 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.124 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.124 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.124 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.124 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.124 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.124 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.124 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.124 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.124 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.125 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.125 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.125 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.125 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.125 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.125 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.125 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.125 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.125 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.125 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.125 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.125 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.125 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.125 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.125 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.126 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.126 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.126 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.126 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.126 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.126 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.126 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.126 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.126 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.126 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.126 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.126 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.126 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.126 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.126 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.126 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.127 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.127 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.127 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.127 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.127 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.127 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.127 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.127 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.127 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.127 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.127 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.127 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.127 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.127 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.127 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.128 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.128 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.128 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.128 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.128 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.128 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.128 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.128 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.128 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.128 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.128 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.128 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.128 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.128 INFO:teuthology.orchestra.run.smithi032.stdout: "max_completed_events": { 2024-04-25T00:44:45.128 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "max_completed_events", 2024-04-25T00:44:45.129 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.129 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.129 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.129 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "50", 2024-04-25T00:44:45.129 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.129 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.129 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.129 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "number of past completed events to remember", 2024-04-25T00:44:45.129 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.129 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.129 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.129 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.129 INFO:teuthology.orchestra.run.smithi032.stdout: "sleep_interval": { 2024-04-25T00:44:45.129 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "sleep_interval", 2024-04-25T00:44:45.129 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.130 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.130 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.130 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "5", 2024-04-25T00:44:45.130 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.130 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.130 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.130 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "how long the module is going to sleep", 2024-04-25T00:44:45.130 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.130 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.130 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.130 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.130 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.130 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.130 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.131 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "prometheus", 2024-04-25T00:44:45.131 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.131 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.131 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.131 INFO:teuthology.orchestra.run.smithi032.stdout: "cache": { 2024-04-25T00:44:45.131 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "cache", 2024-04-25T00:44:45.131 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.131 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.131 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.131 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.131 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.131 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.131 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.132 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.132 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.132 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.132 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.132 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.132 INFO:teuthology.orchestra.run.smithi032.stdout: "exclude_perf_counters": { 2024-04-25T00:44:45.132 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "exclude_perf_counters", 2024-04-25T00:44:45.132 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.132 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.132 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.132 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.132 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.132 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.132 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.132 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Do not include perf-counters in the metrics output", 2024-04-25T00:44:45.133 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.", 2024-04-25T00:44:45.133 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.133 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.133 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.133 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.133 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.133 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.133 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.133 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.133 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.133 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.133 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.133 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.133 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.134 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.134 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.134 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.134 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.134 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.134 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.134 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.134 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.134 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.134 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.134 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.134 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.134 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.134 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.135 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.135 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.135 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.135 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.135 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.135 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.135 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.135 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.135 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.135 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.135 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.135 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.135 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.135 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.136 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.136 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.136 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.136 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.136 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.136 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.136 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.136 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.136 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.136 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.136 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.136 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.136 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.136 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.137 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.137 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.137 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.137 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.137 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.137 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.137 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.137 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.137 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.137 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.137 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.137 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.137 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.137 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.137 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.138 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.138 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.138 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.138 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_stats_pools": { 2024-04-25T00:44:45.138 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rbd_stats_pools", 2024-04-25T00:44:45.138 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.138 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.138 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.138 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.138 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.138 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.138 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.138 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.138 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.139 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.139 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.139 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.139 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_stats_pools_refresh_interval": { 2024-04-25T00:44:45.139 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rbd_stats_pools_refresh_interval", 2024-04-25T00:44:45.139 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.139 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.139 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.139 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "300", 2024-04-25T00:44:45.139 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.139 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.139 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.139 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.139 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.139 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.140 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.140 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.140 INFO:teuthology.orchestra.run.smithi032.stdout: "scrape_interval": { 2024-04-25T00:44:45.140 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "scrape_interval", 2024-04-25T00:44:45.140 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "float", 2024-04-25T00:44:45.140 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.140 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.140 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "15.0", 2024-04-25T00:44:45.140 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.140 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.140 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.140 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.140 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.140 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.140 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.141 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.141 INFO:teuthology.orchestra.run.smithi032.stdout: "server_addr": { 2024-04-25T00:44:45.141 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "server_addr", 2024-04-25T00:44:45.141 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.141 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.141 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.141 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "::", 2024-04-25T00:44:45.141 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.141 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.141 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.141 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "the IPv4 or IPv6 address on which the module listens for HTTP requests", 2024-04-25T00:44:45.141 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.141 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.141 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.142 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.142 INFO:teuthology.orchestra.run.smithi032.stdout: "server_port": { 2024-04-25T00:44:45.142 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "server_port", 2024-04-25T00:44:45.142 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.142 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.142 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.142 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "9283", 2024-04-25T00:44:45.142 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.142 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.142 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.142 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "the port on which the module listens for HTTP requests", 2024-04-25T00:44:45.142 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.142 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.142 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.142 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.143 INFO:teuthology.orchestra.run.smithi032.stdout: "stale_cache_strategy": { 2024-04-25T00:44:45.143 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "stale_cache_strategy", 2024-04-25T00:44:45.143 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.143 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.143 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.143 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "log", 2024-04-25T00:44:45.143 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.143 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.143 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.143 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.143 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.143 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.143 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.143 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.144 INFO:teuthology.orchestra.run.smithi032.stdout: "standby_behaviour": { 2024-04-25T00:44:45.144 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "standby_behaviour", 2024-04-25T00:44:45.144 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.144 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.144 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.144 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "default", 2024-04-25T00:44:45.144 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.144 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.144 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.144 INFO:teuthology.orchestra.run.smithi032.stdout: "default", 2024-04-25T00:44:45.144 INFO:teuthology.orchestra.run.smithi032.stdout: "error" 2024-04-25T00:44:45.144 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.144 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.144 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.144 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.145 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.145 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.145 INFO:teuthology.orchestra.run.smithi032.stdout: "standby_error_status_code": { 2024-04-25T00:44:45.145 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "standby_error_status_code", 2024-04-25T00:44:45.145 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.145 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.145 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.145 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "500", 2024-04-25T00:44:45.145 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "400", 2024-04-25T00:44:45.145 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "599", 2024-04-25T00:44:45.145 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.145 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.145 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.145 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.145 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.146 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.146 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.146 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.146 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.146 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rbd_support", 2024-04-25T00:44:45.146 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.146 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.146 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.146 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.146 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.146 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.146 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.146 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.146 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.146 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.147 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.147 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.147 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.147 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.147 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.147 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.147 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.147 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.147 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.147 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.147 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.147 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.147 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.147 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.148 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.148 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.148 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.148 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.148 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.148 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.148 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.148 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.148 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.148 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.148 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.148 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.148 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.148 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.148 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.148 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.149 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.149 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.149 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.149 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.149 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.149 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.149 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.149 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.149 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.149 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.149 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.149 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.149 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.149 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.149 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.149 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.150 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.150 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.150 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.150 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.150 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.150 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.150 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.150 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.150 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.150 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.150 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.150 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.150 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.150 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.151 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.151 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.151 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.151 INFO:teuthology.orchestra.run.smithi032.stdout: "max_concurrent_snap_create": { 2024-04-25T00:44:45.151 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "max_concurrent_snap_create", 2024-04-25T00:44:45.151 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.151 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.151 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.151 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10", 2024-04-25T00:44:45.151 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.151 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.151 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.151 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.151 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.151 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.152 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.152 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.152 INFO:teuthology.orchestra.run.smithi032.stdout: "mirror_snapshot_schedule": { 2024-04-25T00:44:45.152 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "mirror_snapshot_schedule", 2024-04-25T00:44:45.152 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.152 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.152 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.152 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.152 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.152 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.152 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.152 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.152 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.152 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.153 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.153 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.153 INFO:teuthology.orchestra.run.smithi032.stdout: "trash_purge_schedule": { 2024-04-25T00:44:45.153 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "trash_purge_schedule", 2024-04-25T00:44:45.153 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.153 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.153 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.153 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.153 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.153 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.153 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.153 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.153 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.153 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.153 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.154 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.154 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.154 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.154 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.154 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "restful", 2024-04-25T00:44:45.154 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.154 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.154 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.154 INFO:teuthology.orchestra.run.smithi032.stdout: "enable_auth": { 2024-04-25T00:44:45.154 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "enable_auth", 2024-04-25T00:44:45.154 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.154 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.154 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.154 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.155 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.155 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.155 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.155 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.155 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.155 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.155 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.155 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.155 INFO:teuthology.orchestra.run.smithi032.stdout: "key_file": { 2024-04-25T00:44:45.155 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "key_file", 2024-04-25T00:44:45.155 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.155 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.155 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.155 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.155 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.155 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.156 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.156 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.156 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.156 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.156 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.156 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.156 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.156 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.156 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.156 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.156 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.156 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.156 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.156 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.157 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.157 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.157 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.157 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.157 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.157 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.157 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.157 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.157 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.157 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.157 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.157 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.157 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.157 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.157 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.158 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.158 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.158 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.158 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.158 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.158 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.158 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.158 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.158 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.158 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.158 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.158 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.158 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.158 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.159 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.159 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.159 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.159 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.159 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.159 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.159 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.159 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.159 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.159 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.159 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.159 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.159 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.159 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.159 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.160 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.160 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.160 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.160 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.160 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.160 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.160 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.160 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.160 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.160 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.160 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.160 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.160 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.160 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.160 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.161 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.161 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.161 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.161 INFO:teuthology.orchestra.run.smithi032.stdout: "server_addr": { 2024-04-25T00:44:45.161 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "server_addr", 2024-04-25T00:44:45.161 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.161 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.161 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.161 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.161 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.161 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.161 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.161 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.161 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.162 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.162 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.162 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.162 INFO:teuthology.orchestra.run.smithi032.stdout: "server_port": { 2024-04-25T00:44:45.162 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "server_port", 2024-04-25T00:44:45.162 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.162 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.162 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.162 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.162 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.162 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.162 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.162 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.162 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.163 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.163 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.163 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.163 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.163 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.163 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.163 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rgw", 2024-04-25T00:44:45.163 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.163 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.163 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.163 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.163 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.163 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.163 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.163 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.164 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.164 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.164 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.164 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.164 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.164 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.164 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.164 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.164 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.164 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.164 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.164 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.164 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.164 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.165 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.165 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.165 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.165 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.165 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.165 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.165 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.165 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.165 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.165 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.165 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.165 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.165 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.165 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.165 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.166 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.166 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.166 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.166 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.166 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.166 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.166 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.166 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.166 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.166 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.166 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.166 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.166 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.166 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.167 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.167 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.167 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.167 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.167 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.167 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.167 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.167 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.167 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.167 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.167 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.167 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.167 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.167 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.167 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.168 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.168 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.168 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.168 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.168 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.168 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.168 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.168 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.168 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.168 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.168 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rook", 2024-04-25T00:44:45.168 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.168 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.168 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.168 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.169 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.169 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.169 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.169 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.169 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.169 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.169 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.169 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.169 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.169 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.169 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.169 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.169 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.169 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.169 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.170 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.170 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.170 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.170 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.170 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.170 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.170 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.170 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.170 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.170 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.170 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.170 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.170 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.170 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.171 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.171 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.171 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.171 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.171 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.171 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.171 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.171 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.171 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.171 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.171 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.171 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.171 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.171 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.171 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.172 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.172 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.172 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.172 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.172 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.172 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.172 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.172 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.172 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.172 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.173 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.173 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.173 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.173 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.173 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.173 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.173 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.173 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.174 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.174 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.174 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.174 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.174 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.174 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.174 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.174 INFO:teuthology.orchestra.run.smithi032.stdout: "storage_class": { 2024-04-25T00:44:45.174 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "storage_class", 2024-04-25T00:44:45.174 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.174 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.174 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.174 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "local", 2024-04-25T00:44:45.174 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.175 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.175 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.175 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "storage class name for LSO-discovered PVs", 2024-04-25T00:44:45.175 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.175 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.175 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.175 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.175 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.175 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.175 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.175 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "selftest", 2024-04-25T00:44:45.175 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.175 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.175 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.175 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.176 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.176 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.176 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.176 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.176 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.176 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.176 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.176 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.176 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.176 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.176 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.176 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.176 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.176 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.177 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.177 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.177 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.177 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.177 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.177 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.177 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.177 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.177 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.177 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.177 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.177 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.177 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.177 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.177 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.178 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.178 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.178 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.178 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.178 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.178 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.178 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.178 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.178 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.178 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.178 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.178 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.178 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.178 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.178 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.179 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.179 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.179 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.179 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.179 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.179 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.179 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.179 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.179 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.179 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.179 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.179 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.179 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.179 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.179 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.180 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.180 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.180 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.180 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.180 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.180 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.180 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.180 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.180 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.180 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.180 INFO:teuthology.orchestra.run.smithi032.stdout: "roption1": { 2024-04-25T00:44:45.180 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "roption1", 2024-04-25T00:44:45.180 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.180 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.181 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.181 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.181 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.181 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.181 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.181 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.181 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.181 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.181 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.181 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.181 INFO:teuthology.orchestra.run.smithi032.stdout: "roption2": { 2024-04-25T00:44:45.181 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "roption2", 2024-04-25T00:44:45.181 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.181 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.181 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.182 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "xyz", 2024-04-25T00:44:45.182 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.182 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.182 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.182 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.182 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.182 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.182 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.182 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.182 INFO:teuthology.orchestra.run.smithi032.stdout: "rwoption1": { 2024-04-25T00:44:45.182 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rwoption1", 2024-04-25T00:44:45.182 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.182 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.182 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.182 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.183 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.183 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.183 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.183 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.183 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.183 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.183 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.183 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.183 INFO:teuthology.orchestra.run.smithi032.stdout: "rwoption2": { 2024-04-25T00:44:45.183 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rwoption2", 2024-04-25T00:44:45.183 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.183 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.183 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.183 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.184 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.184 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.184 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.184 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.184 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.184 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.184 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.184 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.184 INFO:teuthology.orchestra.run.smithi032.stdout: "rwoption3": { 2024-04-25T00:44:45.184 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rwoption3", 2024-04-25T00:44:45.184 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "float", 2024-04-25T00:44:45.184 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.184 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.184 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.185 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.185 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.185 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.185 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.185 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.185 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.185 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.185 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.185 INFO:teuthology.orchestra.run.smithi032.stdout: "rwoption4": { 2024-04-25T00:44:45.185 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rwoption4", 2024-04-25T00:44:45.185 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.185 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.185 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.185 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.185 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.186 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.186 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.186 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.186 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.186 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.186 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.186 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.186 INFO:teuthology.orchestra.run.smithi032.stdout: "rwoption5": { 2024-04-25T00:44:45.186 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rwoption5", 2024-04-25T00:44:45.186 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.186 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.186 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.186 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.186 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.186 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.187 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.187 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.187 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.187 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.187 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.187 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.187 INFO:teuthology.orchestra.run.smithi032.stdout: "rwoption6": { 2024-04-25T00:44:45.187 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rwoption6", 2024-04-25T00:44:45.187 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.187 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.187 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.187 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.187 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.187 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.187 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.188 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.188 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.188 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.188 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.188 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.188 INFO:teuthology.orchestra.run.smithi032.stdout: "rwoption7": { 2024-04-25T00:44:45.188 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rwoption7", 2024-04-25T00:44:45.188 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.188 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.188 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.188 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.188 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "1", 2024-04-25T00:44:45.188 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "42", 2024-04-25T00:44:45.188 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.189 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.189 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.189 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.189 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.189 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.189 INFO:teuthology.orchestra.run.smithi032.stdout: "testkey": { 2024-04-25T00:44:45.189 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "testkey", 2024-04-25T00:44:45.189 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.189 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.189 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.189 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.189 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.189 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.189 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.189 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.190 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.190 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.190 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.190 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.190 INFO:teuthology.orchestra.run.smithi032.stdout: "testlkey": { 2024-04-25T00:44:45.190 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "testlkey", 2024-04-25T00:44:45.190 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.190 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.190 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.190 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.190 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.190 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.190 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.190 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.191 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.191 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.191 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.191 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.191 INFO:teuthology.orchestra.run.smithi032.stdout: "testnewline": { 2024-04-25T00:44:45.191 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "testnewline", 2024-04-25T00:44:45.191 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.191 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.191 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.191 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.191 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.191 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.191 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.191 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.192 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.192 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.192 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.192 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.192 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.192 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.192 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.192 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "snap_schedule", 2024-04-25T00:44:45.192 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.192 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.192 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.192 INFO:teuthology.orchestra.run.smithi032.stdout: "allow_m_granularity": { 2024-04-25T00:44:45.192 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "allow_m_granularity", 2024-04-25T00:44:45.192 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.192 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.193 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.193 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.193 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.193 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.193 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.193 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "allow minute scheduled snapshots", 2024-04-25T00:44:45.193 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.193 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.193 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.193 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.193 INFO:teuthology.orchestra.run.smithi032.stdout: "dump_on_update": { 2024-04-25T00:44:45.193 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "dump_on_update", 2024-04-25T00:44:45.193 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.193 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.193 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.194 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.194 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.194 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.194 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.194 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "dump database to debug log on update", 2024-04-25T00:44:45.194 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.194 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.194 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.194 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.194 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.194 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.194 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.194 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.194 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.195 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.195 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.195 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.195 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.195 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.195 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.195 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.195 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.195 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.195 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.195 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.195 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.195 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.195 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.195 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.196 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.196 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.196 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.196 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.196 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.196 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.196 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.196 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.196 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.196 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.196 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.196 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.196 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.196 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.196 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.197 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.197 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.197 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.197 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.197 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.197 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.197 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.197 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.197 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.197 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.197 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.197 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.197 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.197 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.197 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.198 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.198 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.198 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.198 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.198 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.198 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.198 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.198 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.198 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.198 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.198 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.198 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.198 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.198 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.199 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.199 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.199 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.199 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.199 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.199 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.199 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.199 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.199 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.199 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "stats", 2024-04-25T00:44:45.199 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.199 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.199 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.199 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.200 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.200 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.200 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.200 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.200 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.200 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.200 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.200 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.200 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.200 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.200 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.200 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.200 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.200 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.200 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.201 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.201 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.201 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.201 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.201 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.201 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.201 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.201 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.201 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.201 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.201 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.201 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.201 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.201 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.202 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.202 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.202 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.202 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.202 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.202 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.202 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.202 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.202 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.202 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.202 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.202 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.202 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.202 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.203 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.203 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.203 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.203 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.203 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.203 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.203 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.203 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.203 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.203 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.203 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.203 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.203 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.204 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.204 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.204 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.204 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.204 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.204 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.204 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.204 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.204 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.204 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.204 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.204 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.204 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.204 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.205 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.205 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.205 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "status", 2024-04-25T00:44:45.205 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.205 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.205 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.205 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.205 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.205 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.205 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.205 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.205 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.205 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.205 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.206 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.206 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.206 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.206 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.206 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.206 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.206 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.206 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.206 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.206 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.206 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.206 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.206 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.206 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.206 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.207 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.207 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.207 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.207 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.207 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.207 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.207 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.207 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.207 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.207 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.207 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.207 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.207 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.208 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.208 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.208 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.208 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.208 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.208 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.208 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.208 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.208 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.208 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.208 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.208 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.208 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.208 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.208 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.209 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.209 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.209 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.209 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.209 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.209 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.209 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.209 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.209 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.209 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.209 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.209 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.209 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.209 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.210 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.210 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.210 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.210 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.210 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.210 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.210 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.210 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.210 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "telegraf", 2024-04-25T00:44:45.210 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.210 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.210 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.210 INFO:teuthology.orchestra.run.smithi032.stdout: "address": { 2024-04-25T00:44:45.210 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "address", 2024-04-25T00:44:45.211 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.211 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.211 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.211 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "unixgram:///tmp/telegraf.sock", 2024-04-25T00:44:45.211 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.211 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.211 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.211 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.211 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.211 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.211 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.211 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.211 INFO:teuthology.orchestra.run.smithi032.stdout: "interval": { 2024-04-25T00:44:45.211 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "interval", 2024-04-25T00:44:45.211 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.212 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.212 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.212 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "15", 2024-04-25T00:44:45.212 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.212 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.212 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.212 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.212 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.212 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.212 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.212 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.212 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.212 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.212 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.213 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.213 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.213 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.213 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.213 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.213 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.213 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.213 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.213 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.213 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.213 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.213 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.213 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.213 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.213 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.214 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.214 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.214 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.214 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.214 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.214 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.214 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.214 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.214 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.214 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.214 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.214 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.214 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.214 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.215 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.215 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.215 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.215 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.215 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.215 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.215 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.215 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.215 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.215 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.215 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.215 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.215 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.215 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.215 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.216 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.216 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.216 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.216 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.216 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.216 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.216 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.216 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.216 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.216 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.216 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.216 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.216 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.216 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.217 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.217 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.217 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.217 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.217 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.217 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.217 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.217 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.217 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.217 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.217 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.217 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.217 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "telemetry", 2024-04-25T00:44:45.217 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.217 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.218 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.218 INFO:teuthology.orchestra.run.smithi032.stdout: "channel_basic": { 2024-04-25T00:44:45.218 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "channel_basic", 2024-04-25T00:44:45.218 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.218 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.218 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.218 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.218 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.218 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.218 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.218 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Share basic cluster information (size, version)", 2024-04-25T00:44:45.218 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.218 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.218 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.219 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.219 INFO:teuthology.orchestra.run.smithi032.stdout: "channel_crash": { 2024-04-25T00:44:45.219 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "channel_crash", 2024-04-25T00:44:45.219 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.219 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.219 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.219 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.219 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.219 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.219 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.219 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Share metadata about Ceph daemon crashes (version, stack straces, etc)", 2024-04-25T00:44:45.219 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.219 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.219 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.219 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.220 INFO:teuthology.orchestra.run.smithi032.stdout: "channel_device": { 2024-04-25T00:44:45.220 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "channel_device", 2024-04-25T00:44:45.220 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.220 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.220 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.220 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.220 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.220 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.220 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.220 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)", 2024-04-25T00:44:45.220 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.220 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.220 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.220 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.221 INFO:teuthology.orchestra.run.smithi032.stdout: "channel_ident": { 2024-04-25T00:44:45.221 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "channel_ident", 2024-04-25T00:44:45.221 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.221 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.221 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.221 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.221 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.221 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.221 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.221 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Share a user-provided description and/or contact email for the cluster", 2024-04-25T00:44:45.221 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.221 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.221 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.221 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.221 INFO:teuthology.orchestra.run.smithi032.stdout: "channel_perf": { 2024-04-25T00:44:45.222 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "channel_perf", 2024-04-25T00:44:45.222 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.222 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.222 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.222 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.222 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.222 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.222 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.222 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Share various performance metrics of a cluster", 2024-04-25T00:44:45.222 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.222 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.222 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.222 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.222 INFO:teuthology.orchestra.run.smithi032.stdout: "contact": { 2024-04-25T00:44:45.222 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "contact", 2024-04-25T00:44:45.223 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.223 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.223 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.223 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.223 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.223 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.223 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.223 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.223 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.223 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.223 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.223 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.223 INFO:teuthology.orchestra.run.smithi032.stdout: "description": { 2024-04-25T00:44:45.223 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "description", 2024-04-25T00:44:45.224 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.224 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.224 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.224 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.224 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.224 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.224 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.224 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.224 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.224 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.224 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.224 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.224 INFO:teuthology.orchestra.run.smithi032.stdout: "device_url": { 2024-04-25T00:44:45.224 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "device_url", 2024-04-25T00:44:45.224 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.224 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.225 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.225 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "https://telemetry.ceph.com/device", 2024-04-25T00:44:45.225 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.225 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.225 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.225 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.225 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.225 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.225 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.225 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.225 INFO:teuthology.orchestra.run.smithi032.stdout: "enabled": { 2024-04-25T00:44:45.225 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "enabled", 2024-04-25T00:44:45.225 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.225 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.225 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.226 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.226 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.226 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.226 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.226 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.226 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.226 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.226 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.226 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.226 INFO:teuthology.orchestra.run.smithi032.stdout: "interval": { 2024-04-25T00:44:45.226 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "interval", 2024-04-25T00:44:45.226 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.226 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.226 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.226 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "24", 2024-04-25T00:44:45.227 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "8", 2024-04-25T00:44:45.227 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.227 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.227 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.227 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.227 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.227 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.227 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.227 INFO:teuthology.orchestra.run.smithi032.stdout: "last_opt_revision": { 2024-04-25T00:44:45.227 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "last_opt_revision", 2024-04-25T00:44:45.227 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.227 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.227 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.227 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "1", 2024-04-25T00:44:45.228 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.228 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.228 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.228 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.228 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.228 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.228 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.228 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.228 INFO:teuthology.orchestra.run.smithi032.stdout: "leaderboard": { 2024-04-25T00:44:45.228 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "leaderboard", 2024-04-25T00:44:45.228 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.228 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.228 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.228 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.228 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.229 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.229 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.229 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.229 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.229 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.229 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.229 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.229 INFO:teuthology.orchestra.run.smithi032.stdout: "leaderboard_description": { 2024-04-25T00:44:45.229 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "leaderboard_description", 2024-04-25T00:44:45.229 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.229 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.229 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.229 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.229 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.229 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.230 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.230 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.230 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.230 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.230 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.230 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.230 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.230 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.230 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.230 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.230 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.230 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.230 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.230 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.231 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.231 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.231 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.231 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.231 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.231 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.231 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.231 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.231 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.231 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.231 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.231 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.231 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.231 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.231 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.232 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.232 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.232 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.232 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.232 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.232 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.232 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.232 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.232 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.232 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.232 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.232 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.232 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.232 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.232 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.233 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.233 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.233 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.233 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.233 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.233 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.233 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.233 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.233 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.233 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.233 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.233 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.233 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.233 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.233 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.234 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.234 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.234 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.234 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.234 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.234 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.234 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.234 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.234 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.234 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.234 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.234 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.234 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.234 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.234 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.235 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.235 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.235 INFO:teuthology.orchestra.run.smithi032.stdout: "organization": { 2024-04-25T00:44:45.235 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "organization", 2024-04-25T00:44:45.235 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.235 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.235 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.235 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.235 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.235 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.235 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.235 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.235 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.235 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.235 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.236 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.236 INFO:teuthology.orchestra.run.smithi032.stdout: "proxy": { 2024-04-25T00:44:45.236 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "proxy", 2024-04-25T00:44:45.236 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.236 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.236 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.236 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.236 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.236 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.236 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.236 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.236 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.236 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.236 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.236 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.237 INFO:teuthology.orchestra.run.smithi032.stdout: "url": { 2024-04-25T00:44:45.237 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "url", 2024-04-25T00:44:45.237 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.237 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.237 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.237 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "https://telemetry.ceph.com/report", 2024-04-25T00:44:45.237 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.237 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.237 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.237 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.237 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.237 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.237 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.237 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.237 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.238 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.238 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.238 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "test_orchestrator", 2024-04-25T00:44:45.238 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.238 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.238 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.238 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.238 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.238 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.238 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.238 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.238 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.238 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.238 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.238 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.239 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.239 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.239 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.239 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.239 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.239 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.239 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.239 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.239 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.239 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.239 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.239 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.239 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.239 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.239 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.240 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.240 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.240 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.240 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.240 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.240 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.240 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.240 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.240 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.240 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.240 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.240 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.240 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.240 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.240 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.241 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.241 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.241 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.241 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.241 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.241 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.241 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.241 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.241 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.241 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.241 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.241 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.241 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.241 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.241 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.242 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.242 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.242 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.242 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.242 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.242 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.242 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.242 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.242 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.242 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.242 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.242 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.242 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.242 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.243 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.243 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.243 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.243 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.243 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.243 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "volumes", 2024-04-25T00:44:45.243 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.243 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.243 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.243 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.243 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.243 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.243 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.243 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.243 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.244 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.244 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.244 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.244 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.244 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.244 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.244 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.244 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.244 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.244 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.244 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.244 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.244 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.244 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.244 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.244 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.245 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.245 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.245 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.245 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.245 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.245 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.245 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.245 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.245 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.245 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.245 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.245 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.245 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.245 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.245 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.246 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.246 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.246 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.246 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.246 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.246 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.246 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.246 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.246 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.246 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.246 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.246 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.246 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.246 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.246 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.247 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.247 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.247 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.247 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.247 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.247 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.247 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.247 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.247 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.247 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.247 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.247 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.247 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.247 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.247 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.248 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.248 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.248 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.248 INFO:teuthology.orchestra.run.smithi032.stdout: "max_concurrent_clones": { 2024-04-25T00:44:45.248 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "max_concurrent_clones", 2024-04-25T00:44:45.248 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.248 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.248 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.248 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "4", 2024-04-25T00:44:45.248 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.248 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.248 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.248 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Number of asynchronous cloner threads", 2024-04-25T00:44:45.248 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.248 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.249 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.249 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.249 INFO:teuthology.orchestra.run.smithi032.stdout: "periodic_async_work": { 2024-04-25T00:44:45.249 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "periodic_async_work", 2024-04-25T00:44:45.249 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.249 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.249 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.249 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.249 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.249 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.249 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.249 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Periodically check for async work", 2024-04-25T00:44:45.249 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.249 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.249 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.250 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.250 INFO:teuthology.orchestra.run.smithi032.stdout: "snapshot_clone_delay": { 2024-04-25T00:44:45.250 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "snapshot_clone_delay", 2024-04-25T00:44:45.250 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.250 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.250 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.250 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "0", 2024-04-25T00:44:45.250 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.250 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.250 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.250 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Delay clone begin operation by snapshot_clone_delay seconds", 2024-04-25T00:44:45.250 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.250 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.250 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.251 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.251 INFO:teuthology.orchestra.run.smithi032.stdout: "snapshot_clone_no_wait": { 2024-04-25T00:44:45.251 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "snapshot_clone_no_wait", 2024-04-25T00:44:45.251 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.251 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.251 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.251 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.251 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.251 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.251 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.251 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Reject subvolume clone request when cloner threads are busy", 2024-04-25T00:44:45.251 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.251 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.251 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.251 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.252 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.252 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.252 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.252 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "zabbix", 2024-04-25T00:44:45.252 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.252 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.252 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.252 INFO:teuthology.orchestra.run.smithi032.stdout: "discovery_interval": { 2024-04-25T00:44:45.252 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "discovery_interval", 2024-04-25T00:44:45.252 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "uint", 2024-04-25T00:44:45.252 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.252 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.252 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "100", 2024-04-25T00:44:45.252 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.252 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.253 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.253 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.253 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.253 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.253 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.253 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.253 INFO:teuthology.orchestra.run.smithi032.stdout: "identifier": { 2024-04-25T00:44:45.253 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "identifier", 2024-04-25T00:44:45.253 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.253 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.253 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.253 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.253 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.253 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.253 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.253 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.254 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.254 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.254 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.254 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.254 INFO:teuthology.orchestra.run.smithi032.stdout: "interval": { 2024-04-25T00:44:45.254 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "interval", 2024-04-25T00:44:45.254 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.254 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.254 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.254 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "60", 2024-04-25T00:44:45.254 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.254 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.254 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.254 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.255 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.255 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.255 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.255 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.255 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.255 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.255 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.255 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.255 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.255 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.255 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.255 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.255 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.255 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.256 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.256 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.256 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.256 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.256 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.256 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.256 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.256 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.256 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.256 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.256 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.256 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.256 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.256 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.256 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.257 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.257 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.257 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.257 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.257 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.257 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.257 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.257 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.257 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.257 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.257 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.257 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.257 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.257 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.257 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.258 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.258 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.258 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.258 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.258 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.258 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.258 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.258 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.258 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.258 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.258 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.258 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.258 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.258 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.258 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.259 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.259 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.259 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.259 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.259 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.259 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.259 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.259 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.259 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.259 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.259 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.259 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.259 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.259 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.259 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.260 INFO:teuthology.orchestra.run.smithi032.stdout: "zabbix_host": { 2024-04-25T00:44:45.260 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "zabbix_host", 2024-04-25T00:44:45.260 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.260 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.260 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.260 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.260 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.260 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.260 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.260 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.260 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.260 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.260 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.260 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.261 INFO:teuthology.orchestra.run.smithi032.stdout: "zabbix_port": { 2024-04-25T00:44:45.261 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "zabbix_port", 2024-04-25T00:44:45.261 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.261 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.261 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.261 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10051", 2024-04-25T00:44:45.261 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.261 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.261 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.261 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.261 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.261 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.261 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.261 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.261 INFO:teuthology.orchestra.run.smithi032.stdout: "zabbix_sender": { 2024-04-25T00:44:45.261 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "zabbix_sender", 2024-04-25T00:44:45.262 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.262 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.262 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.262 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "/usr/bin/zabbix_sender", 2024-04-25T00:44:45.262 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.262 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.262 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.262 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.262 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.262 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.262 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.262 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.262 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.262 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.263 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-04-25T00:44:45.263 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.263 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.263 INFO:teuthology.orchestra.run.smithi032.stdout: "modules": [ 2024-04-25T00:44:45.263 INFO:teuthology.orchestra.run.smithi032.stdout: "cephadm", 2024-04-25T00:44:45.263 INFO:teuthology.orchestra.run.smithi032.stdout: "dashboard", 2024-04-25T00:44:45.263 INFO:teuthology.orchestra.run.smithi032.stdout: "iostat", 2024-04-25T00:44:45.263 INFO:teuthology.orchestra.run.smithi032.stdout: "nfs", 2024-04-25T00:44:45.263 INFO:teuthology.orchestra.run.smithi032.stdout: "restful", 2024-04-25T00:44:45.263 INFO:teuthology.orchestra.run.smithi032.stdout: "snap_schedule" 2024-04-25T00:44:45.263 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.263 INFO:teuthology.orchestra.run.smithi032.stdout: "available_modules": [ 2024-04-25T00:44:45.263 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.263 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "alerts", 2024-04-25T00:44:45.263 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.264 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.264 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.264 INFO:teuthology.orchestra.run.smithi032.stdout: "interval": { 2024-04-25T00:44:45.264 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "interval", 2024-04-25T00:44:45.264 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.264 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.264 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.264 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "60", 2024-04-25T00:44:45.264 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.264 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.264 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.264 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "How frequently to reexamine health status", 2024-04-25T00:44:45.264 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.264 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.265 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.265 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.265 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.265 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.265 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.265 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.265 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.265 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.265 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.265 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.265 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.265 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.265 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.265 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.265 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.266 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.266 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.266 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.266 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.266 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.266 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.266 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.266 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.266 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.266 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.266 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.266 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.266 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.266 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.267 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.267 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.267 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.267 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.267 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.267 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.267 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.267 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.267 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.267 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.267 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.267 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.267 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.267 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.267 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.268 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.268 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.268 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.268 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.268 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.268 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.268 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.268 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.268 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.268 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.268 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.268 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.268 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.268 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.268 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.269 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.269 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.269 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.269 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.269 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.269 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.269 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.269 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.269 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.269 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.269 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.269 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.269 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.269 INFO:teuthology.orchestra.run.smithi032.stdout: "smtp_destination": { 2024-04-25T00:44:45.270 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "smtp_destination", 2024-04-25T00:44:45.270 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.270 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.270 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.270 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.270 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.270 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.270 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.270 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Email address to send alerts to", 2024-04-25T00:44:45.270 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.270 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.270 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.270 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.270 INFO:teuthology.orchestra.run.smithi032.stdout: "smtp_from_name": { 2024-04-25T00:44:45.271 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "smtp_from_name", 2024-04-25T00:44:45.271 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.271 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.271 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.271 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "Ceph", 2024-04-25T00:44:45.271 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.271 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.271 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.271 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Email From: name", 2024-04-25T00:44:45.271 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.271 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.271 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.271 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.271 INFO:teuthology.orchestra.run.smithi032.stdout: "smtp_host": { 2024-04-25T00:44:45.271 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "smtp_host", 2024-04-25T00:44:45.272 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.272 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.272 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.272 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.272 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.272 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.272 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.272 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "SMTP server", 2024-04-25T00:44:45.272 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.272 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.272 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.272 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.272 INFO:teuthology.orchestra.run.smithi032.stdout: "smtp_password": { 2024-04-25T00:44:45.272 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "smtp_password", 2024-04-25T00:44:45.272 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.273 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.273 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.273 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.273 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.273 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.273 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.273 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Password to authenticate with", 2024-04-25T00:44:45.273 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.273 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.273 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.273 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.273 INFO:teuthology.orchestra.run.smithi032.stdout: "smtp_port": { 2024-04-25T00:44:45.273 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "smtp_port", 2024-04-25T00:44:45.273 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.273 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.274 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.274 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "465", 2024-04-25T00:44:45.274 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.274 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.274 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.274 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "SMTP port", 2024-04-25T00:44:45.274 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.274 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.274 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.274 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.274 INFO:teuthology.orchestra.run.smithi032.stdout: "smtp_sender": { 2024-04-25T00:44:45.274 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "smtp_sender", 2024-04-25T00:44:45.274 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.274 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.274 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.275 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.275 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.275 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.275 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.275 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "SMTP envelope sender", 2024-04-25T00:44:45.275 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.275 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.275 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.275 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.275 INFO:teuthology.orchestra.run.smithi032.stdout: "smtp_ssl": { 2024-04-25T00:44:45.275 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "smtp_ssl", 2024-04-25T00:44:45.275 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.275 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.275 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.275 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.276 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.276 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.276 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.276 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Use SSL to connect to SMTP server", 2024-04-25T00:44:45.276 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.276 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.276 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.276 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.276 INFO:teuthology.orchestra.run.smithi032.stdout: "smtp_user": { 2024-04-25T00:44:45.276 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "smtp_user", 2024-04-25T00:44:45.276 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.276 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.276 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.276 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.276 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.277 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.277 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.277 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "User to authenticate as", 2024-04-25T00:44:45.277 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.277 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.277 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.277 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.277 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.277 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.277 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.277 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "balancer", 2024-04-25T00:44:45.277 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.277 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.277 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.277 INFO:teuthology.orchestra.run.smithi032.stdout: "active": { 2024-04-25T00:44:45.278 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "active", 2024-04-25T00:44:45.278 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.278 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.278 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.278 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.278 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.278 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.278 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.278 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "automatically balance PGs across cluster", 2024-04-25T00:44:45.278 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.278 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.278 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.278 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.278 INFO:teuthology.orchestra.run.smithi032.stdout: "begin_time": { 2024-04-25T00:44:45.278 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "begin_time", 2024-04-25T00:44:45.279 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.279 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.279 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.279 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "0000", 2024-04-25T00:44:45.279 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.279 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.279 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.279 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "beginning time of day to automatically balance", 2024-04-25T00:44:45.279 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "This is a time of day in the format HHMM.", 2024-04-25T00:44:45.279 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.279 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.279 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.279 INFO:teuthology.orchestra.run.smithi032.stdout: "begin_weekday": { 2024-04-25T00:44:45.279 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "begin_weekday", 2024-04-25T00:44:45.279 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "uint", 2024-04-25T00:44:45.280 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.280 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.280 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "0", 2024-04-25T00:44:45.280 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "0", 2024-04-25T00:44:45.280 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "6", 2024-04-25T00:44:45.280 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.280 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Restrict automatic balancing to this day of the week or later", 2024-04-25T00:44:45.280 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2024-04-25T00:44:45.280 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.280 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.280 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.280 INFO:teuthology.orchestra.run.smithi032.stdout: "crush_compat_max_iterations": { 2024-04-25T00:44:45.280 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "crush_compat_max_iterations", 2024-04-25T00:44:45.280 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "uint", 2024-04-25T00:44:45.280 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.281 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.281 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "25", 2024-04-25T00:44:45.281 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "1", 2024-04-25T00:44:45.281 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "250", 2024-04-25T00:44:45.281 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.281 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "maximum number of iterations to attempt optimization", 2024-04-25T00:44:45.281 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.281 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.281 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.281 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.281 INFO:teuthology.orchestra.run.smithi032.stdout: "crush_compat_metrics": { 2024-04-25T00:44:45.281 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "crush_compat_metrics", 2024-04-25T00:44:45.281 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.281 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.281 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.282 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "pgs,objects,bytes", 2024-04-25T00:44:45.282 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.282 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.282 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.282 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "metrics with which to calculate OSD utilization", 2024-04-25T00:44:45.282 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.", 2024-04-25T00:44:45.282 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.282 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.282 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.282 INFO:teuthology.orchestra.run.smithi032.stdout: "crush_compat_step": { 2024-04-25T00:44:45.282 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "crush_compat_step", 2024-04-25T00:44:45.282 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "float", 2024-04-25T00:44:45.282 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.282 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.283 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "0.5", 2024-04-25T00:44:45.283 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "0.001", 2024-04-25T00:44:45.283 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "0.999", 2024-04-25T00:44:45.283 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.283 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "aggressiveness of optimization", 2024-04-25T00:44:45.283 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": ".99 is very aggressive, .01 is less aggressive", 2024-04-25T00:44:45.283 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.283 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.283 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.283 INFO:teuthology.orchestra.run.smithi032.stdout: "end_time": { 2024-04-25T00:44:45.283 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "end_time", 2024-04-25T00:44:45.283 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.283 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.283 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.283 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "2359", 2024-04-25T00:44:45.284 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.284 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.284 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.284 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "ending time of day to automatically balance", 2024-04-25T00:44:45.284 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "This is a time of day in the format HHMM.", 2024-04-25T00:44:45.284 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.284 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.284 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.284 INFO:teuthology.orchestra.run.smithi032.stdout: "end_weekday": { 2024-04-25T00:44:45.284 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "end_weekday", 2024-04-25T00:44:45.284 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "uint", 2024-04-25T00:44:45.284 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.284 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.284 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "0", 2024-04-25T00:44:45.284 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "0", 2024-04-25T00:44:45.285 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "6", 2024-04-25T00:44:45.285 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.285 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Restrict automatic balancing to days of the week earlier than this", 2024-04-25T00:44:45.285 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2024-04-25T00:44:45.285 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.285 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.285 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.285 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.285 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.285 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.285 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.285 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.285 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.285 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.285 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.285 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.286 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.286 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.286 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.286 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.286 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.286 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.286 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.286 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.286 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.286 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.286 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.286 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.286 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.286 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.287 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.287 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.287 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.287 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.287 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.287 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.287 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.287 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.287 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.287 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.287 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.287 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.287 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.287 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.288 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.288 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.288 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.288 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.288 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.288 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.288 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.288 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.288 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.288 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.288 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.288 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.288 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.288 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.288 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.288 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.289 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.289 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.289 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.289 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.289 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.289 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.289 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.289 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.289 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.289 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.289 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.289 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.289 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.289 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.289 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.290 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.290 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.290 INFO:teuthology.orchestra.run.smithi032.stdout: "min_score": { 2024-04-25T00:44:45.290 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "min_score", 2024-04-25T00:44:45.290 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "float", 2024-04-25T00:44:45.290 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.290 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.290 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "0", 2024-04-25T00:44:45.290 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.290 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.290 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.290 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "minimum score, below which no optimization is attempted", 2024-04-25T00:44:45.290 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.290 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.290 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.291 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.291 INFO:teuthology.orchestra.run.smithi032.stdout: "mode": { 2024-04-25T00:44:45.291 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "mode", 2024-04-25T00:44:45.291 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.291 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.291 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.291 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "upmap", 2024-04-25T00:44:45.291 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.291 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.291 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.291 INFO:teuthology.orchestra.run.smithi032.stdout: "crush-compat", 2024-04-25T00:44:45.291 INFO:teuthology.orchestra.run.smithi032.stdout: "none", 2024-04-25T00:44:45.291 INFO:teuthology.orchestra.run.smithi032.stdout: "read", 2024-04-25T00:44:45.291 INFO:teuthology.orchestra.run.smithi032.stdout: "upmap", 2024-04-25T00:44:45.291 INFO:teuthology.orchestra.run.smithi032.stdout: "upmap-read" 2024-04-25T00:44:45.292 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.292 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Balancer mode", 2024-04-25T00:44:45.292 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.292 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.292 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.292 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.292 INFO:teuthology.orchestra.run.smithi032.stdout: "pool_ids": { 2024-04-25T00:44:45.292 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "pool_ids", 2024-04-25T00:44:45.292 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.292 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.292 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.292 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.292 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.292 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.292 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.292 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "pools which the automatic balancing will be limited to", 2024-04-25T00:44:45.293 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.293 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.293 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.293 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.293 INFO:teuthology.orchestra.run.smithi032.stdout: "sleep_interval": { 2024-04-25T00:44:45.293 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "sleep_interval", 2024-04-25T00:44:45.293 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.293 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.293 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.293 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "60", 2024-04-25T00:44:45.293 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.293 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.293 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.293 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "how frequently to wake up and attempt optimization", 2024-04-25T00:44:45.293 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.294 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.294 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.294 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.294 INFO:teuthology.orchestra.run.smithi032.stdout: "upmap_max_deviation": { 2024-04-25T00:44:45.294 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "upmap_max_deviation", 2024-04-25T00:44:45.294 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.294 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.294 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.294 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "5", 2024-04-25T00:44:45.294 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "1", 2024-04-25T00:44:45.294 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.294 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.294 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "deviation below which no optimization is attempted", 2024-04-25T00:44:45.294 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "If the number of PGs are within this count then no optimization is attempted", 2024-04-25T00:44:45.295 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.295 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.295 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.295 INFO:teuthology.orchestra.run.smithi032.stdout: "upmap_max_optimizations": { 2024-04-25T00:44:45.295 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "upmap_max_optimizations", 2024-04-25T00:44:45.295 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "uint", 2024-04-25T00:44:45.295 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.295 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.295 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10", 2024-04-25T00:44:45.295 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.295 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.295 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.295 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "maximum upmap optimizations to make per attempt", 2024-04-25T00:44:45.295 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.295 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.296 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.296 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.296 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.296 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.296 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.296 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "cephadm", 2024-04-25T00:44:45.296 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.296 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.296 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.296 INFO:teuthology.orchestra.run.smithi032.stdout: "agent_down_multiplier": { 2024-04-25T00:44:45.296 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "agent_down_multiplier", 2024-04-25T00:44:45.296 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "float", 2024-04-25T00:44:45.296 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.296 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.296 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "3.0", 2024-04-25T00:44:45.297 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.297 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.297 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.297 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Multiplied by agent refresh rate to calculate how long agent must not report before being marked down", 2024-04-25T00:44:45.297 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.297 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.297 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.297 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.297 INFO:teuthology.orchestra.run.smithi032.stdout: "agent_refresh_rate": { 2024-04-25T00:44:45.297 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "agent_refresh_rate", 2024-04-25T00:44:45.297 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.297 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.297 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.297 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "20", 2024-04-25T00:44:45.297 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.298 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.298 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.298 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "How often agent on each host will try to gather and send metadata", 2024-04-25T00:44:45.298 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.298 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.298 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.298 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.298 INFO:teuthology.orchestra.run.smithi032.stdout: "agent_starting_port": { 2024-04-25T00:44:45.298 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "agent_starting_port", 2024-04-25T00:44:45.298 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.298 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.298 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.298 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "4721", 2024-04-25T00:44:45.298 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.299 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.299 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.299 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)", 2024-04-25T00:44:45.299 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.299 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.299 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.299 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.299 INFO:teuthology.orchestra.run.smithi032.stdout: "allow_ptrace": { 2024-04-25T00:44:45.299 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "allow_ptrace", 2024-04-25T00:44:45.299 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.299 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.299 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.299 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.299 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.299 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.299 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.300 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "allow SYS_PTRACE capability on ceph containers", 2024-04-25T00:44:45.300 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.", 2024-04-25T00:44:45.300 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.300 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.300 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.300 INFO:teuthology.orchestra.run.smithi032.stdout: "autotune_interval": { 2024-04-25T00:44:45.300 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "autotune_interval", 2024-04-25T00:44:45.300 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.300 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.300 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.300 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "600", 2024-04-25T00:44:45.300 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.300 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.300 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.300 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "how frequently to autotune daemon memory", 2024-04-25T00:44:45.301 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.301 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.301 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.301 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.301 INFO:teuthology.orchestra.run.smithi032.stdout: "autotune_memory_target_ratio": { 2024-04-25T00:44:45.301 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "autotune_memory_target_ratio", 2024-04-25T00:44:45.301 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "float", 2024-04-25T00:44:45.301 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.301 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.301 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "0.7", 2024-04-25T00:44:45.301 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.301 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.301 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.301 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "ratio of total system memory to divide amongst autotuned daemons", 2024-04-25T00:44:45.301 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.302 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.302 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.302 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.302 INFO:teuthology.orchestra.run.smithi032.stdout: "cephadm_log_destination": { 2024-04-25T00:44:45.302 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "cephadm_log_destination", 2024-04-25T00:44:45.302 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.302 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.302 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.302 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.302 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.302 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.302 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.302 INFO:teuthology.orchestra.run.smithi032.stdout: "file", 2024-04-25T00:44:45.302 INFO:teuthology.orchestra.run.smithi032.stdout: "file,syslog", 2024-04-25T00:44:45.303 INFO:teuthology.orchestra.run.smithi032.stdout: "syslog" 2024-04-25T00:44:45.303 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.303 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Destination for cephadm command's persistent logging", 2024-04-25T00:44:45.303 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.303 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.303 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.303 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.303 INFO:teuthology.orchestra.run.smithi032.stdout: "cgroups_split": { 2024-04-25T00:44:45.303 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "cgroups_split", 2024-04-25T00:44:45.303 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.303 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.303 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.303 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.303 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.303 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.303 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.304 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Pass --cgroups=split when cephadm creates containers (currently podman only)", 2024-04-25T00:44:45.304 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.304 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.304 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.304 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.304 INFO:teuthology.orchestra.run.smithi032.stdout: "config_checks_enabled": { 2024-04-25T00:44:45.304 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "config_checks_enabled", 2024-04-25T00:44:45.304 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.304 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.304 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.304 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.304 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.304 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.304 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.304 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Enable or disable the cephadm configuration analysis", 2024-04-25T00:44:45.305 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.305 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.305 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.305 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.305 INFO:teuthology.orchestra.run.smithi032.stdout: "config_dashboard": { 2024-04-25T00:44:45.305 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "config_dashboard", 2024-04-25T00:44:45.305 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.305 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.305 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.305 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.305 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.305 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.305 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.305 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "manage configs like API endpoints in Dashboard.", 2024-04-25T00:44:45.305 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.306 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.306 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.306 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.306 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_alertmanager": { 2024-04-25T00:44:45.306 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_alertmanager", 2024-04-25T00:44:45.306 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.306 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.306 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.306 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/prometheus/alertmanager:v0.25.0", 2024-04-25T00:44:45.306 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.306 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.306 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.306 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Prometheus container image", 2024-04-25T00:44:45.306 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.307 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.307 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.307 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.307 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_base": { 2024-04-25T00:44:45.307 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_base", 2024-04-25T00:44:45.307 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.307 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.307 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.307 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/ceph/ceph", 2024-04-25T00:44:45.307 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.307 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.307 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.307 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Container image name, without the tag", 2024-04-25T00:44:45.307 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.308 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.308 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.308 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.308 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_elasticsearch": { 2024-04-25T00:44:45.308 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_elasticsearch", 2024-04-25T00:44:45.308 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.308 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.308 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.308 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/omrizeneva/elasticsearch:6.8.23", 2024-04-25T00:44:45.308 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.308 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.308 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.308 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "elasticsearch container image", 2024-04-25T00:44:45.308 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.308 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.309 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.309 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.309 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_grafana": { 2024-04-25T00:44:45.309 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_grafana", 2024-04-25T00:44:45.309 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.309 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.309 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.309 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/ceph/grafana:9.4.12", 2024-04-25T00:44:45.309 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.309 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.309 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.309 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Prometheus container image", 2024-04-25T00:44:45.309 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.309 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.309 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.310 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.310 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_haproxy": { 2024-04-25T00:44:45.310 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_haproxy", 2024-04-25T00:44:45.310 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.310 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.310 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.310 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/ceph/haproxy:2.3", 2024-04-25T00:44:45.310 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.310 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.310 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.310 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "HAproxy container image", 2024-04-25T00:44:45.310 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.310 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.310 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.311 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.311 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_jaeger_agent": { 2024-04-25T00:44:45.311 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_jaeger_agent", 2024-04-25T00:44:45.311 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.311 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.311 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.311 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/jaegertracing/jaeger-agent:1.29", 2024-04-25T00:44:45.311 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.311 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.311 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.311 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Jaeger agent container image", 2024-04-25T00:44:45.311 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.311 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.311 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.311 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.312 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_jaeger_collector": { 2024-04-25T00:44:45.312 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_jaeger_collector", 2024-04-25T00:44:45.312 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.312 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.312 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.312 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/jaegertracing/jaeger-collector:1.29", 2024-04-25T00:44:45.312 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.312 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.312 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.312 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Jaeger collector container image", 2024-04-25T00:44:45.312 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.312 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.312 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.312 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.312 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_jaeger_query": { 2024-04-25T00:44:45.313 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_jaeger_query", 2024-04-25T00:44:45.313 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.313 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.313 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.313 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/jaegertracing/jaeger-query:1.29", 2024-04-25T00:44:45.313 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.313 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.313 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.313 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Jaeger query container image", 2024-04-25T00:44:45.313 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.313 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.313 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.313 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.313 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_keepalived": { 2024-04-25T00:44:45.313 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_keepalived", 2024-04-25T00:44:45.314 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.314 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.314 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.314 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/ceph/keepalived:2.2.4", 2024-04-25T00:44:45.314 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.314 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.314 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.314 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Keepalived container image", 2024-04-25T00:44:45.314 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.314 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.314 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.314 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.314 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_loki": { 2024-04-25T00:44:45.314 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_loki", 2024-04-25T00:44:45.315 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.315 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.315 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.315 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "docker.io/grafana/loki:2.4.0", 2024-04-25T00:44:45.315 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.315 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.315 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.315 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Loki container image", 2024-04-25T00:44:45.315 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.315 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.315 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.315 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.315 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_node_exporter": { 2024-04-25T00:44:45.315 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_node_exporter", 2024-04-25T00:44:45.315 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.315 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.316 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.316 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/prometheus/node-exporter:v1.5.0", 2024-04-25T00:44:45.316 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.316 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.316 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.316 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Prometheus container image", 2024-04-25T00:44:45.316 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.316 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.316 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.316 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.316 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_nvmeof": { 2024-04-25T00:44:45.316 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_nvmeof", 2024-04-25T00:44:45.316 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.316 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.316 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.317 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/ceph/nvmeof:1.2.1", 2024-04-25T00:44:45.317 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.317 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.317 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.317 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Nvme-of container image", 2024-04-25T00:44:45.317 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.317 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.317 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.317 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.317 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_prometheus": { 2024-04-25T00:44:45.317 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_prometheus", 2024-04-25T00:44:45.317 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.317 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.317 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.317 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/prometheus/prometheus:v2.43.0", 2024-04-25T00:44:45.318 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.318 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.318 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.318 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Prometheus container image", 2024-04-25T00:44:45.318 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.318 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.318 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.318 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.318 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_promtail": { 2024-04-25T00:44:45.318 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_promtail", 2024-04-25T00:44:45.318 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.318 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.318 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.318 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "docker.io/grafana/promtail:2.4.0", 2024-04-25T00:44:45.318 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.319 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.319 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.319 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Promtail container image", 2024-04-25T00:44:45.319 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.319 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.319 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.319 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.319 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_samba": { 2024-04-25T00:44:45.319 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_samba", 2024-04-25T00:44:45.319 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.319 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.319 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.319 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "quay.io/samba.org/samba-server:devbuilds-centos-amd64", 2024-04-25T00:44:45.319 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.319 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.320 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.320 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Samba/SMB container image", 2024-04-25T00:44:45.320 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.320 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.320 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.320 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.320 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image_snmp_gateway": { 2024-04-25T00:44:45.320 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_image_snmp_gateway", 2024-04-25T00:44:45.320 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.320 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.320 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.320 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "docker.io/maxwo/snmp-notifier:v1.2.1", 2024-04-25T00:44:45.320 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.320 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.320 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.321 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "SNMP Gateway container image", 2024-04-25T00:44:45.321 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.321 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.321 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.321 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.321 INFO:teuthology.orchestra.run.smithi032.stdout: "container_init": { 2024-04-25T00:44:45.321 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "container_init", 2024-04-25T00:44:45.321 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.321 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.321 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.321 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.321 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.321 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.321 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.322 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Run podman/docker with `--init`", 2024-04-25T00:44:45.322 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.322 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.322 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.322 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.322 INFO:teuthology.orchestra.run.smithi032.stdout: "daemon_cache_timeout": { 2024-04-25T00:44:45.322 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "daemon_cache_timeout", 2024-04-25T00:44:45.322 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.322 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.322 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.322 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "600", 2024-04-25T00:44:45.322 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.322 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.322 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.322 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "seconds to cache service (daemon) inventory", 2024-04-25T00:44:45.323 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.323 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.323 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.323 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.323 INFO:teuthology.orchestra.run.smithi032.stdout: "default_cephadm_command_timeout": { 2024-04-25T00:44:45.323 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "default_cephadm_command_timeout", 2024-04-25T00:44:45.323 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.323 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.323 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.323 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "900", 2024-04-25T00:44:45.323 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.323 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.323 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.323 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Default timeout applied to cephadm commands run directly on the host (in seconds)", 2024-04-25T00:44:45.324 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.324 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.324 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.324 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.324 INFO:teuthology.orchestra.run.smithi032.stdout: "default_registry": { 2024-04-25T00:44:45.324 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "default_registry", 2024-04-25T00:44:45.324 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.324 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.324 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.324 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "docker.io", 2024-04-25T00:44:45.324 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.324 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.324 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.324 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Search-registry to which we should normalize unqualified image names. This is not the default registry", 2024-04-25T00:44:45.324 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.325 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.325 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.325 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.325 INFO:teuthology.orchestra.run.smithi032.stdout: "device_cache_timeout": { 2024-04-25T00:44:45.325 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "device_cache_timeout", 2024-04-25T00:44:45.325 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.325 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.325 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.325 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "1800", 2024-04-25T00:44:45.325 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.325 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.325 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.325 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "seconds to cache device inventory", 2024-04-25T00:44:45.325 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.325 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.326 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.326 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.326 INFO:teuthology.orchestra.run.smithi032.stdout: "device_enhanced_scan": { 2024-04-25T00:44:45.326 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "device_enhanced_scan", 2024-04-25T00:44:45.326 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.326 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.326 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.326 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.326 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.326 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.326 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.326 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Use libstoragemgmt during device scans", 2024-04-25T00:44:45.326 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.326 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.326 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.327 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.327 INFO:teuthology.orchestra.run.smithi032.stdout: "facts_cache_timeout": { 2024-04-25T00:44:45.327 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "facts_cache_timeout", 2024-04-25T00:44:45.327 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.327 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.327 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.327 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "60", 2024-04-25T00:44:45.327 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.327 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.327 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.327 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "seconds to cache host facts data", 2024-04-25T00:44:45.327 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.327 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.327 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.327 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.328 INFO:teuthology.orchestra.run.smithi032.stdout: "grafana_dashboards_path": { 2024-04-25T00:44:45.328 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "grafana_dashboards_path", 2024-04-25T00:44:45.328 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.328 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.328 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.328 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "/etc/grafana/dashboards/ceph-dashboard/", 2024-04-25T00:44:45.328 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.328 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.328 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.328 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "location of dashboards to include in grafana deployments", 2024-04-25T00:44:45.328 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.328 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.328 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.328 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.328 INFO:teuthology.orchestra.run.smithi032.stdout: "host_check_interval": { 2024-04-25T00:44:45.329 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "host_check_interval", 2024-04-25T00:44:45.329 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.329 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.329 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.329 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "600", 2024-04-25T00:44:45.329 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.329 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.329 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.329 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "how frequently to perform a host check", 2024-04-25T00:44:45.329 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.329 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.329 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.329 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.329 INFO:teuthology.orchestra.run.smithi032.stdout: "hw_monitoring": { 2024-04-25T00:44:45.329 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "hw_monitoring", 2024-04-25T00:44:45.330 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.330 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.330 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.330 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.330 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.330 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.330 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.330 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Deploy hw monitoring daemon on every host.", 2024-04-25T00:44:45.330 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.330 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.330 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.330 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.330 INFO:teuthology.orchestra.run.smithi032.stdout: "inventory_list_all": { 2024-04-25T00:44:45.330 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "inventory_list_all", 2024-04-25T00:44:45.330 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.331 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.331 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.331 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.331 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.331 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.331 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.331 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)", 2024-04-25T00:44:45.331 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.331 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.331 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.331 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.331 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.331 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.331 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.331 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.332 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.332 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.332 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.332 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.332 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.332 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.332 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.332 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.332 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.332 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.332 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.332 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.332 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.332 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.332 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.332 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.333 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.333 INFO:teuthology.orchestra.run.smithi032.stdout: "log_refresh_metadata": { 2024-04-25T00:44:45.333 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_refresh_metadata", 2024-04-25T00:44:45.333 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.333 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.333 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.333 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.333 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.333 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.333 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.333 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level", 2024-04-25T00:44:45.333 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.333 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.333 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.333 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.334 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.334 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.334 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.334 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.334 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.334 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.334 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.334 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.334 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.334 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "log to the \"cephadm\" cluster log channel\"", 2024-04-25T00:44:45.334 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.334 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.334 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.334 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.334 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.335 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.335 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.335 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.335 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.335 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.335 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.335 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.335 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.335 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.335 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.335 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.335 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.335 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.335 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.336 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.336 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.336 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.336 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.336 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.336 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.336 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.336 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.336 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.336 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.336 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.336 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.336 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.336 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.337 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.337 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.337 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.337 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.337 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.337 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.337 INFO:teuthology.orchestra.run.smithi032.stdout: "manage_etc_ceph_ceph_conf": { 2024-04-25T00:44:45.337 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "manage_etc_ceph_ceph_conf", 2024-04-25T00:44:45.337 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.337 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.337 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.337 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.337 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.337 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.337 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.338 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Manage and own /etc/ceph/ceph.conf on the hosts.", 2024-04-25T00:44:45.338 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.338 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.338 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.338 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.338 INFO:teuthology.orchestra.run.smithi032.stdout: "manage_etc_ceph_ceph_conf_hosts": { 2024-04-25T00:44:45.338 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "manage_etc_ceph_ceph_conf_hosts", 2024-04-25T00:44:45.338 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.338 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.338 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.338 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "*", 2024-04-25T00:44:45.338 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.338 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.338 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.338 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf", 2024-04-25T00:44:45.339 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.339 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.339 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.339 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.339 INFO:teuthology.orchestra.run.smithi032.stdout: "max_count_per_host": { 2024-04-25T00:44:45.339 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "max_count_per_host", 2024-04-25T00:44:45.339 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.339 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.339 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.339 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10", 2024-04-25T00:44:45.339 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.339 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.339 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.339 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "max number of daemons per service per host", 2024-04-25T00:44:45.340 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.340 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.340 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.340 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.340 INFO:teuthology.orchestra.run.smithi032.stdout: "max_osd_draining_count": { 2024-04-25T00:44:45.340 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "max_osd_draining_count", 2024-04-25T00:44:45.340 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.340 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.340 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.340 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10", 2024-04-25T00:44:45.340 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.340 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.340 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.340 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "max number of osds that will be drained simultaneously when osds are removed", 2024-04-25T00:44:45.340 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.341 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.341 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.341 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.341 INFO:teuthology.orchestra.run.smithi032.stdout: "migration_current": { 2024-04-25T00:44:45.341 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "migration_current", 2024-04-25T00:44:45.341 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.341 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.341 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.341 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.341 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.341 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.341 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.341 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "internal - do not modify", 2024-04-25T00:44:45.341 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.341 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.342 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.342 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.342 INFO:teuthology.orchestra.run.smithi032.stdout: "mode": { 2024-04-25T00:44:45.342 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "mode", 2024-04-25T00:44:45.342 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.342 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.342 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.342 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "root", 2024-04-25T00:44:45.342 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.342 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.342 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.342 INFO:teuthology.orchestra.run.smithi032.stdout: "cephadm-package", 2024-04-25T00:44:45.342 INFO:teuthology.orchestra.run.smithi032.stdout: "root" 2024-04-25T00:44:45.342 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.343 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "mode for remote execution of cephadm", 2024-04-25T00:44:45.343 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.343 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.343 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.343 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.343 INFO:teuthology.orchestra.run.smithi032.stdout: "oob_default_addr": { 2024-04-25T00:44:45.343 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "oob_default_addr", 2024-04-25T00:44:45.343 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.343 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.343 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.343 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "169.254.1.1", 2024-04-25T00:44:45.343 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.343 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.343 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.343 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Default address for RedFish API (oob management).", 2024-04-25T00:44:45.344 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.344 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.344 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.344 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.344 INFO:teuthology.orchestra.run.smithi032.stdout: "prometheus_alerts_path": { 2024-04-25T00:44:45.344 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "prometheus_alerts_path", 2024-04-25T00:44:45.344 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.344 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.344 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.344 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "/etc/prometheus/ceph/ceph_default_alerts.yml", 2024-04-25T00:44:45.344 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.344 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.344 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.344 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "location of alerts to include in prometheus deployments", 2024-04-25T00:44:45.344 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.344 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.345 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.345 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.345 INFO:teuthology.orchestra.run.smithi032.stdout: "registry_insecure": { 2024-04-25T00:44:45.345 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "registry_insecure", 2024-04-25T00:44:45.345 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.345 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.345 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.345 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.345 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.345 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.345 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.345 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Registry is to be considered insecure (no TLS available). Only for development purposes.", 2024-04-25T00:44:45.345 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.345 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.345 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.346 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.346 INFO:teuthology.orchestra.run.smithi032.stdout: "registry_password": { 2024-04-25T00:44:45.346 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "registry_password", 2024-04-25T00:44:45.346 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.346 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.346 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.346 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.346 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.346 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.346 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.346 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Custom repository password. Only used for logging into a registry.", 2024-04-25T00:44:45.346 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.346 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.347 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.347 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.347 INFO:teuthology.orchestra.run.smithi032.stdout: "registry_url": { 2024-04-25T00:44:45.347 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "registry_url", 2024-04-25T00:44:45.347 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.347 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.347 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.347 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.347 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.347 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.347 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.347 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Registry url for login purposes. This is not the default registry", 2024-04-25T00:44:45.347 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.347 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.347 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.348 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.348 INFO:teuthology.orchestra.run.smithi032.stdout: "registry_username": { 2024-04-25T00:44:45.348 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "registry_username", 2024-04-25T00:44:45.348 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.348 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.348 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.348 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.348 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.348 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.348 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.348 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Custom repository username. Only used for logging into a registry.", 2024-04-25T00:44:45.348 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.348 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.348 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.348 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.349 INFO:teuthology.orchestra.run.smithi032.stdout: "secure_monitoring_stack": { 2024-04-25T00:44:45.349 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "secure_monitoring_stack", 2024-04-25T00:44:45.349 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.349 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.349 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.349 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.349 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.349 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.349 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.349 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Enable TLS security for all the monitoring stack daemons", 2024-04-25T00:44:45.349 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.349 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.349 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.349 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.350 INFO:teuthology.orchestra.run.smithi032.stdout: "service_discovery_port": { 2024-04-25T00:44:45.350 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "service_discovery_port", 2024-04-25T00:44:45.350 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.350 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.350 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.350 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "8765", 2024-04-25T00:44:45.350 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.350 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.350 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.350 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "cephadm service discovery port", 2024-04-25T00:44:45.350 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.350 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.350 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.350 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.351 INFO:teuthology.orchestra.run.smithi032.stdout: "ssh_config_file": { 2024-04-25T00:44:45.351 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ssh_config_file", 2024-04-25T00:44:45.351 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.351 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.351 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.351 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.351 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.351 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.351 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.351 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "customized SSH config file to connect to managed hosts", 2024-04-25T00:44:45.351 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.351 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.351 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.351 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.352 INFO:teuthology.orchestra.run.smithi032.stdout: "use_agent": { 2024-04-25T00:44:45.352 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "use_agent", 2024-04-25T00:44:45.352 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.352 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.352 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.352 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.352 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.352 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.352 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.352 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Use cephadm agent on each host to gather and send metadata", 2024-04-25T00:44:45.352 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.352 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.352 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.352 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.353 INFO:teuthology.orchestra.run.smithi032.stdout: "use_repo_digest": { 2024-04-25T00:44:45.353 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "use_repo_digest", 2024-04-25T00:44:45.353 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.353 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.353 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.353 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.353 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.353 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.353 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.353 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Automatically convert image tags to image digest. Make sure all daemons use the same image", 2024-04-25T00:44:45.353 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.353 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.353 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.353 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.353 INFO:teuthology.orchestra.run.smithi032.stdout: "warn_on_failed_host_check": { 2024-04-25T00:44:45.354 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "warn_on_failed_host_check", 2024-04-25T00:44:45.354 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.354 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.354 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.354 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.354 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.354 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.354 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.354 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "raise a health warning if the host check fails", 2024-04-25T00:44:45.354 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.354 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.354 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.354 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.354 INFO:teuthology.orchestra.run.smithi032.stdout: "warn_on_stray_daemons": { 2024-04-25T00:44:45.355 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "warn_on_stray_daemons", 2024-04-25T00:44:45.355 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.355 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.355 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.355 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.355 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.355 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.355 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.355 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "raise a health warning if daemons are detected that are not managed by cephadm", 2024-04-25T00:44:45.355 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.355 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.355 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.355 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.355 INFO:teuthology.orchestra.run.smithi032.stdout: "warn_on_stray_hosts": { 2024-04-25T00:44:45.356 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "warn_on_stray_hosts", 2024-04-25T00:44:45.356 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.356 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.356 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.356 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.356 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.356 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.356 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.356 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "raise a health warning if daemons are detected on a host that is not managed by cephadm", 2024-04-25T00:44:45.356 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.356 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.356 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.356 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.356 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.356 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.357 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.357 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "crash", 2024-04-25T00:44:45.357 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.357 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.357 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.357 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.357 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.357 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.357 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.357 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.357 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.357 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.357 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.357 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.357 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.358 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.358 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.358 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.358 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.358 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.358 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.358 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.358 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.358 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.358 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.358 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.358 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.358 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.358 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.358 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.359 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.359 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.359 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.359 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.359 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.359 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.359 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.359 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.359 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.359 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.359 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.359 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.359 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.359 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.359 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.360 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.360 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.360 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.360 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.360 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.360 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.360 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.360 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.360 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.360 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.360 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.361 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.361 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.361 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.361 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.361 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.361 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.361 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.361 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.361 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.361 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.361 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.361 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.361 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.361 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.361 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.362 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.362 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.362 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.362 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.362 INFO:teuthology.orchestra.run.smithi032.stdout: "retain_interval": { 2024-04-25T00:44:45.362 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "retain_interval", 2024-04-25T00:44:45.362 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.362 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.362 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.362 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "31536000", 2024-04-25T00:44:45.362 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.362 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.362 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.362 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "how long to retain crashes before pruning them", 2024-04-25T00:44:45.362 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.363 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.363 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.363 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.363 INFO:teuthology.orchestra.run.smithi032.stdout: "warn_recent_interval": { 2024-04-25T00:44:45.363 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "warn_recent_interval", 2024-04-25T00:44:45.363 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.363 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.363 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.363 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "1209600", 2024-04-25T00:44:45.363 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.363 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.363 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.363 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "time interval in which to warn about recent crashes", 2024-04-25T00:44:45.363 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.364 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.364 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.364 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.364 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.364 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.364 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.364 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "dashboard", 2024-04-25T00:44:45.364 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.364 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.364 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.364 INFO:teuthology.orchestra.run.smithi032.stdout: "ACCOUNT_LOCKOUT_ATTEMPTS": { 2024-04-25T00:44:45.364 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ACCOUNT_LOCKOUT_ATTEMPTS", 2024-04-25T00:44:45.364 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.364 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.365 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.365 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10", 2024-04-25T00:44:45.365 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.365 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.365 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.365 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.365 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.365 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.365 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.365 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.365 INFO:teuthology.orchestra.run.smithi032.stdout: "ALERTMANAGER_API_HOST": { 2024-04-25T00:44:45.365 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ALERTMANAGER_API_HOST", 2024-04-25T00:44:45.365 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.365 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.365 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.366 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.366 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.366 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.366 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.366 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.366 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.366 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.366 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.366 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.366 INFO:teuthology.orchestra.run.smithi032.stdout: "ALERTMANAGER_API_SSL_VERIFY": { 2024-04-25T00:44:45.366 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ALERTMANAGER_API_SSL_VERIFY", 2024-04-25T00:44:45.366 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.366 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.366 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.367 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.367 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.367 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.367 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.367 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.367 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.367 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.367 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.367 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.367 INFO:teuthology.orchestra.run.smithi032.stdout: "AUDIT_API_ENABLED": { 2024-04-25T00:44:45.367 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "AUDIT_API_ENABLED", 2024-04-25T00:44:45.367 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.367 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.367 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.367 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.368 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.368 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.368 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.368 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.368 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.368 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.368 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.368 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.368 INFO:teuthology.orchestra.run.smithi032.stdout: "AUDIT_API_LOG_PAYLOAD": { 2024-04-25T00:44:45.368 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "AUDIT_API_LOG_PAYLOAD", 2024-04-25T00:44:45.368 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.368 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.368 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.368 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.368 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.369 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.369 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.369 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.369 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.369 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.369 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.369 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.369 INFO:teuthology.orchestra.run.smithi032.stdout: "ENABLE_BROWSABLE_API": { 2024-04-25T00:44:45.369 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ENABLE_BROWSABLE_API", 2024-04-25T00:44:45.369 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.369 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.369 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.369 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.369 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.370 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.370 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.370 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.370 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.370 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.370 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.370 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.370 INFO:teuthology.orchestra.run.smithi032.stdout: "FEATURE_TOGGLE_CEPHFS": { 2024-04-25T00:44:45.370 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "FEATURE_TOGGLE_CEPHFS", 2024-04-25T00:44:45.370 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.370 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.370 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.370 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.370 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.370 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.371 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.371 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.371 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.371 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.371 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.371 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.371 INFO:teuthology.orchestra.run.smithi032.stdout: "FEATURE_TOGGLE_DASHBOARD": { 2024-04-25T00:44:45.371 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "FEATURE_TOGGLE_DASHBOARD", 2024-04-25T00:44:45.371 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.371 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.371 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.371 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.371 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.371 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.371 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.372 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.372 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.372 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.372 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.372 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.372 INFO:teuthology.orchestra.run.smithi032.stdout: "FEATURE_TOGGLE_ISCSI": { 2024-04-25T00:44:45.372 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "FEATURE_TOGGLE_ISCSI", 2024-04-25T00:44:45.372 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.372 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.372 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.372 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.372 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.372 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.372 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.372 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.373 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.373 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.373 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.373 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.373 INFO:teuthology.orchestra.run.smithi032.stdout: "FEATURE_TOGGLE_MIRRORING": { 2024-04-25T00:44:45.373 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "FEATURE_TOGGLE_MIRRORING", 2024-04-25T00:44:45.373 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.373 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.373 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.373 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.373 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.373 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.373 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.373 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.373 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.374 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.374 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.374 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.374 INFO:teuthology.orchestra.run.smithi032.stdout: "FEATURE_TOGGLE_NFS": { 2024-04-25T00:44:45.374 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "FEATURE_TOGGLE_NFS", 2024-04-25T00:44:45.374 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.374 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.374 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.374 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.374 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.374 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.374 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.374 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.374 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.375 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.375 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.375 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.375 INFO:teuthology.orchestra.run.smithi032.stdout: "FEATURE_TOGGLE_RBD": { 2024-04-25T00:44:45.375 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "FEATURE_TOGGLE_RBD", 2024-04-25T00:44:45.375 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.375 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.375 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.375 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.375 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.375 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.375 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.375 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.375 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.376 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.376 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.376 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.376 INFO:teuthology.orchestra.run.smithi032.stdout: "FEATURE_TOGGLE_RGW": { 2024-04-25T00:44:45.376 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "FEATURE_TOGGLE_RGW", 2024-04-25T00:44:45.376 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.376 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.376 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.376 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.376 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.376 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.376 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.376 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.376 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.377 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.377 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.377 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.377 INFO:teuthology.orchestra.run.smithi032.stdout: "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE": { 2024-04-25T00:44:45.377 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE", 2024-04-25T00:44:45.377 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.377 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.377 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.377 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.377 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.377 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.377 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.377 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.377 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.378 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.378 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.378 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.378 INFO:teuthology.orchestra.run.smithi032.stdout: "GRAFANA_API_PASSWORD": { 2024-04-25T00:44:45.378 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "GRAFANA_API_PASSWORD", 2024-04-25T00:44:45.378 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.378 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.378 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.378 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "admin", 2024-04-25T00:44:45.378 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.378 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.378 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.378 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.378 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.378 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.379 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.379 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.379 INFO:teuthology.orchestra.run.smithi032.stdout: "GRAFANA_API_SSL_VERIFY": { 2024-04-25T00:44:45.379 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "GRAFANA_API_SSL_VERIFY", 2024-04-25T00:44:45.379 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.379 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.379 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.379 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.379 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.381 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.381 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.381 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.381 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.381 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.381 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.381 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.382 INFO:teuthology.orchestra.run.smithi032.stdout: "GRAFANA_API_URL": { 2024-04-25T00:44:45.382 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "GRAFANA_API_URL", 2024-04-25T00:44:45.382 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.382 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.382 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.382 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.382 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.382 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.382 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.382 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.382 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.382 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.382 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.382 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.383 INFO:teuthology.orchestra.run.smithi032.stdout: "GRAFANA_API_USERNAME": { 2024-04-25T00:44:45.383 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "GRAFANA_API_USERNAME", 2024-04-25T00:44:45.383 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.383 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.383 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.383 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "admin", 2024-04-25T00:44:45.383 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.383 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.383 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.383 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.383 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.383 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.383 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.383 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.383 INFO:teuthology.orchestra.run.smithi032.stdout: "GRAFANA_FRONTEND_API_URL": { 2024-04-25T00:44:45.384 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "GRAFANA_FRONTEND_API_URL", 2024-04-25T00:44:45.384 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.384 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.384 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.384 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.384 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.384 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.384 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.384 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.384 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.384 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.384 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.384 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.384 INFO:teuthology.orchestra.run.smithi032.stdout: "GRAFANA_UPDATE_DASHBOARDS": { 2024-04-25T00:44:45.384 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "GRAFANA_UPDATE_DASHBOARDS", 2024-04-25T00:44:45.385 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.385 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.385 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.385 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.385 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.385 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.385 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.385 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.385 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.385 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.385 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.385 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.385 INFO:teuthology.orchestra.run.smithi032.stdout: "ISCSI_API_SSL_VERIFICATION": { 2024-04-25T00:44:45.385 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ISCSI_API_SSL_VERIFICATION", 2024-04-25T00:44:45.386 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.386 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.386 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.386 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.386 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.386 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.386 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.386 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.386 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.386 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.386 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.386 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.386 INFO:teuthology.orchestra.run.smithi032.stdout: "ISSUE_TRACKER_API_KEY": { 2024-04-25T00:44:45.386 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ISSUE_TRACKER_API_KEY", 2024-04-25T00:44:45.387 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.387 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.387 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.387 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.387 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.387 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.387 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.387 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.387 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.387 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.387 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.387 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.387 INFO:teuthology.orchestra.run.smithi032.stdout: "PROMETHEUS_API_HOST": { 2024-04-25T00:44:45.387 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PROMETHEUS_API_HOST", 2024-04-25T00:44:45.387 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.388 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.388 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.388 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.388 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.388 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.388 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.388 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.388 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.388 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.388 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.388 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.388 INFO:teuthology.orchestra.run.smithi032.stdout: "PROMETHEUS_API_SSL_VERIFY": { 2024-04-25T00:44:45.388 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PROMETHEUS_API_SSL_VERIFY", 2024-04-25T00:44:45.388 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.388 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.389 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.389 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.389 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.389 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.389 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.389 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.389 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.389 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.389 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.389 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.389 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_CHECK_COMPLEXITY_ENABLED": { 2024-04-25T00:44:45.389 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_CHECK_COMPLEXITY_ENABLED", 2024-04-25T00:44:45.389 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.389 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.390 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.390 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.390 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.390 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.390 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.390 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.390 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.390 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.390 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.390 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.390 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED": { 2024-04-25T00:44:45.390 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED", 2024-04-25T00:44:45.390 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.390 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.390 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.391 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.391 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.391 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.391 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.391 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.391 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.391 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.391 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.391 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.391 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_CHECK_LENGTH_ENABLED": { 2024-04-25T00:44:45.391 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_CHECK_LENGTH_ENABLED", 2024-04-25T00:44:45.391 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.391 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.391 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.392 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.392 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.392 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.392 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.392 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.392 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.392 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.392 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.392 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.392 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_CHECK_OLDPWD_ENABLED": { 2024-04-25T00:44:45.392 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_CHECK_OLDPWD_ENABLED", 2024-04-25T00:44:45.392 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.392 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.392 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.392 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.393 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.393 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.393 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.393 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.393 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.393 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.393 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.393 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.393 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED": { 2024-04-25T00:44:45.393 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED", 2024-04-25T00:44:45.393 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.393 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.393 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.393 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.393 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.394 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.394 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.394 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.394 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.394 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.394 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.394 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.394 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED": { 2024-04-25T00:44:45.394 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED", 2024-04-25T00:44:45.394 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.394 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.394 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.394 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.394 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.395 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.395 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.395 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.395 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.395 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.395 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.395 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.395 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_CHECK_USERNAME_ENABLED": { 2024-04-25T00:44:45.395 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_CHECK_USERNAME_ENABLED", 2024-04-25T00:44:45.395 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.395 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.395 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.395 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.395 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.395 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.396 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.396 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.396 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.396 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.396 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.396 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.396 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_ENABLED": { 2024-04-25T00:44:45.396 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_ENABLED", 2024-04-25T00:44:45.396 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.396 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.396 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.396 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.396 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.396 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.396 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.397 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.397 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.397 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.397 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.397 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.397 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_EXCLUSION_LIST": { 2024-04-25T00:44:45.397 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_EXCLUSION_LIST", 2024-04-25T00:44:45.397 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.397 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.397 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.397 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps", 2024-04-25T00:44:45.397 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.397 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.397 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.398 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.398 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.398 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.398 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.398 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.398 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_MIN_COMPLEXITY": { 2024-04-25T00:44:45.398 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_MIN_COMPLEXITY", 2024-04-25T00:44:45.398 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.398 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.398 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.398 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10", 2024-04-25T00:44:45.398 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.398 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.398 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.398 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.399 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.399 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.399 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.399 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.399 INFO:teuthology.orchestra.run.smithi032.stdout: "PWD_POLICY_MIN_LENGTH": { 2024-04-25T00:44:45.399 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "PWD_POLICY_MIN_LENGTH", 2024-04-25T00:44:45.399 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.399 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.399 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.399 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "8", 2024-04-25T00:44:45.399 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.399 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.399 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.399 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.400 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.400 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.400 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.400 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.400 INFO:teuthology.orchestra.run.smithi032.stdout: "REST_REQUESTS_TIMEOUT": { 2024-04-25T00:44:45.400 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "REST_REQUESTS_TIMEOUT", 2024-04-25T00:44:45.400 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.400 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.400 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.401 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "45", 2024-04-25T00:44:45.401 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.401 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.401 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.401 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.401 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.401 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.401 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.401 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.401 INFO:teuthology.orchestra.run.smithi032.stdout: "RGW_API_ACCESS_KEY": { 2024-04-25T00:44:45.402 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "RGW_API_ACCESS_KEY", 2024-04-25T00:44:45.402 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.402 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.402 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.402 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.402 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.402 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.402 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.402 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.402 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.402 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.402 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.402 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.402 INFO:teuthology.orchestra.run.smithi032.stdout: "RGW_API_ADMIN_RESOURCE": { 2024-04-25T00:44:45.403 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "RGW_API_ADMIN_RESOURCE", 2024-04-25T00:44:45.403 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.403 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.403 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.403 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "admin", 2024-04-25T00:44:45.403 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.403 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.403 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.403 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.403 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.403 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.403 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.403 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.403 INFO:teuthology.orchestra.run.smithi032.stdout: "RGW_API_SECRET_KEY": { 2024-04-25T00:44:45.403 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "RGW_API_SECRET_KEY", 2024-04-25T00:44:45.404 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.404 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.404 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.404 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.404 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.404 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.404 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.404 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.404 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.404 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.404 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.404 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.404 INFO:teuthology.orchestra.run.smithi032.stdout: "RGW_API_SSL_VERIFY": { 2024-04-25T00:44:45.404 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "RGW_API_SSL_VERIFY", 2024-04-25T00:44:45.404 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.405 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.405 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.405 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.405 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.405 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.405 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.405 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.405 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.405 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.405 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.405 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.405 INFO:teuthology.orchestra.run.smithi032.stdout: "USER_PWD_EXPIRATION_SPAN": { 2024-04-25T00:44:45.405 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "USER_PWD_EXPIRATION_SPAN", 2024-04-25T00:44:45.405 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.406 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.406 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.406 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "0", 2024-04-25T00:44:45.406 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.406 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.406 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.406 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.406 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.406 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.406 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.406 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.406 INFO:teuthology.orchestra.run.smithi032.stdout: "USER_PWD_EXPIRATION_WARNING_1": { 2024-04-25T00:44:45.406 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "USER_PWD_EXPIRATION_WARNING_1", 2024-04-25T00:44:45.406 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.406 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.407 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.407 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10", 2024-04-25T00:44:45.407 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.407 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.407 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.407 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.407 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.407 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.407 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.407 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.407 INFO:teuthology.orchestra.run.smithi032.stdout: "USER_PWD_EXPIRATION_WARNING_2": { 2024-04-25T00:44:45.407 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "USER_PWD_EXPIRATION_WARNING_2", 2024-04-25T00:44:45.407 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.407 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.407 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.408 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "5", 2024-04-25T00:44:45.408 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.408 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.408 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.408 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.408 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.408 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.408 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.408 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.408 INFO:teuthology.orchestra.run.smithi032.stdout: "cross_origin_url": { 2024-04-25T00:44:45.408 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "cross_origin_url", 2024-04-25T00:44:45.408 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.408 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.408 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.408 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.409 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.409 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.409 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.409 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.409 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.409 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.409 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.409 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.409 INFO:teuthology.orchestra.run.smithi032.stdout: "crt_file": { 2024-04-25T00:44:45.409 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "crt_file", 2024-04-25T00:44:45.409 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.409 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.409 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.409 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.409 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.410 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.410 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.410 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.410 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.410 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.410 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.410 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.410 INFO:teuthology.orchestra.run.smithi032.stdout: "debug": { 2024-04-25T00:44:45.410 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "debug", 2024-04-25T00:44:45.410 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.410 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.410 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.410 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.410 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.410 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.411 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.411 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Enable/disable debug options", 2024-04-25T00:44:45.411 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.411 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.411 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.411 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.411 INFO:teuthology.orchestra.run.smithi032.stdout: "jwt_token_ttl": { 2024-04-25T00:44:45.411 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "jwt_token_ttl", 2024-04-25T00:44:45.411 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.411 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.411 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.411 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "28800", 2024-04-25T00:44:45.411 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.411 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.411 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.412 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.412 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.412 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.412 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.412 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.412 INFO:teuthology.orchestra.run.smithi032.stdout: "key_file": { 2024-04-25T00:44:45.412 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "key_file", 2024-04-25T00:44:45.412 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.412 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.412 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.412 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.412 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.412 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.412 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.412 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.413 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.413 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.413 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.413 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.413 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.413 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.413 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.413 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.413 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.413 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.413 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.413 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.413 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.413 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.414 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.414 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.414 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.414 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.414 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.414 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.414 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.414 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.414 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.414 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.414 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.414 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.414 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.414 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.414 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.415 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.415 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.415 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.415 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.415 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.415 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.415 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.415 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.415 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.415 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.415 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.415 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.415 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.415 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.416 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.416 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.416 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.416 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.416 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.416 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.416 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.416 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.416 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.416 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.416 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.416 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.416 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.416 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.417 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.417 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.417 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.417 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.417 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.417 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.417 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.417 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.417 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.417 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.417 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.417 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.417 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.417 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.418 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.418 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.418 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.418 INFO:teuthology.orchestra.run.smithi032.stdout: "motd": { 2024-04-25T00:44:45.418 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "motd", 2024-04-25T00:44:45.418 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.418 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.418 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.418 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.418 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.418 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.418 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.418 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "The message of the day", 2024-04-25T00:44:45.418 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.419 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.419 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.419 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.419 INFO:teuthology.orchestra.run.smithi032.stdout: "redirect_resolve_ip_addr": { 2024-04-25T00:44:45.419 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "redirect_resolve_ip_addr", 2024-04-25T00:44:45.419 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.419 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.419 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.419 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.419 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.419 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.419 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.419 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.419 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.419 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.420 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.420 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.420 INFO:teuthology.orchestra.run.smithi032.stdout: "server_addr": { 2024-04-25T00:44:45.420 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "server_addr", 2024-04-25T00:44:45.420 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.420 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.420 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.420 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "::", 2024-04-25T00:44:45.420 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.420 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.420 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.420 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.420 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.420 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.420 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.421 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.421 INFO:teuthology.orchestra.run.smithi032.stdout: "server_port": { 2024-04-25T00:44:45.421 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "server_port", 2024-04-25T00:44:45.421 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.421 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.421 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.421 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "8080", 2024-04-25T00:44:45.421 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.421 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.421 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.421 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.421 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.421 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.421 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.421 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.422 INFO:teuthology.orchestra.run.smithi032.stdout: "ssl": { 2024-04-25T00:44:45.422 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ssl", 2024-04-25T00:44:45.422 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.422 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.422 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.422 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.422 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.422 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.422 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.422 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.422 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.422 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.422 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.422 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.422 INFO:teuthology.orchestra.run.smithi032.stdout: "ssl_server_port": { 2024-04-25T00:44:45.423 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ssl_server_port", 2024-04-25T00:44:45.423 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.423 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.423 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.423 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "8443", 2024-04-25T00:44:45.423 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.423 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.423 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.423 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.423 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.423 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.423 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.423 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.423 INFO:teuthology.orchestra.run.smithi032.stdout: "standby_behaviour": { 2024-04-25T00:44:45.423 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "standby_behaviour", 2024-04-25T00:44:45.423 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.424 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.424 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.424 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "redirect", 2024-04-25T00:44:45.424 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.424 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.424 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.424 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.424 INFO:teuthology.orchestra.run.smithi032.stdout: "redirect" 2024-04-25T00:44:45.424 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.424 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.424 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.424 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.424 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.424 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.424 INFO:teuthology.orchestra.run.smithi032.stdout: "standby_error_status_code": { 2024-04-25T00:44:45.425 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "standby_error_status_code", 2024-04-25T00:44:45.425 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.425 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.425 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.425 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "500", 2024-04-25T00:44:45.425 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "400", 2024-04-25T00:44:45.425 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "599", 2024-04-25T00:44:45.425 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.425 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.425 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.425 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.425 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.425 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.425 INFO:teuthology.orchestra.run.smithi032.stdout: "url_prefix": { 2024-04-25T00:44:45.425 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "url_prefix", 2024-04-25T00:44:45.426 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.426 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.426 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.426 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.426 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.426 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.426 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.426 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.426 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.426 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.426 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.426 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.426 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.426 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.426 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.427 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "devicehealth", 2024-04-25T00:44:45.427 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.427 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.427 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.427 INFO:teuthology.orchestra.run.smithi032.stdout: "enable_monitoring": { 2024-04-25T00:44:45.427 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "enable_monitoring", 2024-04-25T00:44:45.427 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.427 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.427 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.427 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.427 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.427 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.427 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.427 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "monitor device health metrics", 2024-04-25T00:44:45.427 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.428 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.428 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.428 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.428 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.428 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.428 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.428 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.428 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.428 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.428 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.428 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.428 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.428 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.428 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.428 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.428 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.429 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.429 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.429 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.429 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.429 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.429 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.429 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.429 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.429 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.429 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.429 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.429 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.429 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.429 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.430 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.430 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.430 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.430 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.430 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.430 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.430 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.430 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.430 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.430 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.430 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.430 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.430 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.430 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.431 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.431 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.431 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.431 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.431 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.431 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.431 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.431 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.431 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.431 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.431 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.431 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.431 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.431 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.431 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.432 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.432 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.432 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.432 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.432 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.432 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.432 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.432 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.432 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.432 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.432 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.432 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.432 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.432 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.432 INFO:teuthology.orchestra.run.smithi032.stdout: "mark_out_threshold": { 2024-04-25T00:44:45.433 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "mark_out_threshold", 2024-04-25T00:44:45.433 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.433 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.433 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.433 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "2419200", 2024-04-25T00:44:45.433 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.433 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.433 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.433 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "automatically mark OSD if it may fail before this long", 2024-04-25T00:44:45.433 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.433 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.433 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.433 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.433 INFO:teuthology.orchestra.run.smithi032.stdout: "pool_name": { 2024-04-25T00:44:45.433 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "pool_name", 2024-04-25T00:44:45.433 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.434 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.434 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.434 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "device_health_metrics", 2024-04-25T00:44:45.434 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.434 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.434 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.434 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "name of pool in which to store device health metrics", 2024-04-25T00:44:45.434 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.434 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.434 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.434 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.434 INFO:teuthology.orchestra.run.smithi032.stdout: "retention_period": { 2024-04-25T00:44:45.434 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "retention_period", 2024-04-25T00:44:45.434 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.435 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.435 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.435 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "15552000", 2024-04-25T00:44:45.435 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.435 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.435 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.435 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "how long to retain device health metrics", 2024-04-25T00:44:45.435 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.435 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.435 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.435 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.435 INFO:teuthology.orchestra.run.smithi032.stdout: "scrape_frequency": { 2024-04-25T00:44:45.435 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "scrape_frequency", 2024-04-25T00:44:45.435 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.435 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.436 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.436 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "86400", 2024-04-25T00:44:45.436 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.436 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.436 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.436 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "how frequently to scrape device health metrics", 2024-04-25T00:44:45.436 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.436 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.436 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.436 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.436 INFO:teuthology.orchestra.run.smithi032.stdout: "self_heal": { 2024-04-25T00:44:45.436 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "self_heal", 2024-04-25T00:44:45.436 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.436 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.436 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.436 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.437 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.437 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.437 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.437 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "preemptively heal cluster around devices that may fail", 2024-04-25T00:44:45.437 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.437 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.437 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.437 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.437 INFO:teuthology.orchestra.run.smithi032.stdout: "sleep_interval": { 2024-04-25T00:44:45.437 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "sleep_interval", 2024-04-25T00:44:45.437 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.437 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.437 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.437 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "600", 2024-04-25T00:44:45.437 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.438 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.438 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.438 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "how frequently to wake up and check device health", 2024-04-25T00:44:45.438 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.438 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.438 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.438 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.438 INFO:teuthology.orchestra.run.smithi032.stdout: "warn_threshold": { 2024-04-25T00:44:45.438 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "warn_threshold", 2024-04-25T00:44:45.438 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.438 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.438 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.438 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "7257600", 2024-04-25T00:44:45.438 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.438 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.439 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.439 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "raise health warning if OSD may fail before this long", 2024-04-25T00:44:45.439 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.439 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.439 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.439 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.439 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.439 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.439 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.439 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "diskprediction_local", 2024-04-25T00:44:45.439 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.439 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.439 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.439 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.439 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.440 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.440 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.440 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.440 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.440 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.440 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.440 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.440 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.440 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.440 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.440 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.440 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.440 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.440 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.440 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.440 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.441 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.441 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.441 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.441 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.441 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.441 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.441 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.441 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.441 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.441 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.441 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.441 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.441 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.441 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.441 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.442 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.442 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.442 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.442 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.442 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.442 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.442 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.442 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.442 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.442 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.442 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.442 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.442 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.442 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.442 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.443 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.443 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.443 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.443 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.443 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.443 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.443 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.443 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.443 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.443 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.443 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.443 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.443 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.443 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.443 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.444 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.444 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.444 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.444 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.444 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.444 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.444 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.444 INFO:teuthology.orchestra.run.smithi032.stdout: "predict_interval": { 2024-04-25T00:44:45.444 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "predict_interval", 2024-04-25T00:44:45.444 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.444 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.444 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.445 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "86400", 2024-04-25T00:44:45.445 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.445 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.445 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.445 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.445 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.445 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.445 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.445 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.445 INFO:teuthology.orchestra.run.smithi032.stdout: "predictor_model": { 2024-04-25T00:44:45.445 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "predictor_model", 2024-04-25T00:44:45.445 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.445 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.445 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.445 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "prophetstor", 2024-04-25T00:44:45.446 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.446 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.446 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.446 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.446 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.446 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.446 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.446 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.446 INFO:teuthology.orchestra.run.smithi032.stdout: "sleep_interval": { 2024-04-25T00:44:45.446 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "sleep_interval", 2024-04-25T00:44:45.446 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.446 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.446 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.446 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "600", 2024-04-25T00:44:45.447 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.447 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.447 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.447 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.447 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.447 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.447 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.447 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.447 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.447 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.447 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.447 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "influx", 2024-04-25T00:44:45.447 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": false, 2024-04-25T00:44:45.447 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "influxdb python module not found", 2024-04-25T00:44:45.447 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.448 INFO:teuthology.orchestra.run.smithi032.stdout: "batch_size": { 2024-04-25T00:44:45.448 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "batch_size", 2024-04-25T00:44:45.448 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.448 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.448 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.448 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "5000", 2024-04-25T00:44:45.448 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.448 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.448 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.448 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "How big batches of data points should be when sending to InfluxDB.", 2024-04-25T00:44:45.448 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.448 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.448 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.448 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.448 INFO:teuthology.orchestra.run.smithi032.stdout: "database": { 2024-04-25T00:44:45.449 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "database", 2024-04-25T00:44:45.449 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.449 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.449 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.449 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "ceph", 2024-04-25T00:44:45.449 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.449 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.449 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.449 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.", 2024-04-25T00:44:45.449 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.449 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.449 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.449 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.449 INFO:teuthology.orchestra.run.smithi032.stdout: "hostname": { 2024-04-25T00:44:45.449 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "hostname", 2024-04-25T00:44:45.450 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.450 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.450 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.450 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.450 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.450 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.450 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.450 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "InfluxDB server hostname", 2024-04-25T00:44:45.450 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.450 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.450 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.450 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.450 INFO:teuthology.orchestra.run.smithi032.stdout: "interval": { 2024-04-25T00:44:45.450 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "interval", 2024-04-25T00:44:45.451 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.451 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.451 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.451 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "30", 2024-04-25T00:44:45.451 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "5", 2024-04-25T00:44:45.451 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.451 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.451 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Time between reports to InfluxDB. Default 30 seconds.", 2024-04-25T00:44:45.451 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.451 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.451 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.451 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.451 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.451 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.451 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.452 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.452 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.452 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.452 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.452 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.452 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.452 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.452 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.452 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.452 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.452 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.452 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.452 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.452 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.452 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.452 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.453 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.453 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.453 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.453 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.453 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.453 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.453 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.453 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.453 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.453 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.453 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.453 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.453 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.453 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.453 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.454 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.454 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.454 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.454 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.454 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.454 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.454 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.454 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.454 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.454 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.454 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.454 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.454 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.454 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.454 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.455 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.455 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.455 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.455 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.455 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.455 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.455 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.455 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.455 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.455 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.455 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.455 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.455 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.455 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.455 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.455 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.456 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.456 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.456 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.456 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.456 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.456 INFO:teuthology.orchestra.run.smithi032.stdout: "password": { 2024-04-25T00:44:45.456 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "password", 2024-04-25T00:44:45.456 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.456 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.456 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.456 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.456 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.456 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.456 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.456 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "password of InfluxDB server user", 2024-04-25T00:44:45.457 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.457 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.457 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.457 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.457 INFO:teuthology.orchestra.run.smithi032.stdout: "port": { 2024-04-25T00:44:45.457 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "port", 2024-04-25T00:44:45.457 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.457 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.457 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.457 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "8086", 2024-04-25T00:44:45.457 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.457 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.457 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.457 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "InfluxDB server port", 2024-04-25T00:44:45.457 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.457 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.458 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.458 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.458 INFO:teuthology.orchestra.run.smithi032.stdout: "ssl": { 2024-04-25T00:44:45.458 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ssl", 2024-04-25T00:44:45.458 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.458 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.458 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.458 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "false", 2024-04-25T00:44:45.458 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.458 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.458 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.458 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Use https connection for InfluxDB server. Use \"true\" or \"false\".", 2024-04-25T00:44:45.458 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.458 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.459 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.459 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.459 INFO:teuthology.orchestra.run.smithi032.stdout: "threads": { 2024-04-25T00:44:45.459 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "threads", 2024-04-25T00:44:45.459 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.459 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.459 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.459 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "5", 2024-04-25T00:44:45.459 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "1", 2024-04-25T00:44:45.459 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "32", 2024-04-25T00:44:45.459 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.459 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "How many worker threads should be spawned for sending data to InfluxDB.", 2024-04-25T00:44:45.459 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.459 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.459 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.460 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.460 INFO:teuthology.orchestra.run.smithi032.stdout: "username": { 2024-04-25T00:44:45.460 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "username", 2024-04-25T00:44:45.460 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.460 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.460 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.460 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.460 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.460 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.460 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.460 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "username of InfluxDB server user", 2024-04-25T00:44:45.460 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.460 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.460 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.460 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.461 INFO:teuthology.orchestra.run.smithi032.stdout: "verify_ssl": { 2024-04-25T00:44:45.461 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "verify_ssl", 2024-04-25T00:44:45.461 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.461 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.461 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.461 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "true", 2024-04-25T00:44:45.461 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.461 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.461 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.461 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Verify https cert for InfluxDB server. Use \"true\" or \"false\".", 2024-04-25T00:44:45.461 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.461 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.461 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.461 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.461 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.462 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.462 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.462 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "insights", 2024-04-25T00:44:45.462 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.462 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.462 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.462 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.462 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.462 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.462 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.462 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.462 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.462 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.462 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.462 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.463 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.463 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.463 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.463 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.463 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.463 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.463 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.463 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.463 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.463 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.463 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.463 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.463 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.463 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.463 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.464 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.464 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.464 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.464 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.464 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.464 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.464 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.464 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.464 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.464 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.464 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.464 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.464 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.464 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.464 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.465 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.465 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.465 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.465 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.465 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.465 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.465 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.465 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.465 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.465 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.465 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.465 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.465 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.465 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.465 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.466 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.466 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.466 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.466 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.466 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.466 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.466 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.466 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.466 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.466 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.466 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.466 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.466 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.466 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.466 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.467 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.467 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.467 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.467 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.467 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "iostat", 2024-04-25T00:44:45.467 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.467 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.467 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.467 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.467 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.467 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.467 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.467 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.467 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.467 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.468 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.468 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.468 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.468 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.468 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.468 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.468 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.468 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.468 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.468 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.468 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.468 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.468 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.468 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.468 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.469 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.469 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.469 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.469 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.469 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.469 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.469 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.469 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.469 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.469 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.469 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.469 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.469 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.469 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.469 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.470 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.470 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.470 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.470 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.470 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.470 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.470 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.470 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.470 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.470 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.470 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.470 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.470 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.470 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.470 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.471 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.471 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.471 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.471 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.471 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.471 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.471 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.471 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.471 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.471 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.471 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.471 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.471 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.471 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.471 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.471 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.472 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.472 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.472 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.472 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.472 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.472 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "k8sevents", 2024-04-25T00:44:45.472 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.472 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.472 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.472 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph_event_retention_days": { 2024-04-25T00:44:45.472 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "ceph_event_retention_days", 2024-04-25T00:44:45.472 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.472 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.472 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.472 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "7", 2024-04-25T00:44:45.473 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.473 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.473 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.473 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Days to hold ceph event information within local cache", 2024-04-25T00:44:45.473 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.473 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.473 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.473 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.473 INFO:teuthology.orchestra.run.smithi032.stdout: "config_check_secs": { 2024-04-25T00:44:45.473 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "config_check_secs", 2024-04-25T00:44:45.473 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.473 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.473 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.473 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10", 2024-04-25T00:44:45.474 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "10", 2024-04-25T00:44:45.474 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.474 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.474 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "interval (secs) to check for cluster configuration changes", 2024-04-25T00:44:45.474 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.474 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.474 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.474 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.474 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.474 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.474 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.474 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.474 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.474 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.474 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.475 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.475 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.475 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.475 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.475 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.475 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.475 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.475 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.475 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.475 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.475 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.475 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.475 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.475 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.476 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.476 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.476 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.476 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.476 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.476 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.476 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.476 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.476 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.476 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.476 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.476 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.476 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.476 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.476 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.477 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.477 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.477 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.477 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.477 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.477 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.477 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.477 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.477 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.477 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.477 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.477 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.477 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.477 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.478 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.478 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.478 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.478 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.478 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.478 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.478 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.478 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.478 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.478 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.478 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.478 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.478 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.478 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.478 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.479 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.479 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.479 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.479 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.479 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.479 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.479 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.479 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.479 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "localpool", 2024-04-25T00:44:45.479 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.479 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.479 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.479 INFO:teuthology.orchestra.run.smithi032.stdout: "failure_domain": { 2024-04-25T00:44:45.479 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "failure_domain", 2024-04-25T00:44:45.479 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.480 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.480 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.480 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "host", 2024-04-25T00:44:45.480 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.480 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.480 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.480 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "failure domain for any created local pool", 2024-04-25T00:44:45.480 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "what failure domain we should separate data replicas across.", 2024-04-25T00:44:45.480 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.480 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.480 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.480 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.480 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.480 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.480 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.481 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.481 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.481 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.481 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.481 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.481 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.481 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.481 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.481 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.481 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.481 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.481 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.481 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.481 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.481 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.482 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.482 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.482 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.482 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.482 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.482 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.482 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.482 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.482 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.482 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.482 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.482 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.482 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.482 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.482 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.483 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.483 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.483 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.483 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.483 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.483 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.483 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.483 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.483 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.483 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.483 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.483 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.483 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.483 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.483 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.484 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.484 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.484 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.484 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.484 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.484 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.484 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.484 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.484 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.484 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.484 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.484 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.484 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.484 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.484 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.485 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.485 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.485 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.485 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.485 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.485 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.485 INFO:teuthology.orchestra.run.smithi032.stdout: "min_size": { 2024-04-25T00:44:45.485 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "min_size", 2024-04-25T00:44:45.485 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.485 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.485 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.485 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.485 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.485 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.485 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.486 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "default min_size for any created local pool", 2024-04-25T00:44:45.486 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "value to set min_size to (unchanged from Ceph's default if this option is not set)", 2024-04-25T00:44:45.486 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.486 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.486 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.486 INFO:teuthology.orchestra.run.smithi032.stdout: "num_rep": { 2024-04-25T00:44:45.486 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "num_rep", 2024-04-25T00:44:45.486 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.486 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.486 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.486 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "3", 2024-04-25T00:44:45.486 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.486 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.486 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.486 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "default replica count for any created local pool", 2024-04-25T00:44:45.487 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.487 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.487 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.487 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.487 INFO:teuthology.orchestra.run.smithi032.stdout: "pg_num": { 2024-04-25T00:44:45.487 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "pg_num", 2024-04-25T00:44:45.487 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.487 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.487 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.487 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "128", 2024-04-25T00:44:45.487 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.487 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.487 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.487 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "default pg_num for any created local pool", 2024-04-25T00:44:45.487 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.488 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.488 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.488 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.488 INFO:teuthology.orchestra.run.smithi032.stdout: "prefix": { 2024-04-25T00:44:45.488 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "prefix", 2024-04-25T00:44:45.488 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.488 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.488 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.488 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.488 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.488 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.488 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.488 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "name prefix for any created local pool", 2024-04-25T00:44:45.488 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.488 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.489 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.489 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.489 INFO:teuthology.orchestra.run.smithi032.stdout: "subtree": { 2024-04-25T00:44:45.489 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "subtree", 2024-04-25T00:44:45.489 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.489 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.489 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.489 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "rack", 2024-04-25T00:44:45.489 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.489 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.489 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.489 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "CRUSH level for which to create a local pool", 2024-04-25T00:44:45.489 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "which CRUSH subtree type the module should create a pool for.", 2024-04-25T00:44:45.489 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.489 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.489 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.490 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.490 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.490 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.490 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "mds_autoscaler", 2024-04-25T00:44:45.490 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.490 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.490 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.490 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.490 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.490 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.490 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.490 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.490 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.490 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.490 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.491 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.491 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.491 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.491 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.491 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.491 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.491 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.491 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.491 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.491 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.491 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.491 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.491 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.491 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.492 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.492 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.492 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.492 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.492 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.492 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.492 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.492 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.492 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.492 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.492 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.492 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.492 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.492 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.492 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.493 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.493 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.493 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.493 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.493 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.493 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.493 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.493 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.493 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.493 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.493 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.493 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.493 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.493 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.493 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.494 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.494 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.494 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.494 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.494 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.494 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.494 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.494 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.494 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.494 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.494 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.494 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.494 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.494 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.494 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.495 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.495 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.495 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.495 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.495 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.495 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.495 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "mirroring", 2024-04-25T00:44:45.495 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.495 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.495 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.495 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.495 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.495 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.495 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.495 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.496 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.496 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.496 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.496 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.496 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.496 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.496 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.496 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.496 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.496 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.496 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.496 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.496 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.496 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.496 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.497 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.497 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.497 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.497 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.497 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.497 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.497 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.497 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.497 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.497 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.497 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.497 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.497 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.497 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.497 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.498 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.498 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.498 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.498 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.498 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.498 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.498 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.498 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.498 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.498 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.498 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.498 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.498 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.498 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.498 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.499 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.499 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.499 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.499 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.499 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.499 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.499 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.499 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.499 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.499 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.499 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.499 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.499 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.499 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.499 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.500 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.500 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.500 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.500 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.500 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.500 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.500 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.500 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.500 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "nfs", 2024-04-25T00:44:45.500 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.500 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.500 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.500 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.500 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.500 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.501 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.501 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.501 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.501 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.501 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.501 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.501 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.501 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.501 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.501 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.501 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.501 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.501 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.501 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.501 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.502 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.502 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.502 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.502 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.502 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.502 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.502 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.502 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.502 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.502 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.502 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.502 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.502 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.502 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.502 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.503 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.503 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.503 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.503 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.503 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.503 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.503 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.503 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.503 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.503 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.503 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.503 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.503 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.503 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.503 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.504 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.504 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.504 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.504 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.504 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.504 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.504 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.504 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.504 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.504 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.504 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.504 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.504 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.504 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.504 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.504 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.505 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.505 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.505 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.505 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.505 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.505 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.505 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.505 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.505 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.505 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "orchestrator", 2024-04-25T00:44:45.505 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.505 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.505 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.505 INFO:teuthology.orchestra.run.smithi032.stdout: "fail_fs": { 2024-04-25T00:44:45.505 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "fail_fs", 2024-04-25T00:44:45.506 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.506 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.506 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.506 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.506 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.506 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.506 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.506 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Fail filesystem for rapid multi-rank mds upgrade", 2024-04-25T00:44:45.506 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.506 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.506 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.506 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.506 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.506 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.506 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.507 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.507 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.507 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.507 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.507 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.507 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.507 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.507 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.507 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.507 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.507 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.507 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.507 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.507 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.507 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.508 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.508 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.508 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.508 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.508 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.508 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.508 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.508 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.508 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.508 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.508 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.508 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.508 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.508 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.508 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.509 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.509 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.509 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.509 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.509 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.509 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.509 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.509 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.509 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.509 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.509 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.509 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.509 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.509 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.509 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.510 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.510 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.510 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.510 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.510 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.510 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.510 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.510 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.510 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.510 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.510 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.510 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.510 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.510 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.510 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.511 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.511 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.511 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.511 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.511 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.511 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.511 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.511 INFO:teuthology.orchestra.run.smithi032.stdout: "orchestrator": { 2024-04-25T00:44:45.511 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "orchestrator", 2024-04-25T00:44:45.511 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.511 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.511 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.511 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.511 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.511 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.512 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.512 INFO:teuthology.orchestra.run.smithi032.stdout: "cephadm", 2024-04-25T00:44:45.512 INFO:teuthology.orchestra.run.smithi032.stdout: "rook", 2024-04-25T00:44:45.512 INFO:teuthology.orchestra.run.smithi032.stdout: "test_orchestrator" 2024-04-25T00:44:45.512 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.512 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Orchestrator backend", 2024-04-25T00:44:45.512 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.512 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.512 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.512 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.512 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.512 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.512 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.512 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "osd_perf_query", 2024-04-25T00:44:45.513 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.513 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.513 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.513 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.513 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.513 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.513 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.513 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.513 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.513 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.513 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.513 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.513 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.513 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.513 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.514 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.514 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.514 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.514 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.514 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.514 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.514 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.514 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.514 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.514 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.514 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.514 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.514 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.514 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.514 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.515 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.515 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.515 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.515 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.515 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.515 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.515 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.515 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.515 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.515 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.515 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.515 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.515 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.515 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.515 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.516 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.516 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.516 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.516 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.516 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.516 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.516 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.516 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.516 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.516 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.516 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.516 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.516 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.516 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.516 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.517 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.517 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.517 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.517 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.517 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.517 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.517 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.517 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.517 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.517 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.517 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.517 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.517 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.517 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.517 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.517 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.518 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "osd_support", 2024-04-25T00:44:45.518 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.518 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.518 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.518 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.518 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.518 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.518 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.518 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.518 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.518 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.518 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.518 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.518 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.518 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.519 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.519 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.519 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.519 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.519 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.519 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.519 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.519 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.519 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.519 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.519 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.519 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.519 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.519 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.519 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.520 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.520 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.520 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.520 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.520 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.520 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.520 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.520 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.520 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.520 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.520 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.520 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.520 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.520 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.520 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.520 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.521 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.521 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.521 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.521 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.521 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.521 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.521 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.521 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.521 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.521 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.521 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.521 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.521 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.521 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.521 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.522 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.522 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.522 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.522 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.522 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.522 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.522 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.522 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.522 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.522 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.522 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.522 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.522 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.522 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.522 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.523 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.523 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "pg_autoscaler", 2024-04-25T00:44:45.523 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.523 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.523 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.523 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.523 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.523 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.523 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.523 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.523 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.523 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.523 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.523 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.523 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.524 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.524 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.524 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.524 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.524 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.524 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.524 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.524 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.524 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.524 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.524 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.524 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.524 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.524 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.524 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.525 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.525 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.525 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.525 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.525 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.525 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.525 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.525 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.525 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.525 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.525 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.525 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.525 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.525 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.525 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.526 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.526 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.526 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.526 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.526 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.526 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.526 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.526 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.526 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.526 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.526 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.526 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.526 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.526 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.526 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.527 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.527 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.527 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.527 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.527 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.527 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.527 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.527 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.527 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.527 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.527 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.527 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.527 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.527 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.527 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.528 INFO:teuthology.orchestra.run.smithi032.stdout: "sleep_interval": { 2024-04-25T00:44:45.528 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "sleep_interval", 2024-04-25T00:44:45.528 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.528 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.528 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.528 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "60", 2024-04-25T00:44:45.528 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.528 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.528 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.528 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.528 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.528 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.528 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.528 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.528 INFO:teuthology.orchestra.run.smithi032.stdout: "threshold": { 2024-04-25T00:44:45.529 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "threshold", 2024-04-25T00:44:45.529 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "float", 2024-04-25T00:44:45.529 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.529 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.529 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "3.0", 2024-04-25T00:44:45.529 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "1.0", 2024-04-25T00:44:45.529 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.529 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.529 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "scaling threshold", 2024-04-25T00:44:45.529 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0", 2024-04-25T00:44:45.529 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.529 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.529 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.529 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.529 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.530 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.530 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "progress", 2024-04-25T00:44:45.530 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.530 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.530 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.530 INFO:teuthology.orchestra.run.smithi032.stdout: "allow_pg_recovery_event": { 2024-04-25T00:44:45.530 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "allow_pg_recovery_event", 2024-04-25T00:44:45.530 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.530 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.530 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.530 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.530 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.530 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.530 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.531 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "allow the module to show pg recovery progress", 2024-04-25T00:44:45.531 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.531 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.531 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.531 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.531 INFO:teuthology.orchestra.run.smithi032.stdout: "enabled": { 2024-04-25T00:44:45.531 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "enabled", 2024-04-25T00:44:45.531 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.531 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.531 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.531 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.531 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.531 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.531 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.531 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.531 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.532 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.532 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.532 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.532 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.532 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.532 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.532 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.532 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.532 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.532 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.532 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.532 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.532 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.532 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.532 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.533 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.533 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.533 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.533 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.533 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.533 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.533 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.533 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.533 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.533 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.533 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.533 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.533 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.533 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.533 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.534 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.534 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.534 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.534 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.534 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.534 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.534 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.534 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.534 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.534 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.534 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.534 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.534 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.534 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.534 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.535 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.535 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.535 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.535 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.535 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.535 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.535 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.535 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.535 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.535 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.535 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.535 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.535 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.535 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.535 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.536 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.536 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.536 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.536 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.536 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.536 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.536 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.536 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.536 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.536 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.536 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.536 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.536 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.536 INFO:teuthology.orchestra.run.smithi032.stdout: "max_completed_events": { 2024-04-25T00:44:45.536 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "max_completed_events", 2024-04-25T00:44:45.537 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.537 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.537 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.537 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "50", 2024-04-25T00:44:45.537 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.537 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.537 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.537 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "number of past completed events to remember", 2024-04-25T00:44:45.537 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.537 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.537 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.537 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.537 INFO:teuthology.orchestra.run.smithi032.stdout: "sleep_interval": { 2024-04-25T00:44:45.537 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "sleep_interval", 2024-04-25T00:44:45.537 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.537 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.538 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.538 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "5", 2024-04-25T00:44:45.538 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.538 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.538 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.538 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "how long the module is going to sleep", 2024-04-25T00:44:45.538 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.538 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.538 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.538 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.538 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.538 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.538 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.538 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "prometheus", 2024-04-25T00:44:45.538 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.539 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.539 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.539 INFO:teuthology.orchestra.run.smithi032.stdout: "cache": { 2024-04-25T00:44:45.539 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "cache", 2024-04-25T00:44:45.539 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.539 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.539 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.539 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.539 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.539 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.539 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.539 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.539 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.539 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.540 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.540 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.540 INFO:teuthology.orchestra.run.smithi032.stdout: "exclude_perf_counters": { 2024-04-25T00:44:45.540 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "exclude_perf_counters", 2024-04-25T00:44:45.540 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.540 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.540 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.540 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.540 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.540 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.540 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.540 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Do not include perf-counters in the metrics output", 2024-04-25T00:44:45.540 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.", 2024-04-25T00:44:45.540 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.540 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.541 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.541 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.541 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.541 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.541 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.541 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.541 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.541 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.541 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.541 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.541 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.541 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.541 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.541 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.541 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.541 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.542 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.542 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.542 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.542 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.542 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.542 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.542 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.542 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.542 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.542 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.542 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.542 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.542 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.542 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.543 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.543 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.543 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.543 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.543 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.543 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.543 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.543 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.543 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.543 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.543 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.543 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.543 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.543 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.543 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.544 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.544 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.544 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.544 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.544 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.544 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.544 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.544 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.544 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.544 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.544 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.544 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.544 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.544 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.544 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.545 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.545 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.545 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.545 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.545 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.545 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.545 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.545 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.545 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.545 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.545 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.545 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_stats_pools": { 2024-04-25T00:44:45.545 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rbd_stats_pools", 2024-04-25T00:44:45.545 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.545 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.545 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.546 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.546 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.546 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.546 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.546 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.546 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.546 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.546 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.546 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.546 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_stats_pools_refresh_interval": { 2024-04-25T00:44:45.546 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rbd_stats_pools_refresh_interval", 2024-04-25T00:44:45.546 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.546 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.546 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.547 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "300", 2024-04-25T00:44:45.547 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.547 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.547 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.547 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.547 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.547 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.547 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.547 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.547 INFO:teuthology.orchestra.run.smithi032.stdout: "scrape_interval": { 2024-04-25T00:44:45.547 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "scrape_interval", 2024-04-25T00:44:45.547 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "float", 2024-04-25T00:44:45.547 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.547 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.547 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "15.0", 2024-04-25T00:44:45.548 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.548 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.548 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.548 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.548 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.548 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.548 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.548 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.548 INFO:teuthology.orchestra.run.smithi032.stdout: "server_addr": { 2024-04-25T00:44:45.548 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "server_addr", 2024-04-25T00:44:45.548 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.548 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.548 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.548 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "::", 2024-04-25T00:44:45.548 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.549 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.549 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.549 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "the IPv4 or IPv6 address on which the module listens for HTTP requests", 2024-04-25T00:44:45.549 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.549 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.549 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.549 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.549 INFO:teuthology.orchestra.run.smithi032.stdout: "server_port": { 2024-04-25T00:44:45.549 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "server_port", 2024-04-25T00:44:45.549 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.549 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.549 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.549 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "9283", 2024-04-25T00:44:45.549 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.549 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.549 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.550 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "the port on which the module listens for HTTP requests", 2024-04-25T00:44:45.550 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.550 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.550 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.550 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.550 INFO:teuthology.orchestra.run.smithi032.stdout: "stale_cache_strategy": { 2024-04-25T00:44:45.550 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "stale_cache_strategy", 2024-04-25T00:44:45.550 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.550 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.550 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.550 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "log", 2024-04-25T00:44:45.550 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.550 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.550 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.551 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.551 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.551 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.551 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.551 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.551 INFO:teuthology.orchestra.run.smithi032.stdout: "standby_behaviour": { 2024-04-25T00:44:45.551 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "standby_behaviour", 2024-04-25T00:44:45.551 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.551 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.551 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.551 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "default", 2024-04-25T00:44:45.551 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.551 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.551 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.551 INFO:teuthology.orchestra.run.smithi032.stdout: "default", 2024-04-25T00:44:45.552 INFO:teuthology.orchestra.run.smithi032.stdout: "error" 2024-04-25T00:44:45.552 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.552 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.552 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.552 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.552 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.552 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.552 INFO:teuthology.orchestra.run.smithi032.stdout: "standby_error_status_code": { 2024-04-25T00:44:45.552 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "standby_error_status_code", 2024-04-25T00:44:45.552 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.552 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.552 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.552 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "500", 2024-04-25T00:44:45.552 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "400", 2024-04-25T00:44:45.552 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "599", 2024-04-25T00:44:45.552 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.553 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.553 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.553 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.553 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.553 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.553 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.553 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.553 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.553 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rbd_support", 2024-04-25T00:44:45.553 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.553 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.553 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.553 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.553 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.553 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.554 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.554 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.554 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.554 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.554 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.554 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.554 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.554 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.554 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.554 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.554 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.554 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.554 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.554 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.555 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.555 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.555 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.555 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.555 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.555 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.555 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.555 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.555 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.555 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.555 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.555 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.555 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.555 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.556 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.556 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.556 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.556 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.556 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.556 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.556 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.556 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.556 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.556 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.556 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.556 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.556 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.556 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.556 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.556 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.557 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.557 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.557 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.557 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.557 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.557 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.557 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.557 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.557 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.557 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.557 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.557 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.557 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.557 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.558 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.558 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.558 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.558 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.558 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.558 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.558 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.558 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.558 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.558 INFO:teuthology.orchestra.run.smithi032.stdout: "max_concurrent_snap_create": { 2024-04-25T00:44:45.558 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "max_concurrent_snap_create", 2024-04-25T00:44:45.558 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.558 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.558 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.558 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10", 2024-04-25T00:44:45.559 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.559 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.559 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.559 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.559 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.559 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.559 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.559 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.559 INFO:teuthology.orchestra.run.smithi032.stdout: "mirror_snapshot_schedule": { 2024-04-25T00:44:45.559 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "mirror_snapshot_schedule", 2024-04-25T00:44:45.559 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.559 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.559 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.559 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.560 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.560 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.560 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.560 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.560 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.560 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.560 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.560 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.560 INFO:teuthology.orchestra.run.smithi032.stdout: "trash_purge_schedule": { 2024-04-25T00:44:45.560 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "trash_purge_schedule", 2024-04-25T00:44:45.560 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.560 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.560 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.560 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.560 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.561 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.561 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.561 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.561 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.561 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.561 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.561 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.561 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.561 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.561 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.561 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "restful", 2024-04-25T00:44:45.561 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.561 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.561 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.561 INFO:teuthology.orchestra.run.smithi032.stdout: "enable_auth": { 2024-04-25T00:44:45.562 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "enable_auth", 2024-04-25T00:44:45.562 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.562 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.562 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.562 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.562 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.562 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.562 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.562 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.562 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.562 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.562 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.562 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.562 INFO:teuthology.orchestra.run.smithi032.stdout: "key_file": { 2024-04-25T00:44:45.563 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "key_file", 2024-04-25T00:44:45.563 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.563 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.563 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.563 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.563 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.563 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.563 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.563 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.563 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.563 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.563 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.563 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.563 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.563 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.564 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.564 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.564 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.564 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.564 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.564 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.564 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.564 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.564 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.564 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.564 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.564 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.564 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.564 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.564 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.565 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.565 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.565 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.565 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.565 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.565 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.565 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.565 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.565 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.565 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.565 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.565 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.565 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.565 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.565 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.566 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.566 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.566 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.566 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.566 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.566 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.566 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.566 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.566 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.566 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.566 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.566 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.566 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.566 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.566 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.567 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.567 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.567 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.567 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.567 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.567 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.567 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.567 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.567 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.567 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.567 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.567 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.567 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.567 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.567 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.568 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.568 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.568 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.568 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.568 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.568 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.568 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.568 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.568 INFO:teuthology.orchestra.run.smithi032.stdout: "server_addr": { 2024-04-25T00:44:45.568 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "server_addr", 2024-04-25T00:44:45.568 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.568 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.568 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.568 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.569 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.569 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.569 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.569 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.569 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.569 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.569 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.569 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.569 INFO:teuthology.orchestra.run.smithi032.stdout: "server_port": { 2024-04-25T00:44:45.569 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "server_port", 2024-04-25T00:44:45.569 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.569 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.569 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.569 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.569 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.569 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.570 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.570 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.570 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.570 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.570 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.570 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.570 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.570 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.570 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.570 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rgw", 2024-04-25T00:44:45.570 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.570 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.570 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.570 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.571 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.571 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.571 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.571 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.571 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.571 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.571 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.571 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.571 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.571 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.571 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.571 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.571 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.571 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.571 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.571 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.572 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.572 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.572 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.572 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.572 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.572 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.572 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.572 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.572 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.572 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.572 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.572 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.572 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.572 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.572 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.573 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.573 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.573 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.573 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.573 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.573 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.573 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.573 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.573 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.573 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.573 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.573 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.573 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.573 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.573 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.573 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.574 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.574 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.574 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.574 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.574 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.574 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.574 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.574 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.574 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.574 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.574 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.574 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.574 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.574 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.574 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.575 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.575 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.575 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.575 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.575 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.575 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.575 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.575 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.575 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.575 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.575 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rook", 2024-04-25T00:44:45.575 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.575 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.575 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.576 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.576 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.576 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.576 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.576 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.576 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.576 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.576 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.576 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.576 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.576 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.576 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.576 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.576 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.579 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.579 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.579 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.579 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.579 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.579 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.579 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.579 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.579 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.579 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.579 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.579 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.579 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.579 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.580 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.580 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.580 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.580 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.580 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.580 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.580 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.580 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.580 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.580 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.580 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.580 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.580 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.580 INFO:teuthology.orchestra.run.smithi032.stdout: "storage_class": { 2024-04-25T00:44:45.581 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "storage_class", 2024-04-25T00:44:45.581 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.581 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.581 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.581 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "local", 2024-04-25T00:44:45.581 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.581 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.581 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.581 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "storage class name for LSO-discovered PVs", 2024-04-25T00:44:45.581 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.581 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.581 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.581 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.581 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.581 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.582 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.582 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "selftest", 2024-04-25T00:44:45.582 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.582 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.582 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.582 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.582 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.582 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.582 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.582 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.582 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.582 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.582 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.582 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.585 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.585 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.585 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.585 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.585 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.585 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.585 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.585 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.585 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.585 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.586 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.586 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.586 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.586 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.586 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.586 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.586 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.586 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.586 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.586 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.586 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.586 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.586 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.586 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.587 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.587 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.587 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.587 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.587 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.587 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.587 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.587 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.587 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.587 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.587 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.587 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.587 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.587 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.587 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.587 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.588 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.588 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.588 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.588 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.588 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.588 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.588 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.588 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.588 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.588 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.588 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.588 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.588 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.588 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.588 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.589 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.589 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.589 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.589 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.589 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.589 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.589 INFO:teuthology.orchestra.run.smithi032.stdout: "roption1": { 2024-04-25T00:44:45.589 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "roption1", 2024-04-25T00:44:45.589 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.589 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.589 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.589 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.589 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.589 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.589 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.590 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.590 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.590 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.590 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.590 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.590 INFO:teuthology.orchestra.run.smithi032.stdout: "roption2": { 2024-04-25T00:44:45.590 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "roption2", 2024-04-25T00:44:45.590 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.590 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.590 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.590 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "xyz", 2024-04-25T00:44:45.590 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.590 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.590 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.591 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.591 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.591 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.591 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.591 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.591 INFO:teuthology.orchestra.run.smithi032.stdout: "rwoption1": { 2024-04-25T00:44:45.591 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rwoption1", 2024-04-25T00:44:45.591 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.591 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.591 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.591 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.591 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.591 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.591 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.591 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.592 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.592 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.592 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.592 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.592 INFO:teuthology.orchestra.run.smithi032.stdout: "rwoption2": { 2024-04-25T00:44:45.592 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rwoption2", 2024-04-25T00:44:45.592 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.592 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.592 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.592 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.592 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.592 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.592 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.592 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.593 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.593 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.593 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.593 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.593 INFO:teuthology.orchestra.run.smithi032.stdout: "rwoption3": { 2024-04-25T00:44:45.593 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rwoption3", 2024-04-25T00:44:45.593 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "float", 2024-04-25T00:44:45.593 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.593 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.593 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.593 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.593 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.593 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.593 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.594 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.594 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.594 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.594 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.594 INFO:teuthology.orchestra.run.smithi032.stdout: "rwoption4": { 2024-04-25T00:44:45.594 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rwoption4", 2024-04-25T00:44:45.594 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.594 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.594 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.594 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.594 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.594 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.594 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.594 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.594 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.595 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.595 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.595 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.595 INFO:teuthology.orchestra.run.smithi032.stdout: "rwoption5": { 2024-04-25T00:44:45.595 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rwoption5", 2024-04-25T00:44:45.595 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.595 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.595 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.595 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.595 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.595 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.595 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.595 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.595 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.595 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.596 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.596 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.596 INFO:teuthology.orchestra.run.smithi032.stdout: "rwoption6": { 2024-04-25T00:44:45.596 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rwoption6", 2024-04-25T00:44:45.596 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.596 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.596 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.596 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.596 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.596 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.596 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.596 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.596 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.596 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.596 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.597 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.597 INFO:teuthology.orchestra.run.smithi032.stdout: "rwoption7": { 2024-04-25T00:44:45.597 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rwoption7", 2024-04-25T00:44:45.597 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.597 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.597 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.597 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.597 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "1", 2024-04-25T00:44:45.597 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "42", 2024-04-25T00:44:45.597 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.597 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.597 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.597 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.597 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.598 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.598 INFO:teuthology.orchestra.run.smithi032.stdout: "testkey": { 2024-04-25T00:44:45.598 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "testkey", 2024-04-25T00:44:45.598 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.598 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.598 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.598 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.598 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.598 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.598 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.598 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.598 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.598 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.598 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.598 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.599 INFO:teuthology.orchestra.run.smithi032.stdout: "testlkey": { 2024-04-25T00:44:45.599 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "testlkey", 2024-04-25T00:44:45.599 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.599 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.599 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.599 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.599 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.599 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.599 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.599 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.599 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.599 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.599 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.599 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.599 INFO:teuthology.orchestra.run.smithi032.stdout: "testnewline": { 2024-04-25T00:44:45.600 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "testnewline", 2024-04-25T00:44:45.600 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.600 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.600 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.600 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.600 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.600 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.600 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.600 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.600 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.600 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.600 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.600 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.600 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.600 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.601 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.601 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "snap_schedule", 2024-04-25T00:44:45.601 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.601 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.601 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.601 INFO:teuthology.orchestra.run.smithi032.stdout: "allow_m_granularity": { 2024-04-25T00:44:45.601 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "allow_m_granularity", 2024-04-25T00:44:45.601 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.601 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.601 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.601 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.601 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.601 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.601 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.602 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "allow minute scheduled snapshots", 2024-04-25T00:44:45.602 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.602 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.602 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.602 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.602 INFO:teuthology.orchestra.run.smithi032.stdout: "dump_on_update": { 2024-04-25T00:44:45.602 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "dump_on_update", 2024-04-25T00:44:45.602 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.602 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.602 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.602 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.602 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.602 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.602 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.602 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "dump database to debug log on update", 2024-04-25T00:44:45.603 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.603 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.603 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.603 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.603 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.603 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.603 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.603 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.603 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.603 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.603 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.603 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.603 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.603 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.603 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.604 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.604 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.604 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.604 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.604 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.604 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.604 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.604 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.604 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.604 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.604 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.604 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.604 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.605 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.605 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.605 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.605 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.605 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.605 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.605 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.605 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.605 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.605 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.605 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.605 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.605 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.605 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.606 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.606 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.606 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.606 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.606 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.606 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.606 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.606 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.606 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.606 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.606 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.606 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.606 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.606 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.606 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.607 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.607 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.607 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.607 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.607 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.607 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.607 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.607 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.607 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.607 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.607 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.607 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.607 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.607 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.607 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.608 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.608 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.608 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.608 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.608 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.608 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "stats", 2024-04-25T00:44:45.608 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.608 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.608 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.608 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.608 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.608 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.608 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.608 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.609 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.609 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.609 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.609 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.609 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.609 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.609 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.609 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.609 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.609 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.609 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.609 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.609 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.609 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.609 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.610 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.610 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.610 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.610 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.610 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.610 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.610 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.610 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.610 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.610 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.610 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.610 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.610 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.610 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.610 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.611 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.611 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.611 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.611 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.611 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.611 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.611 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.611 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.611 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.611 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.611 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.611 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.611 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.611 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.611 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.612 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.612 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.612 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.612 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.612 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.612 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.612 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.612 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.612 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.612 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.612 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.612 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.612 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.612 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.612 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.612 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.613 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.613 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.613 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.613 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.613 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.613 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.613 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.613 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "status", 2024-04-25T00:44:45.613 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.613 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.613 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.613 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.613 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.613 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.614 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.614 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.614 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.614 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.614 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.614 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.614 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.614 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.614 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.614 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.614 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.614 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.614 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.614 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.614 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.615 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.615 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.615 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.615 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.615 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.615 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.615 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.615 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.615 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.615 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.615 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.615 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.615 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.615 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.615 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.616 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.616 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.616 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.616 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.616 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.616 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.616 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.616 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.616 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.616 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.616 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.616 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.616 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.616 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.616 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.616 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.617 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.617 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.617 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.617 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.617 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.617 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.617 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.617 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.617 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.617 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.617 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.617 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.617 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.617 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.618 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.618 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.618 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.618 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.618 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.618 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.618 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.618 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.618 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.618 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.618 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "telegraf", 2024-04-25T00:44:45.618 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.618 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.618 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.618 INFO:teuthology.orchestra.run.smithi032.stdout: "address": { 2024-04-25T00:44:45.619 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "address", 2024-04-25T00:44:45.619 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.619 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.619 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.619 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "unixgram:///tmp/telegraf.sock", 2024-04-25T00:44:45.619 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.619 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.619 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.619 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.619 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.619 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.619 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.619 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.619 INFO:teuthology.orchestra.run.smithi032.stdout: "interval": { 2024-04-25T00:44:45.619 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "interval", 2024-04-25T00:44:45.620 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.620 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.620 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.620 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "15", 2024-04-25T00:44:45.620 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.620 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.620 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.620 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.620 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.620 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.620 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.620 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.620 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.620 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.620 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.621 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.621 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.621 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.621 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.621 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.621 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.621 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.621 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.621 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.621 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.621 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.621 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.621 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.621 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.621 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.622 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.622 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.622 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.622 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.622 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.622 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.622 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.622 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.622 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.622 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.622 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.622 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.622 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.622 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.623 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.623 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.623 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.623 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.623 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.623 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.623 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.623 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.623 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.623 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.623 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.623 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.623 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.623 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.623 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.624 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.624 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.624 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.624 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.624 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.624 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.624 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.624 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.624 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.624 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.624 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.624 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.624 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.624 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.624 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.625 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.625 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.625 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.625 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.625 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.625 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.625 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.625 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.625 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.625 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.625 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.625 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "telemetry", 2024-04-25T00:44:45.625 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.625 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.625 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.626 INFO:teuthology.orchestra.run.smithi032.stdout: "channel_basic": { 2024-04-25T00:44:45.626 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "channel_basic", 2024-04-25T00:44:45.626 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.626 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.626 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.626 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.626 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.626 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.626 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.626 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Share basic cluster information (size, version)", 2024-04-25T00:44:45.626 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.626 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.626 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.626 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.627 INFO:teuthology.orchestra.run.smithi032.stdout: "channel_crash": { 2024-04-25T00:44:45.627 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "channel_crash", 2024-04-25T00:44:45.627 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.627 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.627 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.627 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.627 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.627 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.627 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.627 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Share metadata about Ceph daemon crashes (version, stack straces, etc)", 2024-04-25T00:44:45.627 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.627 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.627 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.627 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.627 INFO:teuthology.orchestra.run.smithi032.stdout: "channel_device": { 2024-04-25T00:44:45.627 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "channel_device", 2024-04-25T00:44:45.628 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.628 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.628 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.628 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.628 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.628 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.628 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.628 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)", 2024-04-25T00:44:45.628 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.628 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.628 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.628 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.628 INFO:teuthology.orchestra.run.smithi032.stdout: "channel_ident": { 2024-04-25T00:44:45.628 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "channel_ident", 2024-04-25T00:44:45.629 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.629 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.629 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.629 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.629 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.629 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.629 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.629 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Share a user-provided description and/or contact email for the cluster", 2024-04-25T00:44:45.629 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.629 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.629 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.629 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.629 INFO:teuthology.orchestra.run.smithi032.stdout: "channel_perf": { 2024-04-25T00:44:45.629 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "channel_perf", 2024-04-25T00:44:45.629 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.630 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.630 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.630 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.630 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.630 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.630 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.630 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Share various performance metrics of a cluster", 2024-04-25T00:44:45.630 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.630 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.630 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.630 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.630 INFO:teuthology.orchestra.run.smithi032.stdout: "contact": { 2024-04-25T00:44:45.630 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "contact", 2024-04-25T00:44:45.630 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.631 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.631 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.631 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.631 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.631 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.631 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.631 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.631 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.631 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.631 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.631 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.631 INFO:teuthology.orchestra.run.smithi032.stdout: "description": { 2024-04-25T00:44:45.631 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "description", 2024-04-25T00:44:45.631 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.631 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.632 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.632 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.632 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.632 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.632 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.632 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.632 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.632 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.632 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.632 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.632 INFO:teuthology.orchestra.run.smithi032.stdout: "device_url": { 2024-04-25T00:44:45.632 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "device_url", 2024-04-25T00:44:45.632 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.632 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.632 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.632 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "https://telemetry.ceph.com/device", 2024-04-25T00:44:45.633 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.633 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.633 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.633 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.633 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.633 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.633 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.633 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.633 INFO:teuthology.orchestra.run.smithi032.stdout: "enabled": { 2024-04-25T00:44:45.633 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "enabled", 2024-04-25T00:44:45.633 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.633 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.633 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.633 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.634 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.634 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.634 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.634 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.634 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.634 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.634 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.634 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.634 INFO:teuthology.orchestra.run.smithi032.stdout: "interval": { 2024-04-25T00:44:45.634 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "interval", 2024-04-25T00:44:45.634 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.634 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.634 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.634 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "24", 2024-04-25T00:44:45.634 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "8", 2024-04-25T00:44:45.635 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.635 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.635 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.635 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.635 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.635 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.635 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.635 INFO:teuthology.orchestra.run.smithi032.stdout: "last_opt_revision": { 2024-04-25T00:44:45.635 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "last_opt_revision", 2024-04-25T00:44:45.635 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.635 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.635 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.635 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "1", 2024-04-25T00:44:45.635 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.635 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.635 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.636 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.636 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.636 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.636 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.636 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.636 INFO:teuthology.orchestra.run.smithi032.stdout: "leaderboard": { 2024-04-25T00:44:45.636 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "leaderboard", 2024-04-25T00:44:45.636 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.636 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.636 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.636 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.636 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.636 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.636 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.636 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.637 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.637 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.637 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.637 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.637 INFO:teuthology.orchestra.run.smithi032.stdout: "leaderboard_description": { 2024-04-25T00:44:45.637 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "leaderboard_description", 2024-04-25T00:44:45.637 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.637 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.637 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.637 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.637 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.637 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.637 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.637 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.638 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.638 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.638 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.638 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.638 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.638 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.638 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.638 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.638 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.638 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.638 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.638 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.638 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.638 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.638 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.639 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.639 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.639 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.639 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.639 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.639 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.639 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.639 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.639 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.639 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.639 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.639 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.639 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.639 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.639 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.640 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.640 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.640 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.640 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.640 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.640 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.640 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.640 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.640 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.640 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.640 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.640 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.640 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.640 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.640 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.641 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.641 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.641 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.641 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.641 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.641 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.641 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.641 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.641 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.641 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.641 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.641 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.641 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.641 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.641 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.642 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.642 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.642 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.642 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.642 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.642 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.642 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.642 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.642 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.642 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.642 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.642 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.642 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.642 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.643 INFO:teuthology.orchestra.run.smithi032.stdout: "organization": { 2024-04-25T00:44:45.643 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "organization", 2024-04-25T00:44:45.643 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.643 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.643 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.643 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.643 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.643 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.643 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.643 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.643 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.643 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.643 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.643 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.643 INFO:teuthology.orchestra.run.smithi032.stdout: "proxy": { 2024-04-25T00:44:45.644 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "proxy", 2024-04-25T00:44:45.644 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.644 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.644 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.644 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.644 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.644 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.644 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.644 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.644 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.644 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.644 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.644 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.644 INFO:teuthology.orchestra.run.smithi032.stdout: "url": { 2024-04-25T00:44:45.644 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "url", 2024-04-25T00:44:45.645 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.645 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.645 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.645 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "https://telemetry.ceph.com/report", 2024-04-25T00:44:45.645 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.645 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.645 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.645 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.645 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.645 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.645 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.645 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.645 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.645 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.645 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.645 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "test_orchestrator", 2024-04-25T00:44:45.646 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.646 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.646 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.646 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.646 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.646 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.646 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.646 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.646 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.646 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.646 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.646 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.646 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.646 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.647 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.647 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.647 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.647 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.647 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.647 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.647 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.647 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.647 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.647 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.647 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.647 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.647 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.647 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.647 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.648 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.648 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.648 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.648 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.648 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.648 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.648 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.648 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.648 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.648 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.648 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.648 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.648 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.648 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.648 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.649 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.649 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.649 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.649 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.649 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.649 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.649 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.649 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.649 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.649 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.649 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.649 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.649 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.649 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.649 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.650 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.650 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.650 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.650 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.650 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.650 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.650 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.650 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.650 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.650 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.650 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.650 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.650 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.650 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.650 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.651 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.651 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.651 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "volumes", 2024-04-25T00:44:45.651 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.651 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.651 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.651 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.651 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.651 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.651 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.651 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.651 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.651 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.651 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.651 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.652 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.652 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.652 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.652 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.652 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.652 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.652 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.652 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.652 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.652 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.652 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.652 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.652 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.652 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.653 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.653 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.653 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.653 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.653 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.653 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.653 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.653 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.653 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.653 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.653 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.653 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.653 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.653 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.653 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.654 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.654 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.654 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.654 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.654 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.654 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.654 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.654 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.654 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.654 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.654 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.654 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.654 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.654 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.655 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.655 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.655 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.655 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.655 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.655 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.655 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.655 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.655 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.655 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.655 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.655 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.655 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.655 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.655 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.656 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.656 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.656 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.656 INFO:teuthology.orchestra.run.smithi032.stdout: "max_concurrent_clones": { 2024-04-25T00:44:45.656 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "max_concurrent_clones", 2024-04-25T00:44:45.656 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.656 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.656 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.656 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "4", 2024-04-25T00:44:45.656 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.656 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.656 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.656 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Number of asynchronous cloner threads", 2024-04-25T00:44:45.656 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.657 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.657 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.657 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.657 INFO:teuthology.orchestra.run.smithi032.stdout: "periodic_async_work": { 2024-04-25T00:44:45.657 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "periodic_async_work", 2024-04-25T00:44:45.657 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.657 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.657 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.657 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.657 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.657 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.657 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.657 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Periodically check for async work", 2024-04-25T00:44:45.657 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.657 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.658 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.658 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.658 INFO:teuthology.orchestra.run.smithi032.stdout: "snapshot_clone_delay": { 2024-04-25T00:44:45.658 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "snapshot_clone_delay", 2024-04-25T00:44:45.658 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.658 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.658 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.658 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "0", 2024-04-25T00:44:45.658 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.658 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.658 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.658 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Delay clone begin operation by snapshot_clone_delay seconds", 2024-04-25T00:44:45.658 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.658 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.658 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.659 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.659 INFO:teuthology.orchestra.run.smithi032.stdout: "snapshot_clone_no_wait": { 2024-04-25T00:44:45.659 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "snapshot_clone_no_wait", 2024-04-25T00:44:45.659 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.659 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.659 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.659 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "True", 2024-04-25T00:44:45.659 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.659 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.659 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.659 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "Reject subvolume clone request when cloner threads are busy", 2024-04-25T00:44:45.659 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.659 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.659 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.659 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.660 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.660 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.660 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.660 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "zabbix", 2024-04-25T00:44:45.660 INFO:teuthology.orchestra.run.smithi032.stdout: "can_run": true, 2024-04-25T00:44:45.660 INFO:teuthology.orchestra.run.smithi032.stdout: "error_string": "", 2024-04-25T00:44:45.660 INFO:teuthology.orchestra.run.smithi032.stdout: "module_options": { 2024-04-25T00:44:45.660 INFO:teuthology.orchestra.run.smithi032.stdout: "discovery_interval": { 2024-04-25T00:44:45.660 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "discovery_interval", 2024-04-25T00:44:45.660 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "uint", 2024-04-25T00:44:45.660 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.660 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.660 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "100", 2024-04-25T00:44:45.660 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.660 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.661 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.661 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.661 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.661 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.661 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.661 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.661 INFO:teuthology.orchestra.run.smithi032.stdout: "identifier": { 2024-04-25T00:44:45.661 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "identifier", 2024-04-25T00:44:45.661 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.661 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.661 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.661 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.661 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.661 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.661 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.662 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.662 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.662 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.662 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.662 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.662 INFO:teuthology.orchestra.run.smithi032.stdout: "interval": { 2024-04-25T00:44:45.662 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "interval", 2024-04-25T00:44:45.662 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "secs", 2024-04-25T00:44:45.662 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.662 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.662 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "60", 2024-04-25T00:44:45.662 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.662 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.662 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.662 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.663 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.663 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.663 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.663 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.663 INFO:teuthology.orchestra.run.smithi032.stdout: "log_level": { 2024-04-25T00:44:45.663 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_level", 2024-04-25T00:44:45.663 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.663 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.663 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.663 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.663 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.663 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.663 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.663 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.663 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.663 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.664 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.664 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.664 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.664 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.664 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.664 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.664 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.664 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.664 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.664 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster": { 2024-04-25T00:44:45.664 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster", 2024-04-25T00:44:45.664 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.664 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.664 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.664 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.665 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.665 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.665 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.665 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.665 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.665 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.665 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.665 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.665 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_cluster_level": { 2024-04-25T00:44:45.665 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_cluster_level", 2024-04-25T00:44:45.665 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.665 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.665 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.665 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "info", 2024-04-25T00:44:45.665 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.666 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.666 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [ 2024-04-25T00:44:45.666 INFO:teuthology.orchestra.run.smithi032.stdout: "", 2024-04-25T00:44:45.666 INFO:teuthology.orchestra.run.smithi032.stdout: "critical", 2024-04-25T00:44:45.666 INFO:teuthology.orchestra.run.smithi032.stdout: "debug", 2024-04-25T00:44:45.666 INFO:teuthology.orchestra.run.smithi032.stdout: "error", 2024-04-25T00:44:45.666 INFO:teuthology.orchestra.run.smithi032.stdout: "info", 2024-04-25T00:44:45.666 INFO:teuthology.orchestra.run.smithi032.stdout: "warning" 2024-04-25T00:44:45.666 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.666 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.666 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.666 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.666 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.666 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.666 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": { 2024-04-25T00:44:45.667 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "log_to_file", 2024-04-25T00:44:45.667 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "bool", 2024-04-25T00:44:45.667 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.667 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 1, 2024-04-25T00:44:45.667 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "False", 2024-04-25T00:44:45.667 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.667 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.667 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.667 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.667 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.667 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.667 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.667 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.667 INFO:teuthology.orchestra.run.smithi032.stdout: "zabbix_host": { 2024-04-25T00:44:45.667 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "zabbix_host", 2024-04-25T00:44:45.667 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.668 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.668 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.668 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "", 2024-04-25T00:44:45.668 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.668 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.668 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.668 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.668 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.668 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.668 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.668 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.668 INFO:teuthology.orchestra.run.smithi032.stdout: "zabbix_port": { 2024-04-25T00:44:45.668 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "zabbix_port", 2024-04-25T00:44:45.668 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "int", 2024-04-25T00:44:45.668 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.669 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.669 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "10051", 2024-04-25T00:44:45.669 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.669 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.669 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.669 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.669 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.669 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.669 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.669 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.669 INFO:teuthology.orchestra.run.smithi032.stdout: "zabbix_sender": { 2024-04-25T00:44:45.669 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "zabbix_sender", 2024-04-25T00:44:45.669 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "str", 2024-04-25T00:44:45.669 INFO:teuthology.orchestra.run.smithi032.stdout: "level": "advanced", 2024-04-25T00:44:45.669 INFO:teuthology.orchestra.run.smithi032.stdout: "flags": 0, 2024-04-25T00:44:45.670 INFO:teuthology.orchestra.run.smithi032.stdout: "default_value": "/usr/bin/zabbix_sender", 2024-04-25T00:44:45.670 INFO:teuthology.orchestra.run.smithi032.stdout: "min": "", 2024-04-25T00:44:45.670 INFO:teuthology.orchestra.run.smithi032.stdout: "max": "", 2024-04-25T00:44:45.670 INFO:teuthology.orchestra.run.smithi032.stdout: "enum_allowed": [], 2024-04-25T00:44:45.670 INFO:teuthology.orchestra.run.smithi032.stdout: "desc": "", 2024-04-25T00:44:45.670 INFO:teuthology.orchestra.run.smithi032.stdout: "long_desc": "", 2024-04-25T00:44:45.670 INFO:teuthology.orchestra.run.smithi032.stdout: "tags": [], 2024-04-25T00:44:45.670 INFO:teuthology.orchestra.run.smithi032.stdout: "see_also": [] 2024-04-25T00:44:45.670 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.670 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.670 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.670 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.670 INFO:teuthology.orchestra.run.smithi032.stdout: "services": { 2024-04-25T00:44:45.670 INFO:teuthology.orchestra.run.smithi032.stdout: "dashboard": "https://172.21.15.32:8443/" 2024-04-25T00:44:45.670 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.671 INFO:teuthology.orchestra.run.smithi032.stdout: "always_on_modules": { 2024-04-25T00:44:45.671 INFO:teuthology.orchestra.run.smithi032.stdout: "octopus": [ 2024-04-25T00:44:45.671 INFO:teuthology.orchestra.run.smithi032.stdout: "balancer", 2024-04-25T00:44:45.671 INFO:teuthology.orchestra.run.smithi032.stdout: "crash", 2024-04-25T00:44:45.671 INFO:teuthology.orchestra.run.smithi032.stdout: "devicehealth", 2024-04-25T00:44:45.671 INFO:teuthology.orchestra.run.smithi032.stdout: "orchestrator", 2024-04-25T00:44:45.671 INFO:teuthology.orchestra.run.smithi032.stdout: "pg_autoscaler", 2024-04-25T00:44:45.671 INFO:teuthology.orchestra.run.smithi032.stdout: "progress", 2024-04-25T00:44:45.671 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_support", 2024-04-25T00:44:45.671 INFO:teuthology.orchestra.run.smithi032.stdout: "status", 2024-04-25T00:44:45.671 INFO:teuthology.orchestra.run.smithi032.stdout: "telemetry", 2024-04-25T00:44:45.671 INFO:teuthology.orchestra.run.smithi032.stdout: "volumes" 2024-04-25T00:44:45.671 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.671 INFO:teuthology.orchestra.run.smithi032.stdout: "pacific": [ 2024-04-25T00:44:45.671 INFO:teuthology.orchestra.run.smithi032.stdout: "balancer", 2024-04-25T00:44:45.672 INFO:teuthology.orchestra.run.smithi032.stdout: "crash", 2024-04-25T00:44:45.672 INFO:teuthology.orchestra.run.smithi032.stdout: "devicehealth", 2024-04-25T00:44:45.672 INFO:teuthology.orchestra.run.smithi032.stdout: "orchestrator", 2024-04-25T00:44:45.672 INFO:teuthology.orchestra.run.smithi032.stdout: "pg_autoscaler", 2024-04-25T00:44:45.672 INFO:teuthology.orchestra.run.smithi032.stdout: "progress", 2024-04-25T00:44:45.672 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_support", 2024-04-25T00:44:45.672 INFO:teuthology.orchestra.run.smithi032.stdout: "status", 2024-04-25T00:44:45.672 INFO:teuthology.orchestra.run.smithi032.stdout: "telemetry", 2024-04-25T00:44:45.672 INFO:teuthology.orchestra.run.smithi032.stdout: "volumes" 2024-04-25T00:44:45.672 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.672 INFO:teuthology.orchestra.run.smithi032.stdout: "quincy": [ 2024-04-25T00:44:45.672 INFO:teuthology.orchestra.run.smithi032.stdout: "balancer", 2024-04-25T00:44:45.672 INFO:teuthology.orchestra.run.smithi032.stdout: "crash", 2024-04-25T00:44:45.672 INFO:teuthology.orchestra.run.smithi032.stdout: "devicehealth", 2024-04-25T00:44:45.672 INFO:teuthology.orchestra.run.smithi032.stdout: "orchestrator", 2024-04-25T00:44:45.673 INFO:teuthology.orchestra.run.smithi032.stdout: "pg_autoscaler", 2024-04-25T00:44:45.673 INFO:teuthology.orchestra.run.smithi032.stdout: "progress", 2024-04-25T00:44:45.673 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_support", 2024-04-25T00:44:45.673 INFO:teuthology.orchestra.run.smithi032.stdout: "status", 2024-04-25T00:44:45.673 INFO:teuthology.orchestra.run.smithi032.stdout: "telemetry", 2024-04-25T00:44:45.673 INFO:teuthology.orchestra.run.smithi032.stdout: "volumes" 2024-04-25T00:44:45.673 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.673 INFO:teuthology.orchestra.run.smithi032.stdout: "reef": [ 2024-04-25T00:44:45.673 INFO:teuthology.orchestra.run.smithi032.stdout: "balancer", 2024-04-25T00:44:45.673 INFO:teuthology.orchestra.run.smithi032.stdout: "crash", 2024-04-25T00:44:45.673 INFO:teuthology.orchestra.run.smithi032.stdout: "devicehealth", 2024-04-25T00:44:45.673 INFO:teuthology.orchestra.run.smithi032.stdout: "orchestrator", 2024-04-25T00:44:45.673 INFO:teuthology.orchestra.run.smithi032.stdout: "pg_autoscaler", 2024-04-25T00:44:45.673 INFO:teuthology.orchestra.run.smithi032.stdout: "progress", 2024-04-25T00:44:45.674 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_support", 2024-04-25T00:44:45.674 INFO:teuthology.orchestra.run.smithi032.stdout: "status", 2024-04-25T00:44:45.674 INFO:teuthology.orchestra.run.smithi032.stdout: "telemetry", 2024-04-25T00:44:45.674 INFO:teuthology.orchestra.run.smithi032.stdout: "volumes" 2024-04-25T00:44:45.674 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-04-25T00:44:45.674 INFO:teuthology.orchestra.run.smithi032.stdout: "squid": [ 2024-04-25T00:44:45.674 INFO:teuthology.orchestra.run.smithi032.stdout: "balancer", 2024-04-25T00:44:45.674 INFO:teuthology.orchestra.run.smithi032.stdout: "crash", 2024-04-25T00:44:45.674 INFO:teuthology.orchestra.run.smithi032.stdout: "devicehealth", 2024-04-25T00:44:45.674 INFO:teuthology.orchestra.run.smithi032.stdout: "orchestrator", 2024-04-25T00:44:45.674 INFO:teuthology.orchestra.run.smithi032.stdout: "pg_autoscaler", 2024-04-25T00:44:45.674 INFO:teuthology.orchestra.run.smithi032.stdout: "progress", 2024-04-25T00:44:45.674 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_support", 2024-04-25T00:44:45.674 INFO:teuthology.orchestra.run.smithi032.stdout: "status", 2024-04-25T00:44:45.675 INFO:teuthology.orchestra.run.smithi032.stdout: "telemetry", 2024-04-25T00:44:45.675 INFO:teuthology.orchestra.run.smithi032.stdout: "volumes" 2024-04-25T00:44:45.675 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-04-25T00:44:45.675 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.675 INFO:teuthology.orchestra.run.smithi032.stdout: "last_failure_osd_epoch": 94, 2024-04-25T00:44:45.675 INFO:teuthology.orchestra.run.smithi032.stdout: "active_clients": [ 2024-04-25T00:44:45.675 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.675 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "devicehealth", 2024-04-25T00:44:45.675 INFO:teuthology.orchestra.run.smithi032.stdout: "addrvec": [ 2024-04-25T00:44:45.675 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.675 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "v2", 2024-04-25T00:44:45.675 INFO:teuthology.orchestra.run.smithi032.stdout: "addr": "172.21.15.32:0", 2024-04-25T00:44:45.675 INFO:teuthology.orchestra.run.smithi032.stdout: "nonce": 140245599 2024-04-25T00:44:45.675 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.676 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-04-25T00:44:45.676 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.676 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.676 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "libcephsqlite", 2024-04-25T00:44:45.676 INFO:teuthology.orchestra.run.smithi032.stdout: "addrvec": [ 2024-04-25T00:44:45.676 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.676 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "v2", 2024-04-25T00:44:45.676 INFO:teuthology.orchestra.run.smithi032.stdout: "addr": "172.21.15.32:0", 2024-04-25T00:44:45.676 INFO:teuthology.orchestra.run.smithi032.stdout: "nonce": 3124481284 2024-04-25T00:44:45.676 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.676 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-04-25T00:44:45.676 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.676 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.676 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "rbd_support", 2024-04-25T00:44:45.676 INFO:teuthology.orchestra.run.smithi032.stdout: "addrvec": [ 2024-04-25T00:44:45.677 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.677 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "v2", 2024-04-25T00:44:45.677 INFO:teuthology.orchestra.run.smithi032.stdout: "addr": "172.21.15.32:0", 2024-04-25T00:44:45.677 INFO:teuthology.orchestra.run.smithi032.stdout: "nonce": 3848513877 2024-04-25T00:44:45.677 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.677 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-04-25T00:44:45.677 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.677 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.677 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "snap_schedule", 2024-04-25T00:44:45.677 INFO:teuthology.orchestra.run.smithi032.stdout: "addrvec": [ 2024-04-25T00:44:45.677 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.677 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "v2", 2024-04-25T00:44:45.677 INFO:teuthology.orchestra.run.smithi032.stdout: "addr": "172.21.15.32:0", 2024-04-25T00:44:45.677 INFO:teuthology.orchestra.run.smithi032.stdout: "nonce": 2179386182 2024-04-25T00:44:45.677 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.678 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-04-25T00:44:45.678 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.678 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.678 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "snap_schedule", 2024-04-25T00:44:45.678 INFO:teuthology.orchestra.run.smithi032.stdout: "addrvec": [ 2024-04-25T00:44:45.678 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.678 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "v2", 2024-04-25T00:44:45.678 INFO:teuthology.orchestra.run.smithi032.stdout: "addr": "172.21.15.32:0", 2024-04-25T00:44:45.678 INFO:teuthology.orchestra.run.smithi032.stdout: "nonce": 4004246273 2024-04-25T00:44:45.678 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.678 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-04-25T00:44:45.678 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:45.678 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.678 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "volumes", 2024-04-25T00:44:45.678 INFO:teuthology.orchestra.run.smithi032.stdout: "addrvec": [ 2024-04-25T00:44:45.679 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-04-25T00:44:45.679 INFO:teuthology.orchestra.run.smithi032.stdout: "type": "v2", 2024-04-25T00:44:45.679 INFO:teuthology.orchestra.run.smithi032.stdout: "addr": "172.21.15.32:0", 2024-04-25T00:44:45.679 INFO:teuthology.orchestra.run.smithi032.stdout: "nonce": 1370317093 2024-04-25T00:44:45.679 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.679 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-04-25T00:44:45.679 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:45.679 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-04-25T00:44:45.679 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:44:45.680 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:44 smithi130 ceph-mon[32525]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:44:45.681 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:44 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:44:45.681 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:44 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:44:45.681 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:44 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:44:45.681 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:44 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:44:45.681 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:44 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2786270928' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:44:45.681 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:44 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:44:45.681 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:44 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:44:45.681 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:44 smithi121 ceph-mon[32231]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:44:45.681 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:44 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:44:45.681 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:44 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:44:45.681 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:44 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:44:45.681 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:44 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:44:45.681 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:44 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2786270928' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:44:45.681 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:44 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:44:45.682 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:44 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:44:45.682 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:44.431+0000 7efc4029a640 1 -- 172.21.15.32:0/4097369147 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7efc240a2f10 msgr2=0x7efc240a53d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:45.682 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:44.431+0000 7efc4029a640 1 --2- 172.21.15.32:0/4097369147 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7efc240a2f10 0x7efc240a53d0 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7efc381162a0 tx=0x7efc3400b390 comp rx=0 tx=0).stop 2024-04-25T00:44:45.682 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:44.431+0000 7efc4029a640 1 -- 172.21.15.32:0/4097369147 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efc38108970 msgr2=0x7efc381a6520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:44:45.682 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:44.431+0000 7efc4029a640 1 --2- 172.21.15.32:0/4097369147 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efc38108970 0x7efc381a6520 secure :-1 s=READY pgs=657 cs=0 l=1 rev1=1 crypto rx=0x7efc3000ba50 tx=0x7efc3000bf10 comp rx=0 tx=0).stop 2024-04-25T00:44:45.682 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:44.432+0000 7efc4029a640 1 -- 172.21.15.32:0/4097369147 shutdown_connections 2024-04-25T00:44:45.682 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:44.432+0000 7efc4029a640 1 --2- 172.21.15.32:0/4097369147 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7efc240a2f10 0x7efc240a53d0 unknown :-1 s=CLOSED pgs=111 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:45.682 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:44.432+0000 7efc4029a640 1 --2- 172.21.15.32:0/4097369147 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efc38114890 0x7efc381a0670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:45.683 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:44.432+0000 7efc4029a640 1 --2- 172.21.15.32:0/4097369147 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efc38109290 0x7efc381a6a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:45.683 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:44.432+0000 7efc4029a640 1 --2- 172.21.15.32:0/4097369147 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efc38108970 0x7efc381a6520 unknown :-1 s=CLOSED pgs=657 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:44:45.683 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:44.432+0000 7efc4029a640 1 -- 172.21.15.32:0/4097369147 >> 172.21.15.32:0/4097369147 conn(0x7efc3806c860 msgr2=0x7efc380717a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:44:45.683 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:44.432+0000 7efc4029a640 1 -- 172.21.15.32:0/4097369147 shutdown_connections 2024-04-25T00:44:45.683 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:44:44.432+0000 7efc4029a640 1 -- 172.21.15.32:0/4097369147 wait complete. 2024-04-25T00:44:45.689 DEBUG:tasks.check_counter:Getting stats from a 2024-04-25T00:44:45.689 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph daemon mds.a perf dump 2024-04-25T00:44:45.689 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:44 smithi032 ceph-mon[29357]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-25T00:44:45.689 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:44 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:44:45.689 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:44 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:44:45.690 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:44 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:44:45.690 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:44 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:44:45.690 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:44 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2786270928' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:44:45.690 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:44 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:44:45.690 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:44 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:44:45.727 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:45 smithi032 ceph-mon[29357]: pgmap v357: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 22 KiB/s rd, 17 KiB/s wr, 15 op/s 2024-04-25T00:44:45.727 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:45 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/4097369147' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json-pretty"}]: dispatch 2024-04-25T00:44:45.727 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:45 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:44:45.727 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:45 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:44:45.727 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:45 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:44:45.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:45 smithi121 ceph-mon[32231]: pgmap v357: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 22 KiB/s rd, 17 KiB/s wr, 15 op/s 2024-04-25T00:44:45.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:45 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/4097369147' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json-pretty"}]: dispatch 2024-04-25T00:44:45.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:45 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:44:45.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:45 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:44:45.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:45 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:44:45.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:45 smithi130 ceph-mon[32525]: pgmap v357: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 22 KiB/s rd, 17 KiB/s wr, 15 op/s 2024-04-25T00:44:45.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:45 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/4097369147' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json-pretty"}]: dispatch 2024-04-25T00:44:45.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:45 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T00:44:45.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:45 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T00:44:45.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:45 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' 2024-04-25T00:44:46.136 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:44:47.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:47 smithi032 ceph-mon[29357]: pgmap v358: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 26 KiB/s rd, 19 KiB/s wr, 137 op/s 2024-04-25T00:44:47.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:47 smithi121 ceph-mon[32231]: pgmap v358: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 26 KiB/s rd, 19 KiB/s wr, 137 op/s 2024-04-25T00:44:47.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:47 smithi130 ceph-mon[32525]: pgmap v358: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 26 KiB/s rd, 19 KiB/s wr, 137 op/s 2024-04-25T00:44:49.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:49 smithi032 ceph-mon[29357]: pgmap v359: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 31 KiB/s rd, 26 KiB/s wr, 167 op/s 2024-04-25T00:44:49.795 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:44:49.796 INFO:teuthology.orchestra.run.smithi032.stdout: "AsyncMessenger::Worker-0": { 2024-04-25T00:44:49.796 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_messages": 239, 2024-04-25T00:44:49.796 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_messages": 228, 2024-04-25T00:44:49.796 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_bytes": 1585440, 2024-04-25T00:44:49.796 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_bytes": 140224, 2024-04-25T00:44:49.796 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_created_connections": 4, 2024-04-25T00:44:49.796 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_active_connections": 0, 2024-04-25T00:44:49.796 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_total_time": 0.063069681, 2024-04-25T00:44:49.796 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_send_time": 0.017812040, 2024-04-25T00:44:49.796 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_recv_time": 0.048009962, 2024-04-25T00:44:49.796 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_fast_dispatch_time": 0.006150558, 2024-04-25T00:44:49.796 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:44:49.796 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 227, 2024-04-25T00:44:49.797 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.018510130, 2024-04-25T00:44:49.797 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000081542 2024-04-25T00:44:49.797 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:49.797 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:44:49.797 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 0, 2024-04-25T00:44:49.797 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.000000000, 2024-04-25T00:44:49.797 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000000000 2024-04-25T00:44:49.797 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:49.797 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_encrypted_bytes": 1585440, 2024-04-25T00:44:49.797 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_encrypted_bytes": 140224 2024-04-25T00:44:49.797 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:49.797 INFO:teuthology.orchestra.run.smithi032.stdout: "AsyncMessenger::Worker-1": { 2024-04-25T00:44:49.797 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_messages": 2, 2024-04-25T00:44:49.797 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_messages": 175, 2024-04-25T00:44:49.797 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_bytes": 320, 2024-04-25T00:44:49.797 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_bytes": 186080, 2024-04-25T00:44:49.798 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_created_connections": 5, 2024-04-25T00:44:49.798 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_active_connections": 3, 2024-04-25T00:44:49.798 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_total_time": 0.015861960, 2024-04-25T00:44:49.798 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_send_time": 0.010722317, 2024-04-25T00:44:49.798 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_recv_time": 0.002127489, 2024-04-25T00:44:49.798 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-04-25T00:44:49.798 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:44:49.798 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 172, 2024-04-25T00:44:49.798 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.023896880, 2024-04-25T00:44:49.798 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000138935 2024-04-25T00:44:49.798 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:49.798 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:44:49.798 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 0, 2024-04-25T00:44:49.799 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.000000000, 2024-04-25T00:44:49.799 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000000000 2024-04-25T00:44:49.799 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:49.799 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_encrypted_bytes": 320, 2024-04-25T00:44:49.799 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_encrypted_bytes": 186080 2024-04-25T00:44:49.799 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:49.799 INFO:teuthology.orchestra.run.smithi032.stdout: "AsyncMessenger::Worker-2": { 2024-04-25T00:44:49.799 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_messages": 4, 2024-04-25T00:44:49.799 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_messages": 4, 2024-04-25T00:44:49.799 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_bytes": 2992, 2024-04-25T00:44:49.799 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_bytes": 448, 2024-04-25T00:44:49.799 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_created_connections": 2, 2024-04-25T00:44:49.799 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_active_connections": 1, 2024-04-25T00:44:49.799 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_total_time": 0.000856779, 2024-04-25T00:44:49.799 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_send_time": 0.000048854, 2024-04-25T00:44:49.800 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_recv_time": 0.000806205, 2024-04-25T00:44:49.800 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-04-25T00:44:49.800 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:44:49.800 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 3, 2024-04-25T00:44:49.800 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.000769705, 2024-04-25T00:44:49.800 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000256568 2024-04-25T00:44:49.800 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:49.800 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:44:49.800 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 0, 2024-04-25T00:44:49.800 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.000000000, 2024-04-25T00:44:49.800 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000000000 2024-04-25T00:44:49.800 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:49.800 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_encrypted_bytes": 2992, 2024-04-25T00:44:49.801 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_encrypted_bytes": 448 2024-04-25T00:44:49.801 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:49.801 INFO:teuthology.orchestra.run.smithi032.stdout: "cct": { 2024-04-25T00:44:49.801 INFO:teuthology.orchestra.run.smithi032.stdout: "total_workers": 0, 2024-04-25T00:44:49.801 INFO:teuthology.orchestra.run.smithi032.stdout: "unhealthy_workers": 0 2024-04-25T00:44:49.801 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:49.801 INFO:teuthology.orchestra.run.smithi032.stdout: "mempool": { 2024-04-25T00:44:49.801 INFO:teuthology.orchestra.run.smithi032.stdout: "bloom_filter_bytes": 0, 2024-04-25T00:44:49.801 INFO:teuthology.orchestra.run.smithi032.stdout: "bloom_filter_items": 0, 2024-04-25T00:44:49.801 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_alloc_bytes": 0, 2024-04-25T00:44:49.801 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_alloc_items": 0, 2024-04-25T00:44:49.801 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_data_bytes": 0, 2024-04-25T00:44:49.801 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_data_items": 0, 2024-04-25T00:44:49.801 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_onode_bytes": 0, 2024-04-25T00:44:49.801 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_onode_items": 0, 2024-04-25T00:44:49.802 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_meta_bytes": 0, 2024-04-25T00:44:49.802 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_meta_items": 0, 2024-04-25T00:44:49.802 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_other_bytes": 0, 2024-04-25T00:44:49.802 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_other_items": 0, 2024-04-25T00:44:49.802 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_buffer_bytes": 0, 2024-04-25T00:44:49.802 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_buffer_items": 0, 2024-04-25T00:44:49.802 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_extent_bytes": 0, 2024-04-25T00:44:49.802 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_extent_items": 0, 2024-04-25T00:44:49.802 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_blob_bytes": 0, 2024-04-25T00:44:49.802 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_blob_items": 0, 2024-04-25T00:44:49.802 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_shared_blob_bytes": 0, 2024-04-25T00:44:49.802 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_shared_blob_items": 0, 2024-04-25T00:44:49.802 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_inline_bl_bytes": 0, 2024-04-25T00:44:49.802 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_inline_bl_items": 0, 2024-04-25T00:44:49.803 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_fsck_bytes": 0, 2024-04-25T00:44:49.803 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_fsck_items": 0, 2024-04-25T00:44:49.803 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_txc_bytes": 0, 2024-04-25T00:44:49.803 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_txc_items": 0, 2024-04-25T00:44:49.803 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_writing_deferred_bytes": 0, 2024-04-25T00:44:49.803 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_writing_deferred_items": 0, 2024-04-25T00:44:49.803 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_writing_bytes": 0, 2024-04-25T00:44:49.803 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_writing_items": 0, 2024-04-25T00:44:49.803 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_bytes": 0, 2024-04-25T00:44:49.803 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_items": 0, 2024-04-25T00:44:49.803 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_file_reader_bytes": 0, 2024-04-25T00:44:49.803 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_file_reader_items": 0, 2024-04-25T00:44:49.803 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_file_writer_bytes": 0, 2024-04-25T00:44:49.803 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_file_writer_items": 0, 2024-04-25T00:44:49.803 INFO:teuthology.orchestra.run.smithi032.stdout: "buffer_anon_bytes": 149057, 2024-04-25T00:44:49.804 INFO:teuthology.orchestra.run.smithi032.stdout: "buffer_anon_items": 32, 2024-04-25T00:44:49.804 INFO:teuthology.orchestra.run.smithi032.stdout: "buffer_meta_bytes": 0, 2024-04-25T00:44:49.804 INFO:teuthology.orchestra.run.smithi032.stdout: "buffer_meta_items": 0, 2024-04-25T00:44:49.804 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_bytes": 0, 2024-04-25T00:44:49.804 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_items": 0, 2024-04-25T00:44:49.804 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mapbl_bytes": 0, 2024-04-25T00:44:49.804 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mapbl_items": 0, 2024-04-25T00:44:49.804 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pglog_bytes": 0, 2024-04-25T00:44:49.804 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pglog_items": 0, 2024-04-25T00:44:49.804 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_bytes": 0, 2024-04-25T00:44:49.804 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_items": 0, 2024-04-25T00:44:49.804 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_mapping_bytes": 0, 2024-04-25T00:44:49.804 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_mapping_items": 0, 2024-04-25T00:44:49.804 INFO:teuthology.orchestra.run.smithi032.stdout: "pgmap_bytes": 0, 2024-04-25T00:44:49.804 INFO:teuthology.orchestra.run.smithi032.stdout: "pgmap_items": 0, 2024-04-25T00:44:49.805 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_co_bytes": 928, 2024-04-25T00:44:49.805 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_co_items": 2, 2024-04-25T00:44:49.805 INFO:teuthology.orchestra.run.smithi032.stdout: "unittest_1_bytes": 0, 2024-04-25T00:44:49.805 INFO:teuthology.orchestra.run.smithi032.stdout: "unittest_1_items": 0, 2024-04-25T00:44:49.805 INFO:teuthology.orchestra.run.smithi032.stdout: "unittest_2_bytes": 0, 2024-04-25T00:44:49.805 INFO:teuthology.orchestra.run.smithi032.stdout: "unittest_2_items": 0 2024-04-25T00:44:49.805 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:49.805 INFO:teuthology.orchestra.run.smithi032.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-04-25T00:44:49.805 INFO:teuthology.orchestra.run.smithi032.stdout: "val": 0, 2024-04-25T00:44:49.805 INFO:teuthology.orchestra.run.smithi032.stdout: "max": 104857600, 2024-04-25T00:44:49.805 INFO:teuthology.orchestra.run.smithi032.stdout: "get_started": 0, 2024-04-25T00:44:49.805 INFO:teuthology.orchestra.run.smithi032.stdout: "get": 241, 2024-04-25T00:44:49.805 INFO:teuthology.orchestra.run.smithi032.stdout: "get_sum": 1551730, 2024-04-25T00:44:49.805 INFO:teuthology.orchestra.run.smithi032.stdout: "get_or_fail_fail": 0, 2024-04-25T00:44:49.805 INFO:teuthology.orchestra.run.smithi032.stdout: "get_or_fail_success": 241, 2024-04-25T00:44:49.805 INFO:teuthology.orchestra.run.smithi032.stdout: "take": 0, 2024-04-25T00:44:49.806 INFO:teuthology.orchestra.run.smithi032.stdout: "take_sum": 0, 2024-04-25T00:44:49.806 INFO:teuthology.orchestra.run.smithi032.stdout: "put": 241, 2024-04-25T00:44:49.806 INFO:teuthology.orchestra.run.smithi032.stdout: "put_sum": 1551730, 2024-04-25T00:44:49.806 INFO:teuthology.orchestra.run.smithi032.stdout: "wait": { 2024-04-25T00:44:49.806 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 0, 2024-04-25T00:44:49.806 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.000000000, 2024-04-25T00:44:49.806 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000000000 2024-04-25T00:44:49.806 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:49.806 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:49.806 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:44:49.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:49 smithi121 ceph-mon[32231]: pgmap v359: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 31 KiB/s rd, 26 KiB/s wr, 167 op/s 2024-04-25T00:44:49.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:49 smithi130 ceph-mon[32525]: pgmap v359: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 31 KiB/s rd, 26 KiB/s wr, 167 op/s 2024-04-25T00:44:51.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:51 smithi032 ceph-mon[29357]: pgmap v360: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 33 KiB/s rd, 27 KiB/s wr, 206 op/s 2024-04-25T00:44:51.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:51 smithi121 ceph-mon[32231]: pgmap v360: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 33 KiB/s rd, 27 KiB/s wr, 206 op/s 2024-04-25T00:44:51.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:51 smithi130 ceph-mon[32525]: pgmap v360: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 33 KiB/s rd, 27 KiB/s wr, 206 op/s 2024-04-25T00:44:53.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:53 smithi032 ceph-mon[29357]: pgmap v361: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 29 KiB/s rd, 27 KiB/s wr, 228 op/s 2024-04-25T00:44:53.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:53 smithi121 ceph-mon[32231]: pgmap v361: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 29 KiB/s rd, 27 KiB/s wr, 228 op/s 2024-04-25T00:44:53.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:53 smithi130 ceph-mon[32525]: pgmap v361: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 29 KiB/s rd, 27 KiB/s wr, 228 op/s 2024-04-25T00:44:54.402 WARNING:tasks.check_counter:Counter 'mds.exported' not found on daemon mds.a 2024-04-25T00:44:54.403 WARNING:tasks.check_counter:Counter 'mds.imported' not found on daemon mds.a 2024-04-25T00:44:54.403 WARNING:tasks.check_counter:Counter 'mds.root_rsnaps' not found on daemon mds.a 2024-04-25T00:44:54.403 WARNING:tasks.check_counter:Counter 'mds_server.req_mksnap_latency.avgcount' not found on daemon mds.a 2024-04-25T00:44:54.403 WARNING:tasks.check_counter:Counter 'mds_server.req_rmsnap_latency.avgcount' not found on daemon mds.a 2024-04-25T00:44:54.403 DEBUG:tasks.check_counter:Getting stats from d 2024-04-25T00:44:54.403 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph daemon mds.d perf dump 2024-04-25T00:44:54.701 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:44:55.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:55 smithi032 ceph-mon[29357]: pgmap v362: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 28 KiB/s rd, 27 KiB/s wr, 225 op/s 2024-04-25T00:44:55.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:55 smithi121 ceph-mon[32231]: pgmap v362: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 28 KiB/s rd, 27 KiB/s wr, 225 op/s 2024-04-25T00:44:55.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:55 smithi130 ceph-mon[32525]: pgmap v362: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 28 KiB/s rd, 27 KiB/s wr, 225 op/s 2024-04-25T00:44:56.629 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:44:56.629 INFO:teuthology.orchestra.run.smithi032.stdout: "AsyncMessenger::Worker-0": { 2024-04-25T00:44:56.629 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_messages": 4, 2024-04-25T00:44:56.629 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_messages": 4, 2024-04-25T00:44:56.629 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_bytes": 2992, 2024-04-25T00:44:56.629 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_bytes": 448, 2024-04-25T00:44:56.629 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_created_connections": 4, 2024-04-25T00:44:56.629 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_active_connections": 0, 2024-04-25T00:44:56.629 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_total_time": 0.001720601, 2024-04-25T00:44:56.629 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_send_time": 0.000037446, 2024-04-25T00:44:56.630 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_recv_time": 0.000693361, 2024-04-25T00:44:56.630 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-04-25T00:44:56.630 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:44:56.630 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 3, 2024-04-25T00:44:56.630 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.000802305, 2024-04-25T00:44:56.630 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000267435 2024-04-25T00:44:56.630 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:56.630 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:44:56.630 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 0, 2024-04-25T00:44:56.630 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.000000000, 2024-04-25T00:44:56.630 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000000000 2024-04-25T00:44:56.630 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:56.630 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_encrypted_bytes": 2992, 2024-04-25T00:44:56.631 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_encrypted_bytes": 448 2024-04-25T00:44:56.631 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:56.631 INFO:teuthology.orchestra.run.smithi032.stdout: "AsyncMessenger::Worker-1": { 2024-04-25T00:44:56.631 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_messages": 2, 2024-04-25T00:44:56.631 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_messages": 180, 2024-04-25T00:44:56.631 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_bytes": 320, 2024-04-25T00:44:56.631 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_bytes": 186960, 2024-04-25T00:44:56.631 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_created_connections": 5, 2024-04-25T00:44:56.631 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_active_connections": 3, 2024-04-25T00:44:56.631 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_total_time": 0.017646403, 2024-04-25T00:44:56.631 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_send_time": 0.012211061, 2024-04-25T00:44:56.631 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_recv_time": 0.002031059, 2024-04-25T00:44:56.631 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-04-25T00:44:56.631 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:44:56.632 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 177, 2024-04-25T00:44:56.632 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.016831124, 2024-04-25T00:44:56.632 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000095091 2024-04-25T00:44:56.632 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:56.632 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:44:56.632 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 0, 2024-04-25T00:44:56.632 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.000000000, 2024-04-25T00:44:56.632 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000000000 2024-04-25T00:44:56.632 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:56.632 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_encrypted_bytes": 320, 2024-04-25T00:44:56.632 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_encrypted_bytes": 186960 2024-04-25T00:44:56.632 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:56.632 INFO:teuthology.orchestra.run.smithi032.stdout: "AsyncMessenger::Worker-2": { 2024-04-25T00:44:56.632 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_messages": 246, 2024-04-25T00:44:56.632 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_messages": 235, 2024-04-25T00:44:56.633 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_bytes": 1587344, 2024-04-25T00:44:56.633 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_bytes": 144704, 2024-04-25T00:44:56.633 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_created_connections": 2, 2024-04-25T00:44:56.633 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_active_connections": 1, 2024-04-25T00:44:56.633 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_total_time": 0.057205488, 2024-04-25T00:44:56.633 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_send_time": 0.015546515, 2024-04-25T00:44:56.633 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_recv_time": 0.038972205, 2024-04-25T00:44:56.633 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_fast_dispatch_time": 0.009886062, 2024-04-25T00:44:56.633 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:44:56.633 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 234, 2024-04-25T00:44:56.633 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.024965619, 2024-04-25T00:44:56.633 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000106690 2024-04-25T00:44:56.633 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:56.633 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:44:56.633 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 0, 2024-04-25T00:44:56.634 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.000000000, 2024-04-25T00:44:56.634 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000000000 2024-04-25T00:44:56.634 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:56.634 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_encrypted_bytes": 1587344, 2024-04-25T00:44:56.634 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_encrypted_bytes": 144704 2024-04-25T00:44:56.634 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:56.634 INFO:teuthology.orchestra.run.smithi032.stdout: "cct": { 2024-04-25T00:44:56.634 INFO:teuthology.orchestra.run.smithi032.stdout: "total_workers": 0, 2024-04-25T00:44:56.634 INFO:teuthology.orchestra.run.smithi032.stdout: "unhealthy_workers": 0 2024-04-25T00:44:56.634 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:56.634 INFO:teuthology.orchestra.run.smithi032.stdout: "mempool": { 2024-04-25T00:44:56.634 INFO:teuthology.orchestra.run.smithi032.stdout: "bloom_filter_bytes": 0, 2024-04-25T00:44:56.634 INFO:teuthology.orchestra.run.smithi032.stdout: "bloom_filter_items": 0, 2024-04-25T00:44:56.634 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_alloc_bytes": 0, 2024-04-25T00:44:56.635 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_alloc_items": 0, 2024-04-25T00:44:56.635 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_data_bytes": 0, 2024-04-25T00:44:56.635 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_data_items": 0, 2024-04-25T00:44:56.635 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_onode_bytes": 0, 2024-04-25T00:44:56.635 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_onode_items": 0, 2024-04-25T00:44:56.635 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_meta_bytes": 0, 2024-04-25T00:44:56.635 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_meta_items": 0, 2024-04-25T00:44:56.635 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_other_bytes": 0, 2024-04-25T00:44:56.635 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_other_items": 0, 2024-04-25T00:44:56.635 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_buffer_bytes": 0, 2024-04-25T00:44:56.635 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_buffer_items": 0, 2024-04-25T00:44:56.635 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_extent_bytes": 0, 2024-04-25T00:44:56.635 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_extent_items": 0, 2024-04-25T00:44:56.635 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_blob_bytes": 0, 2024-04-25T00:44:56.635 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_blob_items": 0, 2024-04-25T00:44:56.636 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_shared_blob_bytes": 0, 2024-04-25T00:44:56.636 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_shared_blob_items": 0, 2024-04-25T00:44:56.636 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_inline_bl_bytes": 0, 2024-04-25T00:44:56.636 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_inline_bl_items": 0, 2024-04-25T00:44:56.636 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_fsck_bytes": 0, 2024-04-25T00:44:56.636 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_fsck_items": 0, 2024-04-25T00:44:56.636 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_txc_bytes": 0, 2024-04-25T00:44:56.636 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_txc_items": 0, 2024-04-25T00:44:56.636 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_writing_deferred_bytes": 0, 2024-04-25T00:44:56.636 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_writing_deferred_items": 0, 2024-04-25T00:44:56.636 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_writing_bytes": 0, 2024-04-25T00:44:56.636 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_writing_items": 0, 2024-04-25T00:44:56.636 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_bytes": 0, 2024-04-25T00:44:56.636 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_items": 0, 2024-04-25T00:44:56.637 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_file_reader_bytes": 0, 2024-04-25T00:44:56.637 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_file_reader_items": 0, 2024-04-25T00:44:56.637 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_file_writer_bytes": 0, 2024-04-25T00:44:56.637 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_file_writer_items": 0, 2024-04-25T00:44:56.637 INFO:teuthology.orchestra.run.smithi032.stdout: "buffer_anon_bytes": 149057, 2024-04-25T00:44:56.637 INFO:teuthology.orchestra.run.smithi032.stdout: "buffer_anon_items": 32, 2024-04-25T00:44:56.637 INFO:teuthology.orchestra.run.smithi032.stdout: "buffer_meta_bytes": 0, 2024-04-25T00:44:56.637 INFO:teuthology.orchestra.run.smithi032.stdout: "buffer_meta_items": 0, 2024-04-25T00:44:56.637 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_bytes": 0, 2024-04-25T00:44:56.637 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_items": 0, 2024-04-25T00:44:56.637 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mapbl_bytes": 0, 2024-04-25T00:44:56.637 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mapbl_items": 0, 2024-04-25T00:44:56.637 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pglog_bytes": 0, 2024-04-25T00:44:56.637 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pglog_items": 0, 2024-04-25T00:44:56.638 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_bytes": 0, 2024-04-25T00:44:56.638 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_items": 0, 2024-04-25T00:44:56.638 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_mapping_bytes": 0, 2024-04-25T00:44:56.638 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_mapping_items": 0, 2024-04-25T00:44:56.638 INFO:teuthology.orchestra.run.smithi032.stdout: "pgmap_bytes": 0, 2024-04-25T00:44:56.638 INFO:teuthology.orchestra.run.smithi032.stdout: "pgmap_items": 0, 2024-04-25T00:44:56.638 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_co_bytes": 928, 2024-04-25T00:44:56.638 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_co_items": 2, 2024-04-25T00:44:56.638 INFO:teuthology.orchestra.run.smithi032.stdout: "unittest_1_bytes": 0, 2024-04-25T00:44:56.638 INFO:teuthology.orchestra.run.smithi032.stdout: "unittest_1_items": 0, 2024-04-25T00:44:56.638 INFO:teuthology.orchestra.run.smithi032.stdout: "unittest_2_bytes": 0, 2024-04-25T00:44:56.638 INFO:teuthology.orchestra.run.smithi032.stdout: "unittest_2_items": 0 2024-04-25T00:44:56.638 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:56.638 INFO:teuthology.orchestra.run.smithi032.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-04-25T00:44:56.639 INFO:teuthology.orchestra.run.smithi032.stdout: "val": 0, 2024-04-25T00:44:56.639 INFO:teuthology.orchestra.run.smithi032.stdout: "max": 104857600, 2024-04-25T00:44:56.639 INFO:teuthology.orchestra.run.smithi032.stdout: "get_started": 0, 2024-04-25T00:44:56.639 INFO:teuthology.orchestra.run.smithi032.stdout: "get": 248, 2024-04-25T00:44:56.639 INFO:teuthology.orchestra.run.smithi032.stdout: "get_sum": 1552640, 2024-04-25T00:44:56.639 INFO:teuthology.orchestra.run.smithi032.stdout: "get_or_fail_fail": 0, 2024-04-25T00:44:56.639 INFO:teuthology.orchestra.run.smithi032.stdout: "get_or_fail_success": 248, 2024-04-25T00:44:56.639 INFO:teuthology.orchestra.run.smithi032.stdout: "take": 0, 2024-04-25T00:44:56.639 INFO:teuthology.orchestra.run.smithi032.stdout: "take_sum": 0, 2024-04-25T00:44:56.639 INFO:teuthology.orchestra.run.smithi032.stdout: "put": 248, 2024-04-25T00:44:56.639 INFO:teuthology.orchestra.run.smithi032.stdout: "put_sum": 1552640, 2024-04-25T00:44:56.639 INFO:teuthology.orchestra.run.smithi032.stdout: "wait": { 2024-04-25T00:44:56.639 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 0, 2024-04-25T00:44:56.639 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.000000000, 2024-04-25T00:44:56.640 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000000000 2024-04-25T00:44:56.640 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:56.640 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:56.640 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:44:57.355 WARNING:tasks.check_counter:Counter 'mds.exported' not found on daemon mds.d 2024-04-25T00:44:57.356 WARNING:tasks.check_counter:Counter 'mds.imported' not found on daemon mds.d 2024-04-25T00:44:57.356 WARNING:tasks.check_counter:Counter 'mds.root_rsnaps' not found on daemon mds.d 2024-04-25T00:44:57.356 WARNING:tasks.check_counter:Counter 'mds_server.req_mksnap_latency.avgcount' not found on daemon mds.d 2024-04-25T00:44:57.356 WARNING:tasks.check_counter:Counter 'mds_server.req_rmsnap_latency.avgcount' not found on daemon mds.d 2024-04-25T00:44:57.356 DEBUG:tasks.check_counter:Getting stats from g 2024-04-25T00:44:57.356 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph daemon mds.g perf dump 2024-04-25T00:44:57.606 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:44:57.729 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:57 smithi032 ceph-mon[29357]: pgmap v363: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 13 KiB/s rd, 10 KiB/s wr, 221 op/s 2024-04-25T00:44:57.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:57 smithi121 ceph-mon[32231]: pgmap v363: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 13 KiB/s rd, 10 KiB/s wr, 221 op/s 2024-04-25T00:44:57.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:57 smithi130 ceph-mon[32525]: pgmap v363: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 13 KiB/s rd, 10 KiB/s wr, 221 op/s 2024-04-25T00:44:59.182 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:44:59.182 INFO:teuthology.orchestra.run.smithi032.stdout: "AsyncMessenger::Worker-0": { 2024-04-25T00:44:59.182 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_messages": 0, 2024-04-25T00:44:59.182 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_messages": 2, 2024-04-25T00:44:59.182 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_bytes": 0, 2024-04-25T00:44:59.182 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_bytes": 0, 2024-04-25T00:44:59.182 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_created_connections": 4, 2024-04-25T00:44:59.183 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_active_connections": 0, 2024-04-25T00:44:59.183 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_total_time": 0.001507494, 2024-04-25T00:44:59.183 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_send_time": 0.000000000, 2024-04-25T00:44:59.183 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_recv_time": 0.000112728, 2024-04-25T00:44:59.183 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-04-25T00:44:59.183 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:44:59.183 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 0, 2024-04-25T00:44:59.183 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.000000000, 2024-04-25T00:44:59.183 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000000000 2024-04-25T00:44:59.184 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:59.184 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:44:59.184 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 0, 2024-04-25T00:44:59.184 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.000000000, 2024-04-25T00:44:59.184 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000000000 2024-04-25T00:44:59.184 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:59.184 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_encrypted_bytes": 0, 2024-04-25T00:44:59.184 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_encrypted_bytes": 0 2024-04-25T00:44:59.184 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:59.185 INFO:teuthology.orchestra.run.smithi032.stdout: "AsyncMessenger::Worker-1": { 2024-04-25T00:44:59.185 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_messages": 246, 2024-04-25T00:44:59.185 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_messages": 233, 2024-04-25T00:44:59.185 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_bytes": 1589248, 2024-04-25T00:44:59.185 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_bytes": 142592, 2024-04-25T00:44:59.185 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_created_connections": 2, 2024-04-25T00:44:59.185 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_active_connections": 1, 2024-04-25T00:44:59.185 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_total_time": 0.057774599, 2024-04-25T00:44:59.186 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_send_time": 0.016130811, 2024-04-25T00:44:59.186 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_recv_time": 0.046857466, 2024-04-25T00:44:59.186 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_fast_dispatch_time": 0.006063180, 2024-04-25T00:44:59.186 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:44:59.186 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 233, 2024-04-25T00:44:59.186 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.015317809, 2024-04-25T00:44:59.186 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000065741 2024-04-25T00:44:59.186 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:59.187 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:44:59.187 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 0, 2024-04-25T00:44:59.187 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.000000000, 2024-04-25T00:44:59.187 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000000000 2024-04-25T00:44:59.187 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:59.187 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_encrypted_bytes": 1589248, 2024-04-25T00:44:59.187 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_encrypted_bytes": 142592 2024-04-25T00:44:59.187 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:59.187 INFO:teuthology.orchestra.run.smithi032.stdout: "AsyncMessenger::Worker-2": { 2024-04-25T00:44:59.188 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_messages": 2, 2024-04-25T00:44:59.188 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_messages": 177, 2024-04-25T00:44:59.188 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_bytes": 320, 2024-04-25T00:44:59.188 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_bytes": 186432, 2024-04-25T00:44:59.188 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_created_connections": 5, 2024-04-25T00:44:59.188 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_active_connections": 3, 2024-04-25T00:44:59.188 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_total_time": 0.015621595, 2024-04-25T00:44:59.188 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_send_time": 0.010590424, 2024-04-25T00:44:59.189 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_recv_time": 0.002363536, 2024-04-25T00:44:59.189 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-04-25T00:44:59.189 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:44:59.189 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 174, 2024-04-25T00:44:59.189 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.013680737, 2024-04-25T00:44:59.189 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000078624 2024-04-25T00:44:59.189 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:59.189 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:44:59.189 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 0, 2024-04-25T00:44:59.190 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.000000000, 2024-04-25T00:44:59.190 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000000000 2024-04-25T00:44:59.190 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:59.190 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_encrypted_bytes": 320, 2024-04-25T00:44:59.190 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_encrypted_bytes": 186432 2024-04-25T00:44:59.190 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:59.190 INFO:teuthology.orchestra.run.smithi032.stdout: "cct": { 2024-04-25T00:44:59.190 INFO:teuthology.orchestra.run.smithi032.stdout: "total_workers": 0, 2024-04-25T00:44:59.191 INFO:teuthology.orchestra.run.smithi032.stdout: "unhealthy_workers": 0 2024-04-25T00:44:59.191 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:59.191 INFO:teuthology.orchestra.run.smithi032.stdout: "mempool": { 2024-04-25T00:44:59.191 INFO:teuthology.orchestra.run.smithi032.stdout: "bloom_filter_bytes": 0, 2024-04-25T00:44:59.191 INFO:teuthology.orchestra.run.smithi032.stdout: "bloom_filter_items": 0, 2024-04-25T00:44:59.191 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_alloc_bytes": 0, 2024-04-25T00:44:59.191 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_alloc_items": 0, 2024-04-25T00:44:59.191 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_data_bytes": 0, 2024-04-25T00:44:59.191 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_data_items": 0, 2024-04-25T00:44:59.192 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_onode_bytes": 0, 2024-04-25T00:44:59.192 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_onode_items": 0, 2024-04-25T00:44:59.192 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_meta_bytes": 0, 2024-04-25T00:44:59.192 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_meta_items": 0, 2024-04-25T00:44:59.192 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_other_bytes": 0, 2024-04-25T00:44:59.192 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_other_items": 0, 2024-04-25T00:44:59.192 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_buffer_bytes": 0, 2024-04-25T00:44:59.192 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_buffer_items": 0, 2024-04-25T00:44:59.192 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_extent_bytes": 0, 2024-04-25T00:44:59.193 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_extent_items": 0, 2024-04-25T00:44:59.193 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_blob_bytes": 0, 2024-04-25T00:44:59.193 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_blob_items": 0, 2024-04-25T00:44:59.193 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_shared_blob_bytes": 0, 2024-04-25T00:44:59.193 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_shared_blob_items": 0, 2024-04-25T00:44:59.193 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_inline_bl_bytes": 0, 2024-04-25T00:44:59.194 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_inline_bl_items": 0, 2024-04-25T00:44:59.194 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_fsck_bytes": 0, 2024-04-25T00:44:59.194 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_fsck_items": 0, 2024-04-25T00:44:59.194 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_txc_bytes": 0, 2024-04-25T00:44:59.194 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_txc_items": 0, 2024-04-25T00:44:59.194 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_writing_deferred_bytes": 0, 2024-04-25T00:44:59.194 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_writing_deferred_items": 0, 2024-04-25T00:44:59.194 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_writing_bytes": 0, 2024-04-25T00:44:59.195 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_writing_items": 0, 2024-04-25T00:44:59.195 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_bytes": 0, 2024-04-25T00:44:59.195 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_items": 0, 2024-04-25T00:44:59.195 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_file_reader_bytes": 0, 2024-04-25T00:44:59.195 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_file_reader_items": 0, 2024-04-25T00:44:59.195 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_file_writer_bytes": 0, 2024-04-25T00:44:59.195 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_file_writer_items": 0, 2024-04-25T00:44:59.195 INFO:teuthology.orchestra.run.smithi032.stdout: "buffer_anon_bytes": 148995, 2024-04-25T00:44:59.195 INFO:teuthology.orchestra.run.smithi032.stdout: "buffer_anon_items": 31, 2024-04-25T00:44:59.196 INFO:teuthology.orchestra.run.smithi032.stdout: "buffer_meta_bytes": 0, 2024-04-25T00:44:59.196 INFO:teuthology.orchestra.run.smithi032.stdout: "buffer_meta_items": 0, 2024-04-25T00:44:59.196 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_bytes": 0, 2024-04-25T00:44:59.196 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_items": 0, 2024-04-25T00:44:59.196 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mapbl_bytes": 0, 2024-04-25T00:44:59.196 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mapbl_items": 0, 2024-04-25T00:44:59.196 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pglog_bytes": 0, 2024-04-25T00:44:59.196 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pglog_items": 0, 2024-04-25T00:44:59.197 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_bytes": 0, 2024-04-25T00:44:59.197 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_items": 0, 2024-04-25T00:44:59.197 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_mapping_bytes": 0, 2024-04-25T00:44:59.197 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_mapping_items": 0, 2024-04-25T00:44:59.197 INFO:teuthology.orchestra.run.smithi032.stdout: "pgmap_bytes": 0, 2024-04-25T00:44:59.197 INFO:teuthology.orchestra.run.smithi032.stdout: "pgmap_items": 0, 2024-04-25T00:44:59.197 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_co_bytes": 928, 2024-04-25T00:44:59.197 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_co_items": 2, 2024-04-25T00:44:59.197 INFO:teuthology.orchestra.run.smithi032.stdout: "unittest_1_bytes": 0, 2024-04-25T00:44:59.197 INFO:teuthology.orchestra.run.smithi032.stdout: "unittest_1_items": 0, 2024-04-25T00:44:59.198 INFO:teuthology.orchestra.run.smithi032.stdout: "unittest_2_bytes": 0, 2024-04-25T00:44:59.198 INFO:teuthology.orchestra.run.smithi032.stdout: "unittest_2_items": 0 2024-04-25T00:44:59.198 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:44:59.198 INFO:teuthology.orchestra.run.smithi032.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-04-25T00:44:59.198 INFO:teuthology.orchestra.run.smithi032.stdout: "val": 0, 2024-04-25T00:44:59.198 INFO:teuthology.orchestra.run.smithi032.stdout: "max": 104857600, 2024-04-25T00:44:59.198 INFO:teuthology.orchestra.run.smithi032.stdout: "get_started": 0, 2024-04-25T00:44:59.198 INFO:teuthology.orchestra.run.smithi032.stdout: "get": 244, 2024-04-25T00:44:59.199 INFO:teuthology.orchestra.run.smithi032.stdout: "get_sum": 1552120, 2024-04-25T00:44:59.199 INFO:teuthology.orchestra.run.smithi032.stdout: "get_or_fail_fail": 0, 2024-04-25T00:44:59.199 INFO:teuthology.orchestra.run.smithi032.stdout: "get_or_fail_success": 244, 2024-04-25T00:44:59.199 INFO:teuthology.orchestra.run.smithi032.stdout: "take": 0, 2024-04-25T00:44:59.199 INFO:teuthology.orchestra.run.smithi032.stdout: "take_sum": 0, 2024-04-25T00:44:59.199 INFO:teuthology.orchestra.run.smithi032.stdout: "put": 244, 2024-04-25T00:44:59.199 INFO:teuthology.orchestra.run.smithi032.stdout: "put_sum": 1552120, 2024-04-25T00:44:59.199 INFO:teuthology.orchestra.run.smithi032.stdout: "wait": { 2024-04-25T00:44:59.199 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 0, 2024-04-25T00:44:59.200 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.000000000, 2024-04-25T00:44:59.200 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000000000 2024-04-25T00:44:59.200 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:59.200 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:44:59.200 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:44:59.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:44:59 smithi032 ceph-mon[29357]: pgmap v364: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 9.7 KiB/s rd, 8.1 KiB/s wr, 98 op/s 2024-04-25T00:44:59.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:44:59 smithi121 ceph-mon[32231]: pgmap v364: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 9.7 KiB/s rd, 8.1 KiB/s wr, 98 op/s 2024-04-25T00:44:59.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:44:59 smithi130 ceph-mon[32525]: pgmap v364: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 9.7 KiB/s rd, 8.1 KiB/s wr, 98 op/s 2024-04-25T00:44:59.899 WARNING:tasks.check_counter:Counter 'mds.exported' not found on daemon mds.g 2024-04-25T00:44:59.899 WARNING:tasks.check_counter:Counter 'mds.imported' not found on daemon mds.g 2024-04-25T00:44:59.899 WARNING:tasks.check_counter:Counter 'mds.root_rsnaps' not found on daemon mds.g 2024-04-25T00:44:59.899 WARNING:tasks.check_counter:Counter 'mds_server.req_mksnap_latency.avgcount' not found on daemon mds.g 2024-04-25T00:44:59.900 WARNING:tasks.check_counter:Counter 'mds_server.req_rmsnap_latency.avgcount' not found on daemon mds.g 2024-04-25T00:44:59.900 DEBUG:tasks.check_counter:Getting stats from j 2024-04-25T00:44:59.900 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph daemon mds.j perf dump 2024-04-25T00:45:00.148 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.a/config 2024-04-25T00:45:01.702 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:01 smithi032 ceph-mon[29357]: pgmap v365: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 4.4 KiB/s rd, 1.1 KiB/s wr, 68 op/s 2024-04-25T00:45:01.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:01 smithi121 ceph-mon[32231]: pgmap v365: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 4.4 KiB/s rd, 1.1 KiB/s wr, 68 op/s 2024-04-25T00:45:01.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:01 smithi130 ceph-mon[32525]: pgmap v365: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 4.4 KiB/s rd, 1.1 KiB/s wr, 68 op/s 2024-04-25T00:45:01.948 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-04-25T00:45:01.948 INFO:teuthology.orchestra.run.smithi032.stdout: "AsyncMessenger::Worker-0": { 2024-04-25T00:45:01.948 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_messages": 4, 2024-04-25T00:45:01.948 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_messages": 4, 2024-04-25T00:45:01.948 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_bytes": 2992, 2024-04-25T00:45:01.948 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_bytes": 448, 2024-04-25T00:45:01.948 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_created_connections": 4, 2024-04-25T00:45:01.948 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_active_connections": 0, 2024-04-25T00:45:01.948 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_total_time": 0.001585152, 2024-04-25T00:45:01.948 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_send_time": 0.000031801, 2024-04-25T00:45:01.949 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_recv_time": 0.000558922, 2024-04-25T00:45:01.949 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-04-25T00:45:01.949 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:01.949 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 3, 2024-04-25T00:45:01.949 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.000737714, 2024-04-25T00:45:01.949 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000245904 2024-04-25T00:45:01.949 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:45:01.949 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:01.949 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 0, 2024-04-25T00:45:01.949 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.000000000, 2024-04-25T00:45:01.949 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000000000 2024-04-25T00:45:01.949 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:45:01.949 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_encrypted_bytes": 2992, 2024-04-25T00:45:01.949 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_encrypted_bytes": 448 2024-04-25T00:45:01.950 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:45:01.950 INFO:teuthology.orchestra.run.smithi032.stdout: "AsyncMessenger::Worker-1": { 2024-04-25T00:45:01.950 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_messages": 2, 2024-04-25T00:45:01.950 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_messages": 181, 2024-04-25T00:45:01.950 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_bytes": 320, 2024-04-25T00:45:01.950 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_bytes": 187136, 2024-04-25T00:45:01.950 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_created_connections": 5, 2024-04-25T00:45:01.950 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_active_connections": 3, 2024-04-25T00:45:01.950 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_total_time": 0.016674388, 2024-04-25T00:45:01.950 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_send_time": 0.011514368, 2024-04-25T00:45:01.950 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_recv_time": 0.002267143, 2024-04-25T00:45:01.950 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-04-25T00:45:01.950 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:01.950 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 178, 2024-04-25T00:45:01.951 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.013527888, 2024-04-25T00:45:01.951 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000075999 2024-04-25T00:45:01.951 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:45:01.951 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:01.951 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 0, 2024-04-25T00:45:01.951 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.000000000, 2024-04-25T00:45:01.951 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000000000 2024-04-25T00:45:01.951 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:45:01.951 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_encrypted_bytes": 320, 2024-04-25T00:45:01.951 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_encrypted_bytes": 187136 2024-04-25T00:45:01.951 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:45:01.951 INFO:teuthology.orchestra.run.smithi032.stdout: "AsyncMessenger::Worker-2": { 2024-04-25T00:45:01.951 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_messages": 248, 2024-04-25T00:45:01.951 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_messages": 237, 2024-04-25T00:45:01.952 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_bytes": 1587888, 2024-04-25T00:45:01.952 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_bytes": 145984, 2024-04-25T00:45:01.952 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_created_connections": 2, 2024-04-25T00:45:01.952 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_active_connections": 1, 2024-04-25T00:45:01.952 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_total_time": 0.063579397, 2024-04-25T00:45:01.952 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_send_time": 0.015877261, 2024-04-25T00:45:01.952 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_recv_time": 0.056158083, 2024-04-25T00:45:01.952 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_running_fast_dispatch_time": 0.006157381, 2024-04-25T00:45:01.952 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:01.952 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 236, 2024-04-25T00:45:01.952 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.031652541, 2024-04-25T00:45:01.952 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000134120 2024-04-25T00:45:01.952 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:45:01.952 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:01.952 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 0, 2024-04-25T00:45:01.953 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.000000000, 2024-04-25T00:45:01.953 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000000000 2024-04-25T00:45:01.953 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:45:01.953 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_recv_encrypted_bytes": 1587888, 2024-04-25T00:45:01.953 INFO:teuthology.orchestra.run.smithi032.stdout: "msgr_send_encrypted_bytes": 145984 2024-04-25T00:45:01.953 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:45:01.953 INFO:teuthology.orchestra.run.smithi032.stdout: "cct": { 2024-04-25T00:45:01.953 INFO:teuthology.orchestra.run.smithi032.stdout: "total_workers": 0, 2024-04-25T00:45:01.953 INFO:teuthology.orchestra.run.smithi032.stdout: "unhealthy_workers": 0 2024-04-25T00:45:01.953 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:45:01.954 INFO:teuthology.orchestra.run.smithi032.stdout: "mempool": { 2024-04-25T00:45:01.954 INFO:teuthology.orchestra.run.smithi032.stdout: "bloom_filter_bytes": 0, 2024-04-25T00:45:01.954 INFO:teuthology.orchestra.run.smithi032.stdout: "bloom_filter_items": 0, 2024-04-25T00:45:01.954 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_alloc_bytes": 0, 2024-04-25T00:45:01.954 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_alloc_items": 0, 2024-04-25T00:45:01.954 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_data_bytes": 0, 2024-04-25T00:45:01.954 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_data_items": 0, 2024-04-25T00:45:01.954 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_onode_bytes": 0, 2024-04-25T00:45:01.954 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_onode_items": 0, 2024-04-25T00:45:01.954 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_meta_bytes": 0, 2024-04-25T00:45:01.954 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_meta_items": 0, 2024-04-25T00:45:01.954 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_other_bytes": 0, 2024-04-25T00:45:01.954 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_other_items": 0, 2024-04-25T00:45:01.954 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_buffer_bytes": 0, 2024-04-25T00:45:01.955 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_buffer_items": 0, 2024-04-25T00:45:01.955 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_extent_bytes": 0, 2024-04-25T00:45:01.955 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_extent_items": 0, 2024-04-25T00:45:01.955 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_blob_bytes": 0, 2024-04-25T00:45:01.955 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_blob_items": 0, 2024-04-25T00:45:01.955 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_shared_blob_bytes": 0, 2024-04-25T00:45:01.955 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_shared_blob_items": 0, 2024-04-25T00:45:01.955 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_inline_bl_bytes": 0, 2024-04-25T00:45:01.955 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_inline_bl_items": 0, 2024-04-25T00:45:01.955 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_fsck_bytes": 0, 2024-04-25T00:45:01.955 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_fsck_items": 0, 2024-04-25T00:45:01.955 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_txc_bytes": 0, 2024-04-25T00:45:01.955 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_txc_items": 0, 2024-04-25T00:45:01.955 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_writing_deferred_bytes": 0, 2024-04-25T00:45:01.955 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_writing_deferred_items": 0, 2024-04-25T00:45:01.956 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_writing_bytes": 0, 2024-04-25T00:45:01.956 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_writing_items": 0, 2024-04-25T00:45:01.956 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_bytes": 0, 2024-04-25T00:45:01.956 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_items": 0, 2024-04-25T00:45:01.956 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_file_reader_bytes": 0, 2024-04-25T00:45:01.956 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_file_reader_items": 0, 2024-04-25T00:45:01.956 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_file_writer_bytes": 0, 2024-04-25T00:45:01.956 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_file_writer_items": 0, 2024-04-25T00:45:01.956 INFO:teuthology.orchestra.run.smithi032.stdout: "buffer_anon_bytes": 148771, 2024-04-25T00:45:01.956 INFO:teuthology.orchestra.run.smithi032.stdout: "buffer_anon_items": 32, 2024-04-25T00:45:01.956 INFO:teuthology.orchestra.run.smithi032.stdout: "buffer_meta_bytes": 0, 2024-04-25T00:45:01.956 INFO:teuthology.orchestra.run.smithi032.stdout: "buffer_meta_items": 0, 2024-04-25T00:45:01.956 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_bytes": 0, 2024-04-25T00:45:01.956 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_items": 0, 2024-04-25T00:45:01.957 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mapbl_bytes": 0, 2024-04-25T00:45:01.957 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mapbl_items": 0, 2024-04-25T00:45:01.957 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pglog_bytes": 0, 2024-04-25T00:45:01.957 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pglog_items": 0, 2024-04-25T00:45:01.957 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_bytes": 0, 2024-04-25T00:45:01.957 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_items": 0, 2024-04-25T00:45:01.957 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_mapping_bytes": 0, 2024-04-25T00:45:01.957 INFO:teuthology.orchestra.run.smithi032.stdout: "osdmap_mapping_items": 0, 2024-04-25T00:45:01.957 INFO:teuthology.orchestra.run.smithi032.stdout: "pgmap_bytes": 0, 2024-04-25T00:45:01.957 INFO:teuthology.orchestra.run.smithi032.stdout: "pgmap_items": 0, 2024-04-25T00:45:01.957 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_co_bytes": 928, 2024-04-25T00:45:01.957 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_co_items": 2, 2024-04-25T00:45:01.957 INFO:teuthology.orchestra.run.smithi032.stdout: "unittest_1_bytes": 0, 2024-04-25T00:45:01.957 INFO:teuthology.orchestra.run.smithi032.stdout: "unittest_1_items": 0, 2024-04-25T00:45:01.957 INFO:teuthology.orchestra.run.smithi032.stdout: "unittest_2_bytes": 0, 2024-04-25T00:45:01.958 INFO:teuthology.orchestra.run.smithi032.stdout: "unittest_2_items": 0 2024-04-25T00:45:01.958 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-04-25T00:45:01.958 INFO:teuthology.orchestra.run.smithi032.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-04-25T00:45:01.958 INFO:teuthology.orchestra.run.smithi032.stdout: "val": 0, 2024-04-25T00:45:01.958 INFO:teuthology.orchestra.run.smithi032.stdout: "max": 104857600, 2024-04-25T00:45:01.958 INFO:teuthology.orchestra.run.smithi032.stdout: "get_started": 0, 2024-04-25T00:45:01.958 INFO:teuthology.orchestra.run.smithi032.stdout: "get": 250, 2024-04-25T00:45:01.958 INFO:teuthology.orchestra.run.smithi032.stdout: "get_sum": 1552900, 2024-04-25T00:45:01.958 INFO:teuthology.orchestra.run.smithi032.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:01.958 INFO:teuthology.orchestra.run.smithi032.stdout: "get_or_fail_success": 250, 2024-04-25T00:45:01.958 INFO:teuthology.orchestra.run.smithi032.stdout: "take": 0, 2024-04-25T00:45:01.958 INFO:teuthology.orchestra.run.smithi032.stdout: "take_sum": 0, 2024-04-25T00:45:01.958 INFO:teuthology.orchestra.run.smithi032.stdout: "put": 250, 2024-04-25T00:45:01.958 INFO:teuthology.orchestra.run.smithi032.stdout: "put_sum": 1552900, 2024-04-25T00:45:01.958 INFO:teuthology.orchestra.run.smithi032.stdout: "wait": { 2024-04-25T00:45:01.959 INFO:teuthology.orchestra.run.smithi032.stdout: "avgcount": 0, 2024-04-25T00:45:01.959 INFO:teuthology.orchestra.run.smithi032.stdout: "sum": 0.000000000, 2024-04-25T00:45:01.959 INFO:teuthology.orchestra.run.smithi032.stdout: "avgtime": 0.000000000 2024-04-25T00:45:01.959 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:45:01.959 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-04-25T00:45:01.959 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-04-25T00:45:02.635 WARNING:tasks.check_counter:Counter 'mds.exported' not found on daemon mds.j 2024-04-25T00:45:02.635 WARNING:tasks.check_counter:Counter 'mds.imported' not found on daemon mds.j 2024-04-25T00:45:02.635 WARNING:tasks.check_counter:Counter 'mds.root_rsnaps' not found on daemon mds.j 2024-04-25T00:45:02.635 WARNING:tasks.check_counter:Counter 'mds_server.req_mksnap_latency.avgcount' not found on daemon mds.j 2024-04-25T00:45:02.635 WARNING:tasks.check_counter:Counter 'mds_server.req_rmsnap_latency.avgcount' not found on daemon mds.j 2024-04-25T00:45:02.636 DEBUG:tasks.check_counter:Getting stats from b 2024-04-25T00:45:02.636 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph daemon mds.b perf dump 2024-04-25T00:45:03.008 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.b/config 2024-04-25T00:45:03.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:03 smithi032 ceph-mon[29357]: pgmap v366: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 12 KiB/s rd, 22 KiB/s wr, 44 op/s 2024-04-25T00:45:03.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:03 smithi121 ceph-mon[32231]: pgmap v366: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 12 KiB/s rd, 22 KiB/s wr, 44 op/s 2024-04-25T00:45:03.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:03 smithi130 ceph-mon[32525]: pgmap v366: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 12 KiB/s rd, 22 KiB/s wr, 44 op/s 2024-04-25T00:45:05.035 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-25T00:45:05.035 INFO:teuthology.orchestra.run.smithi121.stdout: "AsyncMessenger::Worker-0": { 2024-04-25T00:45:05.036 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_messages": 8173, 2024-04-25T00:45:05.036 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_messages": 8119, 2024-04-25T00:45:05.036 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_bytes": 2170391, 2024-04-25T00:45:05.036 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_bytes": 176555211, 2024-04-25T00:45:05.036 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_created_connections": 56, 2024-04-25T00:45:05.036 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_active_connections": 6, 2024-04-25T00:45:05.036 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_total_time": 1.632155733, 2024-04-25T00:45:05.036 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_send_time": 0.705628419, 2024-04-25T00:45:05.036 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_recv_time": 0.192966583, 2024-04-25T00:45:05.036 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_fast_dispatch_time": 0.664750655, 2024-04-25T00:45:05.036 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:05.036 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 8120, 2024-04-25T00:45:05.036 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 1.462688442, 2024-04-25T00:45:05.037 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000180134 2024-04-25T00:45:05.037 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.037 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:05.037 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 1847, 2024-04-25T00:45:05.037 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000395445, 2024-04-25T00:45:05.037 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000214 2024-04-25T00:45:05.037 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.037 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_encrypted_bytes": 42224, 2024-04-25T00:45:05.037 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_encrypted_bytes": 976 2024-04-25T00:45:05.037 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.037 INFO:teuthology.orchestra.run.smithi121.stdout: "AsyncMessenger::Worker-1": { 2024-04-25T00:45:05.037 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_messages": 165257, 2024-04-25T00:45:05.037 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_messages": 167755, 2024-04-25T00:45:05.038 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_bytes": 53741833, 2024-04-25T00:45:05.038 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_bytes": 155405194, 2024-04-25T00:45:05.038 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_created_connections": 17, 2024-04-25T00:45:05.038 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_active_connections": 5, 2024-04-25T00:45:05.038 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_total_time": 8.296592393, 2024-04-25T00:45:05.038 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_send_time": 3.120710252, 2024-04-25T00:45:05.038 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_recv_time": 10.801090674, 2024-04-25T00:45:05.038 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_fast_dispatch_time": 0.664460539, 2024-04-25T00:45:05.038 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:05.038 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 167752, 2024-04-25T00:45:05.038 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 40.063709086, 2024-04-25T00:45:05.038 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000238827 2024-04-25T00:45:05.038 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.039 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:05.039 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 254320, 2024-04-25T00:45:05.039 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.069256445, 2024-04-25T00:45:05.039 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000272 2024-04-25T00:45:05.039 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.039 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_encrypted_bytes": 99312, 2024-04-25T00:45:05.039 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_encrypted_bytes": 514288 2024-04-25T00:45:05.039 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.039 INFO:teuthology.orchestra.run.smithi121.stdout: "AsyncMessenger::Worker-2": { 2024-04-25T00:45:05.039 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_messages": 4497, 2024-04-25T00:45:05.039 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_messages": 3858, 2024-04-25T00:45:05.039 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_bytes": 3025653, 2024-04-25T00:45:05.039 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_bytes": 45206489, 2024-04-25T00:45:05.039 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_created_connections": 24, 2024-04-25T00:45:05.040 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_active_connections": 8, 2024-04-25T00:45:05.040 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_total_time": 0.917889740, 2024-04-25T00:45:05.040 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_send_time": 0.304891744, 2024-04-25T00:45:05.040 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_recv_time": 0.173613745, 2024-04-25T00:45:05.040 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_fast_dispatch_time": 0.434257583, 2024-04-25T00:45:05.040 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:05.040 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 3855, 2024-04-25T00:45:05.040 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.803827691, 2024-04-25T00:45:05.040 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000208515 2024-04-25T00:45:05.040 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.040 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:05.040 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 2584, 2024-04-25T00:45:05.040 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000575944, 2024-04-25T00:45:05.041 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000222 2024-04-25T00:45:05.041 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.041 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_encrypted_bytes": 2033456, 2024-04-25T00:45:05.041 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_encrypted_bytes": 274800 2024-04-25T00:45:05.041 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.041 INFO:teuthology.orchestra.run.smithi121.stdout: "cct": { 2024-04-25T00:45:05.041 INFO:teuthology.orchestra.run.smithi121.stdout: "total_workers": 1, 2024-04-25T00:45:05.041 INFO:teuthology.orchestra.run.smithi121.stdout: "unhealthy_workers": 0 2024-04-25T00:45:05.041 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.041 INFO:teuthology.orchestra.run.smithi121.stdout: "finisher-MDSRank": { 2024-04-25T00:45:05.041 INFO:teuthology.orchestra.run.smithi121.stdout: "queue_len": 0, 2024-04-25T00:45:05.041 INFO:teuthology.orchestra.run.smithi121.stdout: "complete_latency": { 2024-04-25T00:45:05.041 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 6876, 2024-04-25T00:45:05.042 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 70.471853121, 2024-04-25T00:45:05.042 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.010248960 2024-04-25T00:45:05.042 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:05.042 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.042 INFO:teuthology.orchestra.run.smithi121.stdout: "finisher-PurgeQueue": { 2024-04-25T00:45:05.042 INFO:teuthology.orchestra.run.smithi121.stdout: "queue_len": 0, 2024-04-25T00:45:05.042 INFO:teuthology.orchestra.run.smithi121.stdout: "complete_latency": { 2024-04-25T00:45:05.042 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 1471, 2024-04-25T00:45:05.042 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.057953219, 2024-04-25T00:45:05.042 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000039397 2024-04-25T00:45:05.042 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:05.042 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.043 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-25T00:45:05.043 INFO:teuthology.orchestra.run.smithi121.stdout: "request": 236, 2024-04-25T00:45:05.043 INFO:teuthology.orchestra.run.smithi121.stdout: "reply": 233, 2024-04-25T00:45:05.043 INFO:teuthology.orchestra.run.smithi121.stdout: "reply_latency": { 2024-04-25T00:45:05.043 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 233, 2024-04-25T00:45:05.043 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 66.526726586, 2024-04-25T00:45:05.043 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.285522431 2024-04-25T00:45:05.043 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.043 INFO:teuthology.orchestra.run.smithi121.stdout: "slow_reply": 0, 2024-04-25T00:45:05.043 INFO:teuthology.orchestra.run.smithi121.stdout: "forward": 3, 2024-04-25T00:45:05.044 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_fetch_complete": 0, 2024-04-25T00:45:05.044 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_fetch_keys": 0, 2024-04-25T00:45:05.044 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_commit": 32, 2024-04-25T00:45:05.044 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_split": 1, 2024-04-25T00:45:05.044 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_merge": 2, 2024-04-25T00:45:05.044 INFO:teuthology.orchestra.run.smithi121.stdout: "inodes": 123, 2024-04-25T00:45:05.044 INFO:teuthology.orchestra.run.smithi121.stdout: "inodes_top": 19, 2024-04-25T00:45:05.044 INFO:teuthology.orchestra.run.smithi121.stdout: "inodes_bottom": 90, 2024-04-25T00:45:05.044 INFO:teuthology.orchestra.run.smithi121.stdout: "inodes_pin_tail": 14, 2024-04-25T00:45:05.044 INFO:teuthology.orchestra.run.smithi121.stdout: "inodes_pinned": 94, 2024-04-25T00:45:05.044 INFO:teuthology.orchestra.run.smithi121.stdout: "inodes_expired": 28, 2024-04-25T00:45:05.044 INFO:teuthology.orchestra.run.smithi121.stdout: "inodes_with_caps": 3, 2024-04-25T00:45:05.044 INFO:teuthology.orchestra.run.smithi121.stdout: "caps": 3, 2024-04-25T00:45:05.044 INFO:teuthology.orchestra.run.smithi121.stdout: "subtrees": 4, 2024-04-25T00:45:05.045 INFO:teuthology.orchestra.run.smithi121.stdout: "traverse": 316, 2024-04-25T00:45:05.045 INFO:teuthology.orchestra.run.smithi121.stdout: "traverse_hit": 297, 2024-04-25T00:45:05.045 INFO:teuthology.orchestra.run.smithi121.stdout: "traverse_forward": 1, 2024-04-25T00:45:05.045 INFO:teuthology.orchestra.run.smithi121.stdout: "traverse_discover": 0, 2024-04-25T00:45:05.045 INFO:teuthology.orchestra.run.smithi121.stdout: "traverse_dir_fetch": 0, 2024-04-25T00:45:05.045 INFO:teuthology.orchestra.run.smithi121.stdout: "traverse_remote_ino": 0, 2024-04-25T00:45:05.045 INFO:teuthology.orchestra.run.smithi121.stdout: "traverse_lock": 0, 2024-04-25T00:45:05.045 INFO:teuthology.orchestra.run.smithi121.stdout: "load_cent": 104, 2024-04-25T00:45:05.045 INFO:teuthology.orchestra.run.smithi121.stdout: "q": 0, 2024-04-25T00:45:05.045 INFO:teuthology.orchestra.run.smithi121.stdout: "exported": 5, 2024-04-25T00:45:05.045 INFO:teuthology.orchestra.run.smithi121.stdout: "exported_inodes": 23, 2024-04-25T00:45:05.045 INFO:teuthology.orchestra.run.smithi121.stdout: "imported": 4, 2024-04-25T00:45:05.045 INFO:teuthology.orchestra.run.smithi121.stdout: "imported_inodes": 58, 2024-04-25T00:45:05.046 INFO:teuthology.orchestra.run.smithi121.stdout: "openino_dir_fetch": 0, 2024-04-25T00:45:05.046 INFO:teuthology.orchestra.run.smithi121.stdout: "openino_backtrace_fetch": 127, 2024-04-25T00:45:05.046 INFO:teuthology.orchestra.run.smithi121.stdout: "openino_peer_discover": 0, 2024-04-25T00:45:05.046 INFO:teuthology.orchestra.run.smithi121.stdout: "root_rfiles": 4, 2024-04-25T00:45:05.046 INFO:teuthology.orchestra.run.smithi121.stdout: "root_rbytes": 234, 2024-04-25T00:45:05.046 INFO:teuthology.orchestra.run.smithi121.stdout: "root_rsnaps": 6, 2024-04-25T00:45:05.046 INFO:teuthology.orchestra.run.smithi121.stdout: "scrub_backtrace_fetch": 127, 2024-04-25T00:45:05.046 INFO:teuthology.orchestra.run.smithi121.stdout: "scrub_set_tag": 0, 2024-04-25T00:45:05.046 INFO:teuthology.orchestra.run.smithi121.stdout: "scrub_backtrace_repaired": 0, 2024-04-25T00:45:05.046 INFO:teuthology.orchestra.run.smithi121.stdout: "scrub_inotable_repaired": 0, 2024-04-25T00:45:05.046 INFO:teuthology.orchestra.run.smithi121.stdout: "scrub_dir_inodes": 63, 2024-04-25T00:45:05.046 INFO:teuthology.orchestra.run.smithi121.stdout: "scrub_dir_base_inodes": 7, 2024-04-25T00:45:05.046 INFO:teuthology.orchestra.run.smithi121.stdout: "scrub_dirfrag_rstats": 56, 2024-04-25T00:45:05.047 INFO:teuthology.orchestra.run.smithi121.stdout: "scrub_file_inodes": 64, 2024-04-25T00:45:05.047 INFO:teuthology.orchestra.run.smithi121.stdout: "handle_inode_file_caps": 5, 2024-04-25T00:45:05.047 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph_cap_op_revoke": 36, 2024-04-25T00:45:05.047 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph_cap_op_grant": 3132, 2024-04-25T00:45:05.047 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph_cap_op_trunc": 0, 2024-04-25T00:45:05.047 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-04-25T00:45:05.047 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph_cap_op_flush_ack": 0, 2024-04-25T00:45:05.047 INFO:teuthology.orchestra.run.smithi121.stdout: "handle_client_caps": 169264, 2024-04-25T00:45:05.047 INFO:teuthology.orchestra.run.smithi121.stdout: "handle_client_caps_dirty": 169205, 2024-04-25T00:45:05.047 INFO:teuthology.orchestra.run.smithi121.stdout: "handle_client_cap_release": 6, 2024-04-25T00:45:05.047 INFO:teuthology.orchestra.run.smithi121.stdout: "process_request_cap_release": 42 2024-04-25T00:45:05.047 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.047 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_cache": { 2024-04-25T00:45:05.047 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_update": 0, 2024-04-25T00:45:05.047 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_update_receipt": 0, 2024-04-25T00:45:05.048 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_try_discover": 0, 2024-04-25T00:45:05.048 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_send_discover": 0, 2024-04-25T00:45:05.048 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_handle_discover": 9, 2024-04-25T00:45:05.048 INFO:teuthology.orchestra.run.smithi121.stdout: "num_strays": 3, 2024-04-25T00:45:05.048 INFO:teuthology.orchestra.run.smithi121.stdout: "num_strays_delayed": 0, 2024-04-25T00:45:05.048 INFO:teuthology.orchestra.run.smithi121.stdout: "num_strays_enqueuing": 0, 2024-04-25T00:45:05.048 INFO:teuthology.orchestra.run.smithi121.stdout: "strays_created": 9, 2024-04-25T00:45:05.048 INFO:teuthology.orchestra.run.smithi121.stdout: "strays_enqueued": 6, 2024-04-25T00:45:05.048 INFO:teuthology.orchestra.run.smithi121.stdout: "strays_reintegrated": 0, 2024-04-25T00:45:05.048 INFO:teuthology.orchestra.run.smithi121.stdout: "strays_migrated": 0, 2024-04-25T00:45:05.048 INFO:teuthology.orchestra.run.smithi121.stdout: "num_recovering_processing": 0, 2024-04-25T00:45:05.048 INFO:teuthology.orchestra.run.smithi121.stdout: "num_recovering_enqueued": 0, 2024-04-25T00:45:05.048 INFO:teuthology.orchestra.run.smithi121.stdout: "num_recovering_prioritized": 0, 2024-04-25T00:45:05.048 INFO:teuthology.orchestra.run.smithi121.stdout: "recovery_started": 0, 2024-04-25T00:45:05.049 INFO:teuthology.orchestra.run.smithi121.stdout: "recovery_completed": 0, 2024-04-25T00:45:05.049 INFO:teuthology.orchestra.run.smithi121.stdout: "ireq_quiesce_path": 0, 2024-04-25T00:45:05.049 INFO:teuthology.orchestra.run.smithi121.stdout: "ireq_quiesce_inode": 0, 2024-04-25T00:45:05.049 INFO:teuthology.orchestra.run.smithi121.stdout: "ireq_enqueue_scrub": 7, 2024-04-25T00:45:05.049 INFO:teuthology.orchestra.run.smithi121.stdout: "ireq_exportdir": 6, 2024-04-25T00:45:05.049 INFO:teuthology.orchestra.run.smithi121.stdout: "ireq_flush": 0, 2024-04-25T00:45:05.049 INFO:teuthology.orchestra.run.smithi121.stdout: "ireq_fragmentdir": 3, 2024-04-25T00:45:05.049 INFO:teuthology.orchestra.run.smithi121.stdout: "ireq_fragstats": 0, 2024-04-25T00:45:05.049 INFO:teuthology.orchestra.run.smithi121.stdout: "ireq_inodestats": 0 2024-04-25T00:45:05.049 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.049 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_log": { 2024-04-25T00:45:05.049 INFO:teuthology.orchestra.run.smithi121.stdout: "evlrg": 0, 2024-04-25T00:45:05.049 INFO:teuthology.orchestra.run.smithi121.stdout: "evadd": 161829, 2024-04-25T00:45:05.049 INFO:teuthology.orchestra.run.smithi121.stdout: "evex": 66508, 2024-04-25T00:45:05.049 INFO:teuthology.orchestra.run.smithi121.stdout: "evtrm": 61451, 2024-04-25T00:45:05.050 INFO:teuthology.orchestra.run.smithi121.stdout: "ev": 100378, 2024-04-25T00:45:05.050 INFO:teuthology.orchestra.run.smithi121.stdout: "evexg": 0, 2024-04-25T00:45:05.050 INFO:teuthology.orchestra.run.smithi121.stdout: "evexd": 5057, 2024-04-25T00:45:05.050 INFO:teuthology.orchestra.run.smithi121.stdout: "segadd": 209, 2024-04-25T00:45:05.050 INFO:teuthology.orchestra.run.smithi121.stdout: "segex": 81, 2024-04-25T00:45:05.050 INFO:teuthology.orchestra.run.smithi121.stdout: "segtrm": 74, 2024-04-25T00:45:05.050 INFO:teuthology.orchestra.run.smithi121.stdout: "seg": 135, 2024-04-25T00:45:05.050 INFO:teuthology.orchestra.run.smithi121.stdout: "segmjr": 9, 2024-04-25T00:45:05.050 INFO:teuthology.orchestra.run.smithi121.stdout: "segexg": 0, 2024-04-25T00:45:05.050 INFO:teuthology.orchestra.run.smithi121.stdout: "segexd": 7, 2024-04-25T00:45:05.050 INFO:teuthology.orchestra.run.smithi121.stdout: "expos": 114541499, 2024-04-25T00:45:05.050 INFO:teuthology.orchestra.run.smithi121.stdout: "wrpos": 363198126, 2024-04-25T00:45:05.050 INFO:teuthology.orchestra.run.smithi121.stdout: "rdpos": 0, 2024-04-25T00:45:05.051 INFO:teuthology.orchestra.run.smithi121.stdout: "jlat": { 2024-04-25T00:45:05.051 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 9851, 2024-04-25T00:45:05.051 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 5021.881249789, 2024-04-25T00:45:05.051 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.509783905 2024-04-25T00:45:05.051 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.051 INFO:teuthology.orchestra.run.smithi121.stdout: "replayed": 0 2024-04-25T00:45:05.051 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.051 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_mem": { 2024-04-25T00:45:05.051 INFO:teuthology.orchestra.run.smithi121.stdout: "ino": 103, 2024-04-25T00:45:05.051 INFO:teuthology.orchestra.run.smithi121.stdout: "ino+": 138, 2024-04-25T00:45:05.051 INFO:teuthology.orchestra.run.smithi121.stdout: "ino-": 35, 2024-04-25T00:45:05.051 INFO:teuthology.orchestra.run.smithi121.stdout: "dir": 27, 2024-04-25T00:45:05.051 INFO:teuthology.orchestra.run.smithi121.stdout: "dir+": 36, 2024-04-25T00:45:05.052 INFO:teuthology.orchestra.run.smithi121.stdout: "dir-": 9, 2024-04-25T00:45:05.052 INFO:teuthology.orchestra.run.smithi121.stdout: "dn": 123, 2024-04-25T00:45:05.052 INFO:teuthology.orchestra.run.smithi121.stdout: "dn+": 157, 2024-04-25T00:45:05.052 INFO:teuthology.orchestra.run.smithi121.stdout: "dn-": 34, 2024-04-25T00:45:05.052 INFO:teuthology.orchestra.run.smithi121.stdout: "cap": 3, 2024-04-25T00:45:05.052 INFO:teuthology.orchestra.run.smithi121.stdout: "cap+": 42, 2024-04-25T00:45:05.052 INFO:teuthology.orchestra.run.smithi121.stdout: "cap-": 39, 2024-04-25T00:45:05.052 INFO:teuthology.orchestra.run.smithi121.stdout: "rss": 86320, 2024-04-25T00:45:05.052 INFO:teuthology.orchestra.run.smithi121.stdout: "heap": 231804 2024-04-25T00:45:05.052 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.052 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_server": { 2024-04-25T00:45:05.052 INFO:teuthology.orchestra.run.smithi121.stdout: "dispatch_client_request": 311, 2024-04-25T00:45:05.052 INFO:teuthology.orchestra.run.smithi121.stdout: "dispatch_server_request": 16, 2024-04-25T00:45:05.052 INFO:teuthology.orchestra.run.smithi121.stdout: "handle_client_request": 236, 2024-04-25T00:45:05.052 INFO:teuthology.orchestra.run.smithi121.stdout: "handle_client_session": 41, 2024-04-25T00:45:05.053 INFO:teuthology.orchestra.run.smithi121.stdout: "handle_peer_request": 36, 2024-04-25T00:45:05.053 INFO:teuthology.orchestra.run.smithi121.stdout: "req_create_latency": { 2024-04-25T00:45:05.053 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 22, 2024-04-25T00:45:05.053 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.023687542, 2024-04-25T00:45:05.053 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.001076706 2024-04-25T00:45:05.053 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.053 INFO:teuthology.orchestra.run.smithi121.stdout: "req_getattr_latency": { 2024-04-25T00:45:05.053 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 133, 2024-04-25T00:45:05.053 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.036123461, 2024-04-25T00:45:05.053 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000271604 2024-04-25T00:45:05.053 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.053 INFO:teuthology.orchestra.run.smithi121.stdout: "req_getfilelock_latency": { 2024-04-25T00:45:05.053 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:05.054 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:05.054 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:05.054 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.054 INFO:teuthology.orchestra.run.smithi121.stdout: "req_link_latency": { 2024-04-25T00:45:05.054 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:05.054 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:05.054 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:05.054 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.054 INFO:teuthology.orchestra.run.smithi121.stdout: "req_lookup_latency": { 2024-04-25T00:45:05.054 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 11, 2024-04-25T00:45:05.054 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 41.490394087, 2024-04-25T00:45:05.054 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 3.771854007 2024-04-25T00:45:05.054 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.055 INFO:teuthology.orchestra.run.smithi121.stdout: "req_lookuphash_latency": { 2024-04-25T00:45:05.055 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:05.055 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:05.055 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:05.055 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.055 INFO:teuthology.orchestra.run.smithi121.stdout: "req_lookupino_latency": { 2024-04-25T00:45:05.055 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 1, 2024-04-25T00:45:05.055 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000123566, 2024-04-25T00:45:05.055 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000123566 2024-04-25T00:45:05.055 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.055 INFO:teuthology.orchestra.run.smithi121.stdout: "req_lookupname_latency": { 2024-04-25T00:45:05.055 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:05.055 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:05.056 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:05.056 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.056 INFO:teuthology.orchestra.run.smithi121.stdout: "req_lookupparent_latency": { 2024-04-25T00:45:05.056 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:05.056 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:05.056 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:05.056 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.056 INFO:teuthology.orchestra.run.smithi121.stdout: "req_lookupsnap_latency": { 2024-04-25T00:45:05.056 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:05.056 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:05.056 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:05.056 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.056 INFO:teuthology.orchestra.run.smithi121.stdout: "req_lssnap_latency": { 2024-04-25T00:45:05.056 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 12, 2024-04-25T00:45:05.057 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.002640976, 2024-04-25T00:45:05.057 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000220081 2024-04-25T00:45:05.057 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.057 INFO:teuthology.orchestra.run.smithi121.stdout: "req_mkdir_latency": { 2024-04-25T00:45:05.057 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 10, 2024-04-25T00:45:05.058 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.012909408, 2024-04-25T00:45:05.058 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.001290940 2024-04-25T00:45:05.058 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.058 INFO:teuthology.orchestra.run.smithi121.stdout: "req_mknod_latency": { 2024-04-25T00:45:05.058 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:05.058 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:05.058 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:05.058 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.058 INFO:teuthology.orchestra.run.smithi121.stdout: "req_mksnap_latency": { 2024-04-25T00:45:05.058 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 12, 2024-04-25T00:45:05.058 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 24.784779378, 2024-04-25T00:45:05.058 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 2.065398281 2024-04-25T00:45:05.058 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.059 INFO:teuthology.orchestra.run.smithi121.stdout: "req_open_latency": { 2024-04-25T00:45:05.059 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 1, 2024-04-25T00:45:05.059 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000328222, 2024-04-25T00:45:05.059 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000328222 2024-04-25T00:45:05.059 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.059 INFO:teuthology.orchestra.run.smithi121.stdout: "req_readdir_latency": { 2024-04-25T00:45:05.059 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 4, 2024-04-25T00:45:05.059 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.017618400, 2024-04-25T00:45:05.059 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.004404600 2024-04-25T00:45:05.059 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.059 INFO:teuthology.orchestra.run.smithi121.stdout: "req_rename_latency": { 2024-04-25T00:45:05.059 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 2, 2024-04-25T00:45:05.060 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.002035326, 2024-04-25T00:45:05.060 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.001017663 2024-04-25T00:45:05.060 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.060 INFO:teuthology.orchestra.run.smithi121.stdout: "req_renamesnap_latency": { 2024-04-25T00:45:05.060 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:05.060 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:05.060 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:05.060 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.060 INFO:teuthology.orchestra.run.smithi121.stdout: "req_snapdiff_latency": { 2024-04-25T00:45:05.060 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:05.060 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:05.060 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:05.061 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.061 INFO:teuthology.orchestra.run.smithi121.stdout: "req_rmdir_latency": { 2024-04-25T00:45:05.061 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 3, 2024-04-25T00:45:05.061 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.039508618, 2024-04-25T00:45:05.061 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.013169539 2024-04-25T00:45:05.061 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.061 INFO:teuthology.orchestra.run.smithi121.stdout: "req_rmsnap_latency": { 2024-04-25T00:45:05.061 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 6, 2024-04-25T00:45:05.061 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.083600201, 2024-04-25T00:45:05.061 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.013933366 2024-04-25T00:45:05.061 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.061 INFO:teuthology.orchestra.run.smithi121.stdout: "req_rmxattr_latency": { 2024-04-25T00:45:05.061 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:05.061 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:05.062 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:05.062 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.062 INFO:teuthology.orchestra.run.smithi121.stdout: "req_setattr_latency": { 2024-04-25T00:45:05.062 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 2, 2024-04-25T00:45:05.062 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.001553499, 2024-04-25T00:45:05.062 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000776749 2024-04-25T00:45:05.062 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.062 INFO:teuthology.orchestra.run.smithi121.stdout: "req_setdirlayout_latency": { 2024-04-25T00:45:05.062 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:05.062 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:05.062 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:05.062 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.062 INFO:teuthology.orchestra.run.smithi121.stdout: "req_setfilelock_latency": { 2024-04-25T00:45:05.062 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:05.063 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:05.063 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:05.063 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.063 INFO:teuthology.orchestra.run.smithi121.stdout: "req_setlayout_latency": { 2024-04-25T00:45:05.063 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:05.063 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:05.063 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:05.063 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.063 INFO:teuthology.orchestra.run.smithi121.stdout: "req_setxattr_latency": { 2024-04-25T00:45:05.063 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 8, 2024-04-25T00:45:05.063 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.019551667, 2024-04-25T00:45:05.063 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.002443958 2024-04-25T00:45:05.063 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.064 INFO:teuthology.orchestra.run.smithi121.stdout: "req_symlink_latency": { 2024-04-25T00:45:05.064 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:05.064 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:05.064 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:05.064 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.064 INFO:teuthology.orchestra.run.smithi121.stdout: "req_unlink_latency": { 2024-04-25T00:45:05.064 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 6, 2024-04-25T00:45:05.064 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.011872235, 2024-04-25T00:45:05.064 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.001978705 2024-04-25T00:45:05.064 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.064 INFO:teuthology.orchestra.run.smithi121.stdout: "cap_revoke_eviction": 0, 2024-04-25T00:45:05.064 INFO:teuthology.orchestra.run.smithi121.stdout: "cap_acquisition_throttle": 0, 2024-04-25T00:45:05.064 INFO:teuthology.orchestra.run.smithi121.stdout: "req_getvxattr_latency": { 2024-04-25T00:45:05.065 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:05.065 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:05.065 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:05.065 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:05.065 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.065 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_sessions": { 2024-04-25T00:45:05.065 INFO:teuthology.orchestra.run.smithi121.stdout: "session_count": 2, 2024-04-25T00:45:05.065 INFO:teuthology.orchestra.run.smithi121.stdout: "session_add": 6, 2024-04-25T00:45:05.065 INFO:teuthology.orchestra.run.smithi121.stdout: "session_remove": 4, 2024-04-25T00:45:05.065 INFO:teuthology.orchestra.run.smithi121.stdout: "sessions_open": 2, 2024-04-25T00:45:05.065 INFO:teuthology.orchestra.run.smithi121.stdout: "sessions_stale": 0, 2024-04-25T00:45:05.065 INFO:teuthology.orchestra.run.smithi121.stdout: "total_load": 42, 2024-04-25T00:45:05.065 INFO:teuthology.orchestra.run.smithi121.stdout: "average_load": 21, 2024-04-25T00:45:05.065 INFO:teuthology.orchestra.run.smithi121.stdout: "avg_session_uptime": 684, 2024-04-25T00:45:05.066 INFO:teuthology.orchestra.run.smithi121.stdout: "mdthresh_evicted": 0 2024-04-25T00:45:05.066 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.066 INFO:teuthology.orchestra.run.smithi121.stdout: "mempool": { 2024-04-25T00:45:05.066 INFO:teuthology.orchestra.run.smithi121.stdout: "bloom_filter_bytes": 120, 2024-04-25T00:45:05.066 INFO:teuthology.orchestra.run.smithi121.stdout: "bloom_filter_items": 120, 2024-04-25T00:45:05.066 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_alloc_bytes": 0, 2024-04-25T00:45:05.066 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_alloc_items": 0, 2024-04-25T00:45:05.066 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_data_bytes": 0, 2024-04-25T00:45:05.066 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_data_items": 0, 2024-04-25T00:45:05.066 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_onode_bytes": 0, 2024-04-25T00:45:05.066 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_onode_items": 0, 2024-04-25T00:45:05.066 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_meta_bytes": 0, 2024-04-25T00:45:05.066 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_meta_items": 0, 2024-04-25T00:45:05.067 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_other_bytes": 0, 2024-04-25T00:45:05.067 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_other_items": 0, 2024-04-25T00:45:05.067 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_buffer_bytes": 0, 2024-04-25T00:45:05.067 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_buffer_items": 0, 2024-04-25T00:45:05.067 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_extent_bytes": 0, 2024-04-25T00:45:05.067 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_extent_items": 0, 2024-04-25T00:45:05.067 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_blob_bytes": 0, 2024-04-25T00:45:05.067 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_blob_items": 0, 2024-04-25T00:45:05.067 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_shared_blob_bytes": 0, 2024-04-25T00:45:05.067 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_shared_blob_items": 0, 2024-04-25T00:45:05.067 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_inline_bl_bytes": 0, 2024-04-25T00:45:05.067 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_inline_bl_items": 0, 2024-04-25T00:45:05.068 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_fsck_bytes": 0, 2024-04-25T00:45:05.068 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_fsck_items": 0, 2024-04-25T00:45:05.068 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_txc_bytes": 0, 2024-04-25T00:45:05.068 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_txc_items": 0, 2024-04-25T00:45:05.068 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_writing_deferred_bytes": 0, 2024-04-25T00:45:05.068 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_writing_deferred_items": 0, 2024-04-25T00:45:05.068 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_writing_bytes": 0, 2024-04-25T00:45:05.068 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_writing_items": 0, 2024-04-25T00:45:05.068 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_bytes": 0, 2024-04-25T00:45:05.068 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_items": 0, 2024-04-25T00:45:05.068 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_file_reader_bytes": 0, 2024-04-25T00:45:05.068 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_file_reader_items": 0, 2024-04-25T00:45:05.069 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_file_writer_bytes": 0, 2024-04-25T00:45:05.069 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_file_writer_items": 0, 2024-04-25T00:45:05.069 INFO:teuthology.orchestra.run.smithi121.stdout: "buffer_anon_bytes": 354648, 2024-04-25T00:45:05.069 INFO:teuthology.orchestra.run.smithi121.stdout: "buffer_anon_items": 183, 2024-04-25T00:45:05.069 INFO:teuthology.orchestra.run.smithi121.stdout: "buffer_meta_bytes": 176, 2024-04-25T00:45:05.069 INFO:teuthology.orchestra.run.smithi121.stdout: "buffer_meta_items": 2, 2024-04-25T00:45:05.069 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_bytes": 0, 2024-04-25T00:45:05.069 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_items": 0, 2024-04-25T00:45:05.069 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mapbl_bytes": 0, 2024-04-25T00:45:05.069 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mapbl_items": 0, 2024-04-25T00:45:05.069 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pglog_bytes": 0, 2024-04-25T00:45:05.069 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pglog_items": 0, 2024-04-25T00:45:05.069 INFO:teuthology.orchestra.run.smithi121.stdout: "osdmap_bytes": 9584, 2024-04-25T00:45:05.069 INFO:teuthology.orchestra.run.smithi121.stdout: "osdmap_items": 183, 2024-04-25T00:45:05.070 INFO:teuthology.orchestra.run.smithi121.stdout: "osdmap_mapping_bytes": 0, 2024-04-25T00:45:05.070 INFO:teuthology.orchestra.run.smithi121.stdout: "osdmap_mapping_items": 0, 2024-04-25T00:45:05.070 INFO:teuthology.orchestra.run.smithi121.stdout: "pgmap_bytes": 0, 2024-04-25T00:45:05.070 INFO:teuthology.orchestra.run.smithi121.stdout: "pgmap_items": 0, 2024-04-25T00:45:05.070 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_co_bytes": 401334, 2024-04-25T00:45:05.070 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_co_items": 3270, 2024-04-25T00:45:05.070 INFO:teuthology.orchestra.run.smithi121.stdout: "unittest_1_bytes": 0, 2024-04-25T00:45:05.070 INFO:teuthology.orchestra.run.smithi121.stdout: "unittest_1_items": 0, 2024-04-25T00:45:05.070 INFO:teuthology.orchestra.run.smithi121.stdout: "unittest_2_bytes": 0, 2024-04-25T00:45:05.070 INFO:teuthology.orchestra.run.smithi121.stdout: "unittest_2_items": 0 2024-04-25T00:45:05.070 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.070 INFO:teuthology.orchestra.run.smithi121.stdout: "objecter": { 2024-04-25T00:45:05.071 INFO:teuthology.orchestra.run.smithi121.stdout: "op_active": 0, 2024-04-25T00:45:05.071 INFO:teuthology.orchestra.run.smithi121.stdout: "op_laggy": 0, 2024-04-25T00:45:05.071 INFO:teuthology.orchestra.run.smithi121.stdout: "op_send": 12492, 2024-04-25T00:45:05.071 INFO:teuthology.orchestra.run.smithi121.stdout: "op_send_bytes": 360331939, 2024-04-25T00:45:05.071 INFO:teuthology.orchestra.run.smithi121.stdout: "op_resend": 33, 2024-04-25T00:45:05.071 INFO:teuthology.orchestra.run.smithi121.stdout: "op_reply": 12492, 2024-04-25T00:45:05.071 INFO:teuthology.orchestra.run.smithi121.stdout: "op_latency": { 2024-04-25T00:45:05.071 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 12492, 2024-04-25T00:45:05.071 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 4581.003198042, 2024-04-25T00:45:05.071 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.366714953 2024-04-25T00:45:05.071 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.071 INFO:teuthology.orchestra.run.smithi121.stdout: "op_inflight": 0, 2024-04-25T00:45:05.072 INFO:teuthology.orchestra.run.smithi121.stdout: "oplen_avg": { 2024-04-25T00:45:05.072 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 12492, 2024-04-25T00:45:05.072 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 12619 2024-04-25T00:45:05.072 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.072 INFO:teuthology.orchestra.run.smithi121.stdout: "op": 12492, 2024-04-25T00:45:05.072 INFO:teuthology.orchestra.run.smithi121.stdout: "op_r": 142, 2024-04-25T00:45:05.072 INFO:teuthology.orchestra.run.smithi121.stdout: "op_w": 12350, 2024-04-25T00:45:05.072 INFO:teuthology.orchestra.run.smithi121.stdout: "op_rmw": 0, 2024-04-25T00:45:05.072 INFO:teuthology.orchestra.run.smithi121.stdout: "op_pg": 0, 2024-04-25T00:45:05.072 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_stat": 1, 2024-04-25T00:45:05.072 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_create": 41, 2024-04-25T00:45:05.072 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_read": 8, 2024-04-25T00:45:05.072 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_write": 9854, 2024-04-25T00:45:05.072 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_writefull": 50, 2024-04-25T00:45:05.072 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_writesame": 0, 2024-04-25T00:45:05.072 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_append": 0, 2024-04-25T00:45:05.073 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_zero": 0, 2024-04-25T00:45:05.073 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_truncate": 1, 2024-04-25T00:45:05.073 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_delete": 2177, 2024-04-25T00:45:05.073 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_mapext": 0, 2024-04-25T00:45:05.073 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_sparse_read": 0, 2024-04-25T00:45:05.073 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_clonerange": 0, 2024-04-25T00:45:05.073 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_getxattr": 134, 2024-04-25T00:45:05.073 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_setxattr": 72, 2024-04-25T00:45:05.073 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_cmpxattr": 0, 2024-04-25T00:45:05.073 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_rmxattr": 0, 2024-04-25T00:45:05.073 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_resetxattrs": 0, 2024-04-25T00:45:05.073 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_call": 0, 2024-04-25T00:45:05.074 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_watch": 0, 2024-04-25T00:45:05.074 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_notify": 0, 2024-04-25T00:45:05.074 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_src_cmpxattr": 0, 2024-04-25T00:45:05.074 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_pgls": 0, 2024-04-25T00:45:05.074 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_pgls_filter": 0, 2024-04-25T00:45:05.074 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_other": 0, 2024-04-25T00:45:05.074 INFO:teuthology.orchestra.run.smithi121.stdout: "linger_active": 0, 2024-04-25T00:45:05.074 INFO:teuthology.orchestra.run.smithi121.stdout: "linger_send": 0, 2024-04-25T00:45:05.074 INFO:teuthology.orchestra.run.smithi121.stdout: "linger_resend": 0, 2024-04-25T00:45:05.075 INFO:teuthology.orchestra.run.smithi121.stdout: "linger_ping": 0, 2024-04-25T00:45:05.075 INFO:teuthology.orchestra.run.smithi121.stdout: "poolop_active": 0, 2024-04-25T00:45:05.075 INFO:teuthology.orchestra.run.smithi121.stdout: "poolop_send": 0, 2024-04-25T00:45:05.075 INFO:teuthology.orchestra.run.smithi121.stdout: "poolop_resend": 0, 2024-04-25T00:45:05.075 INFO:teuthology.orchestra.run.smithi121.stdout: "poolstat_active": 0, 2024-04-25T00:45:05.075 INFO:teuthology.orchestra.run.smithi121.stdout: "poolstat_send": 0, 2024-04-25T00:45:05.075 INFO:teuthology.orchestra.run.smithi121.stdout: "poolstat_resend": 0, 2024-04-25T00:45:05.075 INFO:teuthology.orchestra.run.smithi121.stdout: "statfs_active": 0, 2024-04-25T00:45:05.075 INFO:teuthology.orchestra.run.smithi121.stdout: "statfs_send": 0, 2024-04-25T00:45:05.075 INFO:teuthology.orchestra.run.smithi121.stdout: "statfs_resend": 0, 2024-04-25T00:45:05.075 INFO:teuthology.orchestra.run.smithi121.stdout: "command_active": 0, 2024-04-25T00:45:05.075 INFO:teuthology.orchestra.run.smithi121.stdout: "command_send": 0, 2024-04-25T00:45:05.075 INFO:teuthology.orchestra.run.smithi121.stdout: "command_resend": 0, 2024-04-25T00:45:05.076 INFO:teuthology.orchestra.run.smithi121.stdout: "map_epoch": 105, 2024-04-25T00:45:05.076 INFO:teuthology.orchestra.run.smithi121.stdout: "map_full": 0, 2024-04-25T00:45:05.076 INFO:teuthology.orchestra.run.smithi121.stdout: "map_inc": 21, 2024-04-25T00:45:05.076 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_sessions": 12, 2024-04-25T00:45:05.076 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_session_open": 12, 2024-04-25T00:45:05.076 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_session_close": 0, 2024-04-25T00:45:05.076 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_laggy": 0, 2024-04-25T00:45:05.076 INFO:teuthology.orchestra.run.smithi121.stdout: "omap_wr": 262, 2024-04-25T00:45:05.076 INFO:teuthology.orchestra.run.smithi121.stdout: "omap_rd": 0, 2024-04-25T00:45:05.076 INFO:teuthology.orchestra.run.smithi121.stdout: "omap_del": 19 2024-04-25T00:45:05.076 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.076 INFO:teuthology.orchestra.run.smithi121.stdout: "oft": { 2024-04-25T00:45:05.076 INFO:teuthology.orchestra.run.smithi121.stdout: "omap_total_objs": 1, 2024-04-25T00:45:05.076 INFO:teuthology.orchestra.run.smithi121.stdout: "omap_total_kv_pairs": 0, 2024-04-25T00:45:05.076 INFO:teuthology.orchestra.run.smithi121.stdout: "omap_total_updates": 0, 2024-04-25T00:45:05.076 INFO:teuthology.orchestra.run.smithi121.stdout: "omap_total_removes": 0 2024-04-25T00:45:05.077 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.077 INFO:teuthology.orchestra.run.smithi121.stdout: "osd-slow-ops": { 2024-04-25T00:45:05.077 INFO:teuthology.orchestra.run.smithi121.stdout: "slow_ops_count": 4 2024-04-25T00:45:05.077 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.077 INFO:teuthology.orchestra.run.smithi121.stdout: "purge_queue": { 2024-04-25T00:45:05.077 INFO:teuthology.orchestra.run.smithi121.stdout: "pq_executing_ops": 0, 2024-04-25T00:45:05.077 INFO:teuthology.orchestra.run.smithi121.stdout: "pq_executing_ops_high_water": 516, 2024-04-25T00:45:05.077 INFO:teuthology.orchestra.run.smithi121.stdout: "pq_executing": 0, 2024-04-25T00:45:05.077 INFO:teuthology.orchestra.run.smithi121.stdout: "pq_executing_high_water": 2, 2024-04-25T00:45:05.077 INFO:teuthology.orchestra.run.smithi121.stdout: "pq_executed_ops": 1548, 2024-04-25T00:45:05.077 INFO:teuthology.orchestra.run.smithi121.stdout: "pq_executed": 6, 2024-04-25T00:45:05.077 INFO:teuthology.orchestra.run.smithi121.stdout: "pq_item_in_journal": 0 2024-04-25T00:45:05.077 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.077 INFO:teuthology.orchestra.run.smithi121.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-04-25T00:45:05.078 INFO:teuthology.orchestra.run.smithi121.stdout: "val": 0, 2024-04-25T00:45:05.078 INFO:teuthology.orchestra.run.smithi121.stdout: "max": 104857600, 2024-04-25T00:45:05.078 INFO:teuthology.orchestra.run.smithi121.stdout: "get_started": 0, 2024-04-25T00:45:05.078 INFO:teuthology.orchestra.run.smithi121.stdout: "get": 177923, 2024-04-25T00:45:05.078 INFO:teuthology.orchestra.run.smithi121.stdout: "get_sum": 42852986, 2024-04-25T00:45:05.078 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:05.078 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_success": 177923, 2024-04-25T00:45:05.078 INFO:teuthology.orchestra.run.smithi121.stdout: "take": 0, 2024-04-25T00:45:05.078 INFO:teuthology.orchestra.run.smithi121.stdout: "take_sum": 0, 2024-04-25T00:45:05.078 INFO:teuthology.orchestra.run.smithi121.stdout: "put": 177923, 2024-04-25T00:45:05.078 INFO:teuthology.orchestra.run.smithi121.stdout: "put_sum": 42852986, 2024-04-25T00:45:05.078 INFO:teuthology.orchestra.run.smithi121.stdout: "wait": { 2024-04-25T00:45:05.078 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:05.078 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:05.079 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:05.079 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:05.079 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.079 INFO:teuthology.orchestra.run.smithi121.stdout: "throttle-objecter_bytes": { 2024-04-25T00:45:05.079 INFO:teuthology.orchestra.run.smithi121.stdout: "val": 0, 2024-04-25T00:45:05.079 INFO:teuthology.orchestra.run.smithi121.stdout: "max": 104857600, 2024-04-25T00:45:05.079 INFO:teuthology.orchestra.run.smithi121.stdout: "get_started": 0, 2024-04-25T00:45:05.079 INFO:teuthology.orchestra.run.smithi121.stdout: "get": 0, 2024-04-25T00:45:05.079 INFO:teuthology.orchestra.run.smithi121.stdout: "get_sum": 0, 2024-04-25T00:45:05.079 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:05.079 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_success": 0, 2024-04-25T00:45:05.079 INFO:teuthology.orchestra.run.smithi121.stdout: "take": 12492, 2024-04-25T00:45:05.079 INFO:teuthology.orchestra.run.smithi121.stdout: "take_sum": 360332881, 2024-04-25T00:45:05.079 INFO:teuthology.orchestra.run.smithi121.stdout: "put": 12492, 2024-04-25T00:45:05.080 INFO:teuthology.orchestra.run.smithi121.stdout: "put_sum": 360332881, 2024-04-25T00:45:05.080 INFO:teuthology.orchestra.run.smithi121.stdout: "wait": { 2024-04-25T00:45:05.080 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:05.080 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:05.080 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:05.080 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:05.080 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.080 INFO:teuthology.orchestra.run.smithi121.stdout: "throttle-objecter_ops": { 2024-04-25T00:45:05.080 INFO:teuthology.orchestra.run.smithi121.stdout: "val": 0, 2024-04-25T00:45:05.080 INFO:teuthology.orchestra.run.smithi121.stdout: "max": 1024, 2024-04-25T00:45:05.080 INFO:teuthology.orchestra.run.smithi121.stdout: "get_started": 0, 2024-04-25T00:45:05.080 INFO:teuthology.orchestra.run.smithi121.stdout: "get": 0, 2024-04-25T00:45:05.080 INFO:teuthology.orchestra.run.smithi121.stdout: "get_sum": 0, 2024-04-25T00:45:05.080 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:05.081 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_success": 0, 2024-04-25T00:45:05.081 INFO:teuthology.orchestra.run.smithi121.stdout: "take": 12492, 2024-04-25T00:45:05.081 INFO:teuthology.orchestra.run.smithi121.stdout: "take_sum": 12492, 2024-04-25T00:45:05.081 INFO:teuthology.orchestra.run.smithi121.stdout: "put": 12492, 2024-04-25T00:45:05.081 INFO:teuthology.orchestra.run.smithi121.stdout: "put_sum": 12492, 2024-04-25T00:45:05.081 INFO:teuthology.orchestra.run.smithi121.stdout: "wait": { 2024-04-25T00:45:05.081 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:05.081 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:05.081 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:05.081 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:05.081 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.081 INFO:teuthology.orchestra.run.smithi121.stdout: "throttle-write_buf_throttle": { 2024-04-25T00:45:05.081 INFO:teuthology.orchestra.run.smithi121.stdout: "val": 0, 2024-04-25T00:45:05.082 INFO:teuthology.orchestra.run.smithi121.stdout: "max": 3758096384, 2024-04-25T00:45:05.082 INFO:teuthology.orchestra.run.smithi121.stdout: "get_started": 0, 2024-04-25T00:45:05.082 INFO:teuthology.orchestra.run.smithi121.stdout: "get": 6, 2024-04-25T00:45:05.082 INFO:teuthology.orchestra.run.smithi121.stdout: "get_sum": 942, 2024-04-25T00:45:05.082 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:05.082 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_success": 6, 2024-04-25T00:45:05.082 INFO:teuthology.orchestra.run.smithi121.stdout: "take": 0, 2024-04-25T00:45:05.082 INFO:teuthology.orchestra.run.smithi121.stdout: "take_sum": 0, 2024-04-25T00:45:05.082 INFO:teuthology.orchestra.run.smithi121.stdout: "put": 1, 2024-04-25T00:45:05.082 INFO:teuthology.orchestra.run.smithi121.stdout: "put_sum": 942, 2024-04-25T00:45:05.082 INFO:teuthology.orchestra.run.smithi121.stdout: "wait": { 2024-04-25T00:45:05.082 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:05.082 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:05.082 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:05.083 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:05.083 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:05.083 INFO:teuthology.orchestra.run.smithi121.stdout: "throttle-write_buf_throttle-0x564f5af10500": { 2024-04-25T00:45:05.083 INFO:teuthology.orchestra.run.smithi121.stdout: "val": 60, 2024-04-25T00:45:05.083 INFO:teuthology.orchestra.run.smithi121.stdout: "max": 3758096384, 2024-04-25T00:45:05.083 INFO:teuthology.orchestra.run.smithi121.stdout: "get_started": 0, 2024-04-25T00:45:05.083 INFO:teuthology.orchestra.run.smithi121.stdout: "get": 161829, 2024-04-25T00:45:05.083 INFO:teuthology.orchestra.run.smithi121.stdout: "get_sum": 359003822, 2024-04-25T00:45:05.083 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:05.083 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_success": 161829, 2024-04-25T00:45:05.083 INFO:teuthology.orchestra.run.smithi121.stdout: "take": 0, 2024-04-25T00:45:05.083 INFO:teuthology.orchestra.run.smithi121.stdout: "take_sum": 0, 2024-04-25T00:45:05.083 INFO:teuthology.orchestra.run.smithi121.stdout: "put": 9851, 2024-04-25T00:45:05.083 INFO:teuthology.orchestra.run.smithi121.stdout: "put_sum": 359003762, 2024-04-25T00:45:05.084 INFO:teuthology.orchestra.run.smithi121.stdout: "wait": { 2024-04-25T00:45:05.084 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:05.084 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:05.084 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:05.084 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:05.084 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:05.084 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-25T00:45:05.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:05 smithi032 ceph-mon[29357]: pgmap v367: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 12 KiB/s rd, 23 KiB/s wr, 20 op/s 2024-04-25T00:45:05.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:05 smithi121 ceph-mon[32231]: pgmap v367: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 12 KiB/s rd, 23 KiB/s wr, 20 op/s 2024-04-25T00:45:05.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:05 smithi130 ceph-mon[32525]: pgmap v367: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 12 KiB/s rd, 23 KiB/s wr, 20 op/s 2024-04-25T00:45:06.123 INFO:tasks.check_counter:Daemon mds.b mds.exported=5 2024-04-25T00:45:06.124 INFO:tasks.check_counter:Daemon mds.b mds.imported=4 2024-04-25T00:45:06.124 INFO:tasks.check_counter:Daemon mds.b mds.root_rsnaps=6 2024-04-25T00:45:06.124 INFO:tasks.check_counter:Daemon mds.b mds_server.req_mksnap_latency.avgcount=12 2024-04-25T00:45:06.124 INFO:tasks.check_counter:Daemon mds.b mds_server.req_rmsnap_latency.avgcount=6 2024-04-25T00:45:06.124 DEBUG:tasks.check_counter:Getting stats from e 2024-04-25T00:45:06.124 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph daemon mds.e perf dump 2024-04-25T00:45:06.368 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.b/config 2024-04-25T00:45:07.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:07 smithi032 ceph-mon[29357]: pgmap v368: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 14 KiB/s rd, 45 KiB/s wr, 24 op/s 2024-04-25T00:45:07.812 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:07 smithi121 ceph-mon[32231]: pgmap v368: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 14 KiB/s rd, 45 KiB/s wr, 24 op/s 2024-04-25T00:45:07.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:07 smithi130 ceph-mon[32525]: pgmap v368: 129 pgs: 129 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 14 KiB/s rd, 45 KiB/s wr, 24 op/s 2024-04-25T00:45:08.512 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-25T00:45:08.512 INFO:teuthology.orchestra.run.smithi121.stdout: "AsyncMessenger::Worker-0": { 2024-04-25T00:45:08.512 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_messages": 301, 2024-04-25T00:45:08.512 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_messages": 283, 2024-04-25T00:45:08.512 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_bytes": 5850998, 2024-04-25T00:45:08.512 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_bytes": 157205, 2024-04-25T00:45:08.512 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_created_connections": 6, 2024-04-25T00:45:08.513 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_active_connections": 2, 2024-04-25T00:45:08.513 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_total_time": 0.059186748, 2024-04-25T00:45:08.513 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_send_time": 0.015835389, 2024-04-25T00:45:08.513 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_recv_time": 0.043665574, 2024-04-25T00:45:08.513 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_fast_dispatch_time": 0.007873316, 2024-04-25T00:45:08.513 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:08.513 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 282, 2024-04-25T00:45:08.513 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.133964949, 2024-04-25T00:45:08.513 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000475053 2024-04-25T00:45:08.513 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.513 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:08.513 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.513 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.513 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.514 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.514 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_encrypted_bytes": 1650240, 2024-04-25T00:45:08.514 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_encrypted_bytes": 149808 2024-04-25T00:45:08.514 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.514 INFO:teuthology.orchestra.run.smithi121.stdout: "AsyncMessenger::Worker-1": { 2024-04-25T00:45:08.514 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_messages": 924, 2024-04-25T00:45:08.514 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_messages": 1101, 2024-04-25T00:45:08.514 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_bytes": 14095532, 2024-04-25T00:45:08.514 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_bytes": 764713, 2024-04-25T00:45:08.514 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_created_connections": 8, 2024-04-25T00:45:08.514 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_active_connections": 6, 2024-04-25T00:45:08.514 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_total_time": 0.098052481, 2024-04-25T00:45:08.514 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_send_time": 0.026571133, 2024-04-25T00:45:08.518 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_recv_time": 0.032737957, 2024-04-25T00:45:08.518 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_fast_dispatch_time": 0.030709194, 2024-04-25T00:45:08.518 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:08.518 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 1099, 2024-04-25T00:45:08.518 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.065802708, 2024-04-25T00:45:08.518 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000059875 2024-04-25T00:45:08.518 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.518 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:08.518 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.519 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.519 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.519 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.519 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_encrypted_bytes": 3312, 2024-04-25T00:45:08.519 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_encrypted_bytes": 472784 2024-04-25T00:45:08.519 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.519 INFO:teuthology.orchestra.run.smithi121.stdout: "AsyncMessenger::Worker-2": { 2024-04-25T00:45:08.519 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_messages": 1125, 2024-04-25T00:45:08.519 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_messages": 1127, 2024-04-25T00:45:08.519 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_bytes": 17121557, 2024-04-25T00:45:08.519 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_bytes": 357808, 2024-04-25T00:45:08.519 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_created_connections": 5, 2024-04-25T00:45:08.519 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_active_connections": 4, 2024-04-25T00:45:08.519 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_total_time": 0.136306868, 2024-04-25T00:45:08.520 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_send_time": 0.037775614, 2024-04-25T00:45:08.520 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_recv_time": 0.045877566, 2024-04-25T00:45:08.520 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_fast_dispatch_time": 0.041873420, 2024-04-25T00:45:08.520 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:08.520 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 1125, 2024-04-25T00:45:08.520 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.056695742, 2024-04-25T00:45:08.520 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000050396 2024-04-25T00:45:08.520 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.520 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:08.520 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.520 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.520 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.520 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.521 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_encrypted_bytes": 0, 2024-04-25T00:45:08.521 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_encrypted_bytes": 0 2024-04-25T00:45:08.521 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.521 INFO:teuthology.orchestra.run.smithi121.stdout: "cct": { 2024-04-25T00:45:08.521 INFO:teuthology.orchestra.run.smithi121.stdout: "total_workers": 1, 2024-04-25T00:45:08.521 INFO:teuthology.orchestra.run.smithi121.stdout: "unhealthy_workers": 0 2024-04-25T00:45:08.521 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.521 INFO:teuthology.orchestra.run.smithi121.stdout: "finisher-MDSRank": { 2024-04-25T00:45:08.521 INFO:teuthology.orchestra.run.smithi121.stdout: "queue_len": 0, 2024-04-25T00:45:08.521 INFO:teuthology.orchestra.run.smithi121.stdout: "complete_latency": { 2024-04-25T00:45:08.521 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 3454, 2024-04-25T00:45:08.521 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.104311310, 2024-04-25T00:45:08.521 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000030200 2024-04-25T00:45:08.521 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:08.522 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.522 INFO:teuthology.orchestra.run.smithi121.stdout: "finisher-PurgeQueue": { 2024-04-25T00:45:08.522 INFO:teuthology.orchestra.run.smithi121.stdout: "queue_len": 0, 2024-04-25T00:45:08.522 INFO:teuthology.orchestra.run.smithi121.stdout: "complete_latency": { 2024-04-25T00:45:08.522 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.522 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.522 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.522 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:08.522 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.522 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-25T00:45:08.522 INFO:teuthology.orchestra.run.smithi121.stdout: "request": 0, 2024-04-25T00:45:08.522 INFO:teuthology.orchestra.run.smithi121.stdout: "reply": 0, 2024-04-25T00:45:08.522 INFO:teuthology.orchestra.run.smithi121.stdout: "reply_latency": { 2024-04-25T00:45:08.523 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.523 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.523 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.523 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.523 INFO:teuthology.orchestra.run.smithi121.stdout: "slow_reply": 0, 2024-04-25T00:45:08.523 INFO:teuthology.orchestra.run.smithi121.stdout: "forward": 0, 2024-04-25T00:45:08.523 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_fetch_complete": 0, 2024-04-25T00:45:08.523 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_fetch_keys": 0, 2024-04-25T00:45:08.523 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_commit": 0, 2024-04-25T00:45:08.523 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_split": 0, 2024-04-25T00:45:08.523 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_merge": 0, 2024-04-25T00:45:08.523 INFO:teuthology.orchestra.run.smithi121.stdout: "inodes": 4, 2024-04-25T00:45:08.523 INFO:teuthology.orchestra.run.smithi121.stdout: "inodes_top": 0, 2024-04-25T00:45:08.523 INFO:teuthology.orchestra.run.smithi121.stdout: "inodes_bottom": 4, 2024-04-25T00:45:08.524 INFO:teuthology.orchestra.run.smithi121.stdout: "inodes_pin_tail": 0, 2024-04-25T00:45:08.524 INFO:teuthology.orchestra.run.smithi121.stdout: "inodes_pinned": 4, 2024-04-25T00:45:08.524 INFO:teuthology.orchestra.run.smithi121.stdout: "inodes_expired": 0, 2024-04-25T00:45:08.524 INFO:teuthology.orchestra.run.smithi121.stdout: "inodes_with_caps": 0, 2024-04-25T00:45:08.524 INFO:teuthology.orchestra.run.smithi121.stdout: "caps": 0, 2024-04-25T00:45:08.524 INFO:teuthology.orchestra.run.smithi121.stdout: "subtrees": 1, 2024-04-25T00:45:08.524 INFO:teuthology.orchestra.run.smithi121.stdout: "traverse": 0, 2024-04-25T00:45:08.524 INFO:teuthology.orchestra.run.smithi121.stdout: "traverse_hit": 0, 2024-04-25T00:45:08.524 INFO:teuthology.orchestra.run.smithi121.stdout: "traverse_forward": 0, 2024-04-25T00:45:08.524 INFO:teuthology.orchestra.run.smithi121.stdout: "traverse_discover": 0, 2024-04-25T00:45:08.524 INFO:teuthology.orchestra.run.smithi121.stdout: "traverse_dir_fetch": 0, 2024-04-25T00:45:08.524 INFO:teuthology.orchestra.run.smithi121.stdout: "traverse_remote_ino": 0, 2024-04-25T00:45:08.524 INFO:teuthology.orchestra.run.smithi121.stdout: "traverse_lock": 0, 2024-04-25T00:45:08.524 INFO:teuthology.orchestra.run.smithi121.stdout: "load_cent": 0, 2024-04-25T00:45:08.524 INFO:teuthology.orchestra.run.smithi121.stdout: "q": 0, 2024-04-25T00:45:08.525 INFO:teuthology.orchestra.run.smithi121.stdout: "exported": 0, 2024-04-25T00:45:08.525 INFO:teuthology.orchestra.run.smithi121.stdout: "exported_inodes": 0, 2024-04-25T00:45:08.525 INFO:teuthology.orchestra.run.smithi121.stdout: "imported": 0, 2024-04-25T00:45:08.525 INFO:teuthology.orchestra.run.smithi121.stdout: "imported_inodes": 0, 2024-04-25T00:45:08.525 INFO:teuthology.orchestra.run.smithi121.stdout: "openino_dir_fetch": 0, 2024-04-25T00:45:08.525 INFO:teuthology.orchestra.run.smithi121.stdout: "openino_backtrace_fetch": 0, 2024-04-25T00:45:08.525 INFO:teuthology.orchestra.run.smithi121.stdout: "openino_peer_discover": 0, 2024-04-25T00:45:08.525 INFO:teuthology.orchestra.run.smithi121.stdout: "root_rfiles": 0, 2024-04-25T00:45:08.525 INFO:teuthology.orchestra.run.smithi121.stdout: "root_rbytes": 0, 2024-04-25T00:45:08.525 INFO:teuthology.orchestra.run.smithi121.stdout: "root_rsnaps": 0, 2024-04-25T00:45:08.525 INFO:teuthology.orchestra.run.smithi121.stdout: "scrub_backtrace_fetch": 0, 2024-04-25T00:45:08.525 INFO:teuthology.orchestra.run.smithi121.stdout: "scrub_set_tag": 0, 2024-04-25T00:45:08.525 INFO:teuthology.orchestra.run.smithi121.stdout: "scrub_backtrace_repaired": 0, 2024-04-25T00:45:08.525 INFO:teuthology.orchestra.run.smithi121.stdout: "scrub_inotable_repaired": 0, 2024-04-25T00:45:08.526 INFO:teuthology.orchestra.run.smithi121.stdout: "scrub_dir_inodes": 0, 2024-04-25T00:45:08.526 INFO:teuthology.orchestra.run.smithi121.stdout: "scrub_dir_base_inodes": 0, 2024-04-25T00:45:08.526 INFO:teuthology.orchestra.run.smithi121.stdout: "scrub_dirfrag_rstats": 0, 2024-04-25T00:45:08.526 INFO:teuthology.orchestra.run.smithi121.stdout: "scrub_file_inodes": 0, 2024-04-25T00:45:08.526 INFO:teuthology.orchestra.run.smithi121.stdout: "handle_inode_file_caps": 0, 2024-04-25T00:45:08.526 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph_cap_op_revoke": 0, 2024-04-25T00:45:08.526 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph_cap_op_grant": 0, 2024-04-25T00:45:08.526 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph_cap_op_trunc": 0, 2024-04-25T00:45:08.526 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-04-25T00:45:08.526 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph_cap_op_flush_ack": 0, 2024-04-25T00:45:08.526 INFO:teuthology.orchestra.run.smithi121.stdout: "handle_client_caps": 0, 2024-04-25T00:45:08.526 INFO:teuthology.orchestra.run.smithi121.stdout: "handle_client_caps_dirty": 0, 2024-04-25T00:45:08.526 INFO:teuthology.orchestra.run.smithi121.stdout: "handle_client_cap_release": 0, 2024-04-25T00:45:08.526 INFO:teuthology.orchestra.run.smithi121.stdout: "process_request_cap_release": 0 2024-04-25T00:45:08.526 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.527 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_cache": { 2024-04-25T00:45:08.527 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_update": 0, 2024-04-25T00:45:08.527 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_update_receipt": 0, 2024-04-25T00:45:08.527 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_try_discover": 0, 2024-04-25T00:45:08.527 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_send_discover": 0, 2024-04-25T00:45:08.527 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_handle_discover": 0, 2024-04-25T00:45:08.527 INFO:teuthology.orchestra.run.smithi121.stdout: "num_strays": 0, 2024-04-25T00:45:08.527 INFO:teuthology.orchestra.run.smithi121.stdout: "num_strays_delayed": 0, 2024-04-25T00:45:08.527 INFO:teuthology.orchestra.run.smithi121.stdout: "num_strays_enqueuing": 0, 2024-04-25T00:45:08.527 INFO:teuthology.orchestra.run.smithi121.stdout: "strays_created": 0, 2024-04-25T00:45:08.527 INFO:teuthology.orchestra.run.smithi121.stdout: "strays_enqueued": 0, 2024-04-25T00:45:08.527 INFO:teuthology.orchestra.run.smithi121.stdout: "strays_reintegrated": 0, 2024-04-25T00:45:08.527 INFO:teuthology.orchestra.run.smithi121.stdout: "strays_migrated": 0, 2024-04-25T00:45:08.528 INFO:teuthology.orchestra.run.smithi121.stdout: "num_recovering_processing": 0, 2024-04-25T00:45:08.528 INFO:teuthology.orchestra.run.smithi121.stdout: "num_recovering_enqueued": 0, 2024-04-25T00:45:08.528 INFO:teuthology.orchestra.run.smithi121.stdout: "num_recovering_prioritized": 0, 2024-04-25T00:45:08.528 INFO:teuthology.orchestra.run.smithi121.stdout: "recovery_started": 0, 2024-04-25T00:45:08.528 INFO:teuthology.orchestra.run.smithi121.stdout: "recovery_completed": 0, 2024-04-25T00:45:08.528 INFO:teuthology.orchestra.run.smithi121.stdout: "ireq_quiesce_path": 0, 2024-04-25T00:45:08.528 INFO:teuthology.orchestra.run.smithi121.stdout: "ireq_quiesce_inode": 0, 2024-04-25T00:45:08.528 INFO:teuthology.orchestra.run.smithi121.stdout: "ireq_enqueue_scrub": 0, 2024-04-25T00:45:08.528 INFO:teuthology.orchestra.run.smithi121.stdout: "ireq_exportdir": 0, 2024-04-25T00:45:08.528 INFO:teuthology.orchestra.run.smithi121.stdout: "ireq_flush": 0, 2024-04-25T00:45:08.528 INFO:teuthology.orchestra.run.smithi121.stdout: "ireq_fragmentdir": 0, 2024-04-25T00:45:08.528 INFO:teuthology.orchestra.run.smithi121.stdout: "ireq_fragstats": 0, 2024-04-25T00:45:08.528 INFO:teuthology.orchestra.run.smithi121.stdout: "ireq_inodestats": 0 2024-04-25T00:45:08.528 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.528 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_log": { 2024-04-25T00:45:08.529 INFO:teuthology.orchestra.run.smithi121.stdout: "evlrg": 0, 2024-04-25T00:45:08.529 INFO:teuthology.orchestra.run.smithi121.stdout: "evadd": 0, 2024-04-25T00:45:08.529 INFO:teuthology.orchestra.run.smithi121.stdout: "evex": 0, 2024-04-25T00:45:08.529 INFO:teuthology.orchestra.run.smithi121.stdout: "evtrm": 0, 2024-04-25T00:45:08.529 INFO:teuthology.orchestra.run.smithi121.stdout: "ev": 29191, 2024-04-25T00:45:08.529 INFO:teuthology.orchestra.run.smithi121.stdout: "evexg": 0, 2024-04-25T00:45:08.529 INFO:teuthology.orchestra.run.smithi121.stdout: "evexd": 0, 2024-04-25T00:45:08.529 INFO:teuthology.orchestra.run.smithi121.stdout: "segadd": 0, 2024-04-25T00:45:08.529 INFO:teuthology.orchestra.run.smithi121.stdout: "segex": 0, 2024-04-25T00:45:08.529 INFO:teuthology.orchestra.run.smithi121.stdout: "segtrm": 0, 2024-04-25T00:45:08.529 INFO:teuthology.orchestra.run.smithi121.stdout: "seg": 36, 2024-04-25T00:45:08.529 INFO:teuthology.orchestra.run.smithi121.stdout: "segmjr": 4, 2024-04-25T00:45:08.529 INFO:teuthology.orchestra.run.smithi121.stdout: "segexg": 0, 2024-04-25T00:45:08.529 INFO:teuthology.orchestra.run.smithi121.stdout: "segexd": 0, 2024-04-25T00:45:08.529 INFO:teuthology.orchestra.run.smithi121.stdout: "expos": 4194304, 2024-04-25T00:45:08.530 INFO:teuthology.orchestra.run.smithi121.stdout: "wrpos": 39026770, 2024-04-25T00:45:08.530 INFO:teuthology.orchestra.run.smithi121.stdout: "rdpos": 39024165, 2024-04-25T00:45:08.530 INFO:teuthology.orchestra.run.smithi121.stdout: "jlat": { 2024-04-25T00:45:08.530 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.530 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.530 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.530 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.530 INFO:teuthology.orchestra.run.smithi121.stdout: "replayed": 29191 2024-04-25T00:45:08.530 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.530 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_mem": { 2024-04-25T00:45:08.530 INFO:teuthology.orchestra.run.smithi121.stdout: "ino": 4, 2024-04-25T00:45:08.530 INFO:teuthology.orchestra.run.smithi121.stdout: "ino+": 37, 2024-04-25T00:45:08.530 INFO:teuthology.orchestra.run.smithi121.stdout: "ino-": 33, 2024-04-25T00:45:08.530 INFO:teuthology.orchestra.run.smithi121.stdout: "dir": 2, 2024-04-25T00:45:08.531 INFO:teuthology.orchestra.run.smithi121.stdout: "dir+": 12, 2024-04-25T00:45:08.531 INFO:teuthology.orchestra.run.smithi121.stdout: "dir-": 10, 2024-04-25T00:45:08.531 INFO:teuthology.orchestra.run.smithi121.stdout: "dn": 4, 2024-04-25T00:45:08.531 INFO:teuthology.orchestra.run.smithi121.stdout: "dn+": 37, 2024-04-25T00:45:08.531 INFO:teuthology.orchestra.run.smithi121.stdout: "dn-": 33, 2024-04-25T00:45:08.531 INFO:teuthology.orchestra.run.smithi121.stdout: "cap": 0, 2024-04-25T00:45:08.531 INFO:teuthology.orchestra.run.smithi121.stdout: "cap+": 0, 2024-04-25T00:45:08.531 INFO:teuthology.orchestra.run.smithi121.stdout: "cap-": 0, 2024-04-25T00:45:08.531 INFO:teuthology.orchestra.run.smithi121.stdout: "rss": 55896, 2024-04-25T00:45:08.531 INFO:teuthology.orchestra.run.smithi121.stdout: "heap": 215420 2024-04-25T00:45:08.531 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.531 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_server": { 2024-04-25T00:45:08.531 INFO:teuthology.orchestra.run.smithi121.stdout: "dispatch_client_request": 0, 2024-04-25T00:45:08.531 INFO:teuthology.orchestra.run.smithi121.stdout: "dispatch_server_request": 0, 2024-04-25T00:45:08.531 INFO:teuthology.orchestra.run.smithi121.stdout: "handle_client_request": 0, 2024-04-25T00:45:08.532 INFO:teuthology.orchestra.run.smithi121.stdout: "handle_client_session": 0, 2024-04-25T00:45:08.532 INFO:teuthology.orchestra.run.smithi121.stdout: "handle_peer_request": 0, 2024-04-25T00:45:08.532 INFO:teuthology.orchestra.run.smithi121.stdout: "req_create_latency": { 2024-04-25T00:45:08.532 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.532 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.532 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.532 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.532 INFO:teuthology.orchestra.run.smithi121.stdout: "req_getattr_latency": { 2024-04-25T00:45:08.532 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.532 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.532 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.532 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.532 INFO:teuthology.orchestra.run.smithi121.stdout: "req_getfilelock_latency": { 2024-04-25T00:45:08.532 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.532 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.533 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.533 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.533 INFO:teuthology.orchestra.run.smithi121.stdout: "req_link_latency": { 2024-04-25T00:45:08.533 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.533 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.533 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.533 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.533 INFO:teuthology.orchestra.run.smithi121.stdout: "req_lookup_latency": { 2024-04-25T00:45:08.533 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.533 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.533 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.533 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.533 INFO:teuthology.orchestra.run.smithi121.stdout: "req_lookuphash_latency": { 2024-04-25T00:45:08.533 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.533 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.534 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.534 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.534 INFO:teuthology.orchestra.run.smithi121.stdout: "req_lookupino_latency": { 2024-04-25T00:45:08.534 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.534 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.534 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.534 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.534 INFO:teuthology.orchestra.run.smithi121.stdout: "req_lookupname_latency": { 2024-04-25T00:45:08.534 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.534 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.534 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.534 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.534 INFO:teuthology.orchestra.run.smithi121.stdout: "req_lookupparent_latency": { 2024-04-25T00:45:08.534 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.535 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.535 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.535 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.535 INFO:teuthology.orchestra.run.smithi121.stdout: "req_lookupsnap_latency": { 2024-04-25T00:45:08.535 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.535 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.535 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.535 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.535 INFO:teuthology.orchestra.run.smithi121.stdout: "req_lssnap_latency": { 2024-04-25T00:45:08.535 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.535 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.535 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.535 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.536 INFO:teuthology.orchestra.run.smithi121.stdout: "req_mkdir_latency": { 2024-04-25T00:45:08.536 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.536 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.536 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.536 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.537 INFO:teuthology.orchestra.run.smithi121.stdout: "req_mknod_latency": { 2024-04-25T00:45:08.537 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.537 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.537 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.537 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.537 INFO:teuthology.orchestra.run.smithi121.stdout: "req_mksnap_latency": { 2024-04-25T00:45:08.537 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.537 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.537 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.537 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.537 INFO:teuthology.orchestra.run.smithi121.stdout: "req_open_latency": { 2024-04-25T00:45:08.537 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.537 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.537 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.537 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.538 INFO:teuthology.orchestra.run.smithi121.stdout: "req_readdir_latency": { 2024-04-25T00:45:08.538 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.538 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.538 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.538 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.538 INFO:teuthology.orchestra.run.smithi121.stdout: "req_rename_latency": { 2024-04-25T00:45:08.538 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.538 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.538 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.538 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.538 INFO:teuthology.orchestra.run.smithi121.stdout: "req_renamesnap_latency": { 2024-04-25T00:45:08.538 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.538 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.539 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.539 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.539 INFO:teuthology.orchestra.run.smithi121.stdout: "req_snapdiff_latency": { 2024-04-25T00:45:08.539 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.539 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.539 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.539 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.539 INFO:teuthology.orchestra.run.smithi121.stdout: "req_rmdir_latency": { 2024-04-25T00:45:08.539 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.539 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.539 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.539 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.539 INFO:teuthology.orchestra.run.smithi121.stdout: "req_rmsnap_latency": { 2024-04-25T00:45:08.539 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.539 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.540 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.540 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.540 INFO:teuthology.orchestra.run.smithi121.stdout: "req_rmxattr_latency": { 2024-04-25T00:45:08.540 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.540 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.540 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.540 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.540 INFO:teuthology.orchestra.run.smithi121.stdout: "req_setattr_latency": { 2024-04-25T00:45:08.540 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.540 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.540 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.540 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.540 INFO:teuthology.orchestra.run.smithi121.stdout: "req_setdirlayout_latency": { 2024-04-25T00:45:08.540 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.541 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.541 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.541 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.541 INFO:teuthology.orchestra.run.smithi121.stdout: "req_setfilelock_latency": { 2024-04-25T00:45:08.541 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.541 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.541 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.541 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.541 INFO:teuthology.orchestra.run.smithi121.stdout: "req_setlayout_latency": { 2024-04-25T00:45:08.541 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.541 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.541 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.541 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.541 INFO:teuthology.orchestra.run.smithi121.stdout: "req_setxattr_latency": { 2024-04-25T00:45:08.542 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.542 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.542 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.542 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.542 INFO:teuthology.orchestra.run.smithi121.stdout: "req_symlink_latency": { 2024-04-25T00:45:08.542 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.542 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.542 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.542 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.542 INFO:teuthology.orchestra.run.smithi121.stdout: "req_unlink_latency": { 2024-04-25T00:45:08.542 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.542 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.542 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.542 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.543 INFO:teuthology.orchestra.run.smithi121.stdout: "cap_revoke_eviction": 0, 2024-04-25T00:45:08.543 INFO:teuthology.orchestra.run.smithi121.stdout: "cap_acquisition_throttle": 0, 2024-04-25T00:45:08.543 INFO:teuthology.orchestra.run.smithi121.stdout: "req_getvxattr_latency": { 2024-04-25T00:45:08.543 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.543 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.543 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.543 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:08.543 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.543 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_sessions": { 2024-04-25T00:45:08.543 INFO:teuthology.orchestra.run.smithi121.stdout: "session_count": 1, 2024-04-25T00:45:08.543 INFO:teuthology.orchestra.run.smithi121.stdout: "session_add": 4, 2024-04-25T00:45:08.543 INFO:teuthology.orchestra.run.smithi121.stdout: "session_remove": 3, 2024-04-25T00:45:08.543 INFO:teuthology.orchestra.run.smithi121.stdout: "sessions_open": 3, 2024-04-25T00:45:08.543 INFO:teuthology.orchestra.run.smithi121.stdout: "sessions_stale": 0, 2024-04-25T00:45:08.544 INFO:teuthology.orchestra.run.smithi121.stdout: "total_load": 0, 2024-04-25T00:45:08.544 INFO:teuthology.orchestra.run.smithi121.stdout: "average_load": 0, 2024-04-25T00:45:08.544 INFO:teuthology.orchestra.run.smithi121.stdout: "avg_session_uptime": 4102, 2024-04-25T00:45:08.544 INFO:teuthology.orchestra.run.smithi121.stdout: "mdthresh_evicted": 0 2024-04-25T00:45:08.544 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.544 INFO:teuthology.orchestra.run.smithi121.stdout: "mempool": { 2024-04-25T00:45:08.544 INFO:teuthology.orchestra.run.smithi121.stdout: "bloom_filter_bytes": 0, 2024-04-25T00:45:08.544 INFO:teuthology.orchestra.run.smithi121.stdout: "bloom_filter_items": 0, 2024-04-25T00:45:08.544 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_alloc_bytes": 0, 2024-04-25T00:45:08.544 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_alloc_items": 0, 2024-04-25T00:45:08.544 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_data_bytes": 0, 2024-04-25T00:45:08.544 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_data_items": 0, 2024-04-25T00:45:08.544 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_onode_bytes": 0, 2024-04-25T00:45:08.544 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_onode_items": 0, 2024-04-25T00:45:08.544 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_meta_bytes": 0, 2024-04-25T00:45:08.545 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_meta_items": 0, 2024-04-25T00:45:08.545 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_other_bytes": 0, 2024-04-25T00:45:08.545 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_other_items": 0, 2024-04-25T00:45:08.545 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_buffer_bytes": 0, 2024-04-25T00:45:08.545 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_buffer_items": 0, 2024-04-25T00:45:08.545 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_extent_bytes": 0, 2024-04-25T00:45:08.545 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_extent_items": 0, 2024-04-25T00:45:08.545 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_blob_bytes": 0, 2024-04-25T00:45:08.545 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_blob_items": 0, 2024-04-25T00:45:08.545 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_shared_blob_bytes": 0, 2024-04-25T00:45:08.545 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_shared_blob_items": 0, 2024-04-25T00:45:08.545 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_inline_bl_bytes": 0, 2024-04-25T00:45:08.545 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_inline_bl_items": 0, 2024-04-25T00:45:08.545 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_fsck_bytes": 0, 2024-04-25T00:45:08.546 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_fsck_items": 0, 2024-04-25T00:45:08.546 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_txc_bytes": 0, 2024-04-25T00:45:08.546 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_txc_items": 0, 2024-04-25T00:45:08.546 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_writing_deferred_bytes": 0, 2024-04-25T00:45:08.546 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_writing_deferred_items": 0, 2024-04-25T00:45:08.546 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_writing_bytes": 0, 2024-04-25T00:45:08.546 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_writing_items": 0, 2024-04-25T00:45:08.546 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_bytes": 0, 2024-04-25T00:45:08.546 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_items": 0, 2024-04-25T00:45:08.546 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_file_reader_bytes": 0, 2024-04-25T00:45:08.546 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_file_reader_items": 0, 2024-04-25T00:45:08.546 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_file_writer_bytes": 0, 2024-04-25T00:45:08.547 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_file_writer_items": 0, 2024-04-25T00:45:08.547 INFO:teuthology.orchestra.run.smithi121.stdout: "buffer_anon_bytes": 215241, 2024-04-25T00:45:08.547 INFO:teuthology.orchestra.run.smithi121.stdout: "buffer_anon_items": 68, 2024-04-25T00:45:08.547 INFO:teuthology.orchestra.run.smithi121.stdout: "buffer_meta_bytes": 0, 2024-04-25T00:45:08.547 INFO:teuthology.orchestra.run.smithi121.stdout: "buffer_meta_items": 0, 2024-04-25T00:45:08.547 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_bytes": 0, 2024-04-25T00:45:08.547 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_items": 0, 2024-04-25T00:45:08.547 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mapbl_bytes": 0, 2024-04-25T00:45:08.547 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mapbl_items": 0, 2024-04-25T00:45:08.547 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pglog_bytes": 0, 2024-04-25T00:45:08.547 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pglog_items": 0, 2024-04-25T00:45:08.547 INFO:teuthology.orchestra.run.smithi121.stdout: "osdmap_bytes": 9584, 2024-04-25T00:45:08.547 INFO:teuthology.orchestra.run.smithi121.stdout: "osdmap_items": 183, 2024-04-25T00:45:08.547 INFO:teuthology.orchestra.run.smithi121.stdout: "osdmap_mapping_bytes": 0, 2024-04-25T00:45:08.548 INFO:teuthology.orchestra.run.smithi121.stdout: "osdmap_mapping_items": 0, 2024-04-25T00:45:08.548 INFO:teuthology.orchestra.run.smithi121.stdout: "pgmap_bytes": 0, 2024-04-25T00:45:08.548 INFO:teuthology.orchestra.run.smithi121.stdout: "pgmap_items": 0, 2024-04-25T00:45:08.548 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_co_bytes": 16480, 2024-04-25T00:45:08.548 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_co_items": 48, 2024-04-25T00:45:08.548 INFO:teuthology.orchestra.run.smithi121.stdout: "unittest_1_bytes": 0, 2024-04-25T00:45:08.548 INFO:teuthology.orchestra.run.smithi121.stdout: "unittest_1_items": 0, 2024-04-25T00:45:08.548 INFO:teuthology.orchestra.run.smithi121.stdout: "unittest_2_bytes": 0, 2024-04-25T00:45:08.548 INFO:teuthology.orchestra.run.smithi121.stdout: "unittest_2_items": 0 2024-04-25T00:45:08.548 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.548 INFO:teuthology.orchestra.run.smithi121.stdout: "objecter": { 2024-04-25T00:45:08.548 INFO:teuthology.orchestra.run.smithi121.stdout: "op_active": 0, 2024-04-25T00:45:08.548 INFO:teuthology.orchestra.run.smithi121.stdout: "op_laggy": 0, 2024-04-25T00:45:08.548 INFO:teuthology.orchestra.run.smithi121.stdout: "op_send": 2066, 2024-04-25T00:45:08.549 INFO:teuthology.orchestra.run.smithi121.stdout: "op_send_bytes": 21, 2024-04-25T00:45:08.549 INFO:teuthology.orchestra.run.smithi121.stdout: "op_resend": 3, 2024-04-25T00:45:08.549 INFO:teuthology.orchestra.run.smithi121.stdout: "op_reply": 2066, 2024-04-25T00:45:08.549 INFO:teuthology.orchestra.run.smithi121.stdout: "op_latency": { 2024-04-25T00:45:08.549 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 2066, 2024-04-25T00:45:08.549 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 150.469011004, 2024-04-25T00:45:08.549 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.072831079 2024-04-25T00:45:08.549 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.549 INFO:teuthology.orchestra.run.smithi121.stdout: "op_inflight": 0, 2024-04-25T00:45:08.549 INFO:teuthology.orchestra.run.smithi121.stdout: "oplen_avg": { 2024-04-25T00:45:08.549 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 2066, 2024-04-25T00:45:08.549 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 2067 2024-04-25T00:45:08.549 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.549 INFO:teuthology.orchestra.run.smithi121.stdout: "op": 2066, 2024-04-25T00:45:08.549 INFO:teuthology.orchestra.run.smithi121.stdout: "op_r": 2066, 2024-04-25T00:45:08.550 INFO:teuthology.orchestra.run.smithi121.stdout: "op_w": 0, 2024-04-25T00:45:08.550 INFO:teuthology.orchestra.run.smithi121.stdout: "op_rmw": 0, 2024-04-25T00:45:08.550 INFO:teuthology.orchestra.run.smithi121.stdout: "op_pg": 0, 2024-04-25T00:45:08.550 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_stat": 1285, 2024-04-25T00:45:08.550 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_create": 0, 2024-04-25T00:45:08.550 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_read": 779, 2024-04-25T00:45:08.550 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_write": 0, 2024-04-25T00:45:08.550 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_writefull": 0, 2024-04-25T00:45:08.550 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_writesame": 0, 2024-04-25T00:45:08.550 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_append": 0, 2024-04-25T00:45:08.550 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_zero": 0, 2024-04-25T00:45:08.550 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_truncate": 0, 2024-04-25T00:45:08.550 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_delete": 0, 2024-04-25T00:45:08.551 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_mapext": 0, 2024-04-25T00:45:08.551 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_sparse_read": 0, 2024-04-25T00:45:08.551 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_clonerange": 0, 2024-04-25T00:45:08.551 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_getxattr": 1, 2024-04-25T00:45:08.551 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_setxattr": 0, 2024-04-25T00:45:08.551 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_cmpxattr": 0, 2024-04-25T00:45:08.551 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_rmxattr": 0, 2024-04-25T00:45:08.551 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_resetxattrs": 0, 2024-04-25T00:45:08.551 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_call": 0, 2024-04-25T00:45:08.551 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_watch": 0, 2024-04-25T00:45:08.551 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_notify": 0, 2024-04-25T00:45:08.551 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_src_cmpxattr": 0, 2024-04-25T00:45:08.551 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_pgls": 0, 2024-04-25T00:45:08.551 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_pgls_filter": 0, 2024-04-25T00:45:08.551 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_other": 0, 2024-04-25T00:45:08.552 INFO:teuthology.orchestra.run.smithi121.stdout: "linger_active": 0, 2024-04-25T00:45:08.552 INFO:teuthology.orchestra.run.smithi121.stdout: "linger_send": 0, 2024-04-25T00:45:08.552 INFO:teuthology.orchestra.run.smithi121.stdout: "linger_resend": 0, 2024-04-25T00:45:08.552 INFO:teuthology.orchestra.run.smithi121.stdout: "linger_ping": 0, 2024-04-25T00:45:08.552 INFO:teuthology.orchestra.run.smithi121.stdout: "poolop_active": 0, 2024-04-25T00:45:08.552 INFO:teuthology.orchestra.run.smithi121.stdout: "poolop_send": 0, 2024-04-25T00:45:08.552 INFO:teuthology.orchestra.run.smithi121.stdout: "poolop_resend": 0, 2024-04-25T00:45:08.552 INFO:teuthology.orchestra.run.smithi121.stdout: "poolstat_active": 0, 2024-04-25T00:45:08.552 INFO:teuthology.orchestra.run.smithi121.stdout: "poolstat_send": 0, 2024-04-25T00:45:08.552 INFO:teuthology.orchestra.run.smithi121.stdout: "poolstat_resend": 0, 2024-04-25T00:45:08.552 INFO:teuthology.orchestra.run.smithi121.stdout: "statfs_active": 0, 2024-04-25T00:45:08.552 INFO:teuthology.orchestra.run.smithi121.stdout: "statfs_send": 0, 2024-04-25T00:45:08.552 INFO:teuthology.orchestra.run.smithi121.stdout: "statfs_resend": 0, 2024-04-25T00:45:08.552 INFO:teuthology.orchestra.run.smithi121.stdout: "command_active": 0, 2024-04-25T00:45:08.553 INFO:teuthology.orchestra.run.smithi121.stdout: "command_send": 0, 2024-04-25T00:45:08.553 INFO:teuthology.orchestra.run.smithi121.stdout: "command_resend": 0, 2024-04-25T00:45:08.553 INFO:teuthology.orchestra.run.smithi121.stdout: "map_epoch": 106, 2024-04-25T00:45:08.553 INFO:teuthology.orchestra.run.smithi121.stdout: "map_full": 0, 2024-04-25T00:45:08.553 INFO:teuthology.orchestra.run.smithi121.stdout: "map_inc": 17, 2024-04-25T00:45:08.553 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_sessions": 8, 2024-04-25T00:45:08.553 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_session_open": 8, 2024-04-25T00:45:08.553 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_session_close": 0, 2024-04-25T00:45:08.553 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_laggy": 0, 2024-04-25T00:45:08.553 INFO:teuthology.orchestra.run.smithi121.stdout: "omap_wr": 0, 2024-04-25T00:45:08.553 INFO:teuthology.orchestra.run.smithi121.stdout: "omap_rd": 2, 2024-04-25T00:45:08.553 INFO:teuthology.orchestra.run.smithi121.stdout: "omap_del": 0 2024-04-25T00:45:08.553 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.553 INFO:teuthology.orchestra.run.smithi121.stdout: "oft": { 2024-04-25T00:45:08.554 INFO:teuthology.orchestra.run.smithi121.stdout: "omap_total_objs": 0, 2024-04-25T00:45:08.554 INFO:teuthology.orchestra.run.smithi121.stdout: "omap_total_kv_pairs": 0, 2024-04-25T00:45:08.554 INFO:teuthology.orchestra.run.smithi121.stdout: "omap_total_updates": 0, 2024-04-25T00:45:08.554 INFO:teuthology.orchestra.run.smithi121.stdout: "omap_total_removes": 0 2024-04-25T00:45:08.554 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.554 INFO:teuthology.orchestra.run.smithi121.stdout: "osd-slow-ops": { 2024-04-25T00:45:08.554 INFO:teuthology.orchestra.run.smithi121.stdout: "slow_ops_count": 0 2024-04-25T00:45:08.554 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.554 INFO:teuthology.orchestra.run.smithi121.stdout: "purge_queue": { 2024-04-25T00:45:08.554 INFO:teuthology.orchestra.run.smithi121.stdout: "pq_executing_ops": 0, 2024-04-25T00:45:08.554 INFO:teuthology.orchestra.run.smithi121.stdout: "pq_executing_ops_high_water": 0, 2024-04-25T00:45:08.554 INFO:teuthology.orchestra.run.smithi121.stdout: "pq_executing": 0, 2024-04-25T00:45:08.554 INFO:teuthology.orchestra.run.smithi121.stdout: "pq_executing_high_water": 0, 2024-04-25T00:45:08.554 INFO:teuthology.orchestra.run.smithi121.stdout: "pq_executed_ops": 0, 2024-04-25T00:45:08.554 INFO:teuthology.orchestra.run.smithi121.stdout: "pq_executed": 0, 2024-04-25T00:45:08.555 INFO:teuthology.orchestra.run.smithi121.stdout: "pq_item_in_journal": 0 2024-04-25T00:45:08.555 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.555 INFO:teuthology.orchestra.run.smithi121.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-04-25T00:45:08.555 INFO:teuthology.orchestra.run.smithi121.stdout: "val": 0, 2024-04-25T00:45:08.555 INFO:teuthology.orchestra.run.smithi121.stdout: "max": 104857600, 2024-04-25T00:45:08.555 INFO:teuthology.orchestra.run.smithi121.stdout: "get_started": 0, 2024-04-25T00:45:08.555 INFO:teuthology.orchestra.run.smithi121.stdout: "get": 2346, 2024-04-25T00:45:08.555 INFO:teuthology.orchestra.run.smithi121.stdout: "get_sum": 36839734, 2024-04-25T00:45:08.555 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:08.555 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_success": 2346, 2024-04-25T00:45:08.555 INFO:teuthology.orchestra.run.smithi121.stdout: "take": 0, 2024-04-25T00:45:08.555 INFO:teuthology.orchestra.run.smithi121.stdout: "take_sum": 0, 2024-04-25T00:45:08.555 INFO:teuthology.orchestra.run.smithi121.stdout: "put": 2346, 2024-04-25T00:45:08.555 INFO:teuthology.orchestra.run.smithi121.stdout: "put_sum": 36839734, 2024-04-25T00:45:08.556 INFO:teuthology.orchestra.run.smithi121.stdout: "wait": { 2024-04-25T00:45:08.556 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.556 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.556 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.556 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:08.556 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.556 INFO:teuthology.orchestra.run.smithi121.stdout: "throttle-objecter_bytes": { 2024-04-25T00:45:08.556 INFO:teuthology.orchestra.run.smithi121.stdout: "val": 0, 2024-04-25T00:45:08.556 INFO:teuthology.orchestra.run.smithi121.stdout: "max": 104857600, 2024-04-25T00:45:08.556 INFO:teuthology.orchestra.run.smithi121.stdout: "get_started": 0, 2024-04-25T00:45:08.556 INFO:teuthology.orchestra.run.smithi121.stdout: "get": 0, 2024-04-25T00:45:08.556 INFO:teuthology.orchestra.run.smithi121.stdout: "get_sum": 0, 2024-04-25T00:45:08.556 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:08.556 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_success": 0, 2024-04-25T00:45:08.556 INFO:teuthology.orchestra.run.smithi121.stdout: "take": 2066, 2024-04-25T00:45:08.557 INFO:teuthology.orchestra.run.smithi121.stdout: "take_sum": 34832471, 2024-04-25T00:45:08.557 INFO:teuthology.orchestra.run.smithi121.stdout: "put": 2066, 2024-04-25T00:45:08.557 INFO:teuthology.orchestra.run.smithi121.stdout: "put_sum": 34832471, 2024-04-25T00:45:08.557 INFO:teuthology.orchestra.run.smithi121.stdout: "wait": { 2024-04-25T00:45:08.557 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.557 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.557 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.557 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:08.557 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.557 INFO:teuthology.orchestra.run.smithi121.stdout: "throttle-objecter_ops": { 2024-04-25T00:45:08.557 INFO:teuthology.orchestra.run.smithi121.stdout: "val": 0, 2024-04-25T00:45:08.557 INFO:teuthology.orchestra.run.smithi121.stdout: "max": 1024, 2024-04-25T00:45:08.557 INFO:teuthology.orchestra.run.smithi121.stdout: "get_started": 0, 2024-04-25T00:45:08.557 INFO:teuthology.orchestra.run.smithi121.stdout: "get": 0, 2024-04-25T00:45:08.557 INFO:teuthology.orchestra.run.smithi121.stdout: "get_sum": 0, 2024-04-25T00:45:08.558 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:08.558 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_success": 0, 2024-04-25T00:45:08.558 INFO:teuthology.orchestra.run.smithi121.stdout: "take": 2066, 2024-04-25T00:45:08.558 INFO:teuthology.orchestra.run.smithi121.stdout: "take_sum": 2066, 2024-04-25T00:45:08.558 INFO:teuthology.orchestra.run.smithi121.stdout: "put": 2066, 2024-04-25T00:45:08.558 INFO:teuthology.orchestra.run.smithi121.stdout: "put_sum": 2066, 2024-04-25T00:45:08.558 INFO:teuthology.orchestra.run.smithi121.stdout: "wait": { 2024-04-25T00:45:08.558 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.558 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.558 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.558 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:08.558 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.558 INFO:teuthology.orchestra.run.smithi121.stdout: "throttle-write_buf_throttle": { 2024-04-25T00:45:08.558 INFO:teuthology.orchestra.run.smithi121.stdout: "val": 0, 2024-04-25T00:45:08.559 INFO:teuthology.orchestra.run.smithi121.stdout: "max": 3758096384, 2024-04-25T00:45:08.559 INFO:teuthology.orchestra.run.smithi121.stdout: "get_started": 0, 2024-04-25T00:45:08.559 INFO:teuthology.orchestra.run.smithi121.stdout: "get": 0, 2024-04-25T00:45:08.559 INFO:teuthology.orchestra.run.smithi121.stdout: "get_sum": 0, 2024-04-25T00:45:08.559 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:08.559 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_success": 0, 2024-04-25T00:45:08.559 INFO:teuthology.orchestra.run.smithi121.stdout: "take": 0, 2024-04-25T00:45:08.559 INFO:teuthology.orchestra.run.smithi121.stdout: "take_sum": 0, 2024-04-25T00:45:08.559 INFO:teuthology.orchestra.run.smithi121.stdout: "put": 0, 2024-04-25T00:45:08.559 INFO:teuthology.orchestra.run.smithi121.stdout: "put_sum": 0, 2024-04-25T00:45:08.559 INFO:teuthology.orchestra.run.smithi121.stdout: "wait": { 2024-04-25T00:45:08.559 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.559 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.560 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.560 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:08.560 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:08.560 INFO:teuthology.orchestra.run.smithi121.stdout: "throttle-write_buf_throttle-0x55f0f73766e0": { 2024-04-25T00:45:08.560 INFO:teuthology.orchestra.run.smithi121.stdout: "val": 0, 2024-04-25T00:45:08.560 INFO:teuthology.orchestra.run.smithi121.stdout: "max": 3758096384, 2024-04-25T00:45:08.560 INFO:teuthology.orchestra.run.smithi121.stdout: "get_started": 0, 2024-04-25T00:45:08.560 INFO:teuthology.orchestra.run.smithi121.stdout: "get": 0, 2024-04-25T00:45:08.560 INFO:teuthology.orchestra.run.smithi121.stdout: "get_sum": 0, 2024-04-25T00:45:08.560 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:08.560 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_success": 0, 2024-04-25T00:45:08.560 INFO:teuthology.orchestra.run.smithi121.stdout: "take": 0, 2024-04-25T00:45:08.560 INFO:teuthology.orchestra.run.smithi121.stdout: "take_sum": 0, 2024-04-25T00:45:08.560 INFO:teuthology.orchestra.run.smithi121.stdout: "put": 0, 2024-04-25T00:45:08.560 INFO:teuthology.orchestra.run.smithi121.stdout: "put_sum": 0, 2024-04-25T00:45:08.560 INFO:teuthology.orchestra.run.smithi121.stdout: "wait": { 2024-04-25T00:45:08.561 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:08.561 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:08.561 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:08.561 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:08.561 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:08.561 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-25T00:45:08.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:08 smithi032 ceph-mon[29357]: osdmap e106: 12 total, 12 up, 12 in 2024-04-25T00:45:08.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:08 smithi121 ceph-mon[32231]: osdmap e106: 12 total, 12 up, 12 in 2024-04-25T00:45:08.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:08 smithi130 ceph-mon[32525]: osdmap e106: 12 total, 12 up, 12 in 2024-04-25T00:45:09.500 INFO:tasks.check_counter:Daemon mds.e mds.exported=0 2024-04-25T00:45:09.501 INFO:tasks.check_counter:Daemon mds.e mds.imported=0 2024-04-25T00:45:09.501 INFO:tasks.check_counter:Daemon mds.e mds.root_rsnaps=0 2024-04-25T00:45:09.501 INFO:tasks.check_counter:Daemon mds.e mds_server.req_mksnap_latency.avgcount=0 2024-04-25T00:45:09.501 INFO:tasks.check_counter:Daemon mds.e mds_server.req_rmsnap_latency.avgcount=0 2024-04-25T00:45:09.501 DEBUG:tasks.check_counter:Getting stats from h 2024-04-25T00:45:09.501 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph daemon mds.h perf dump 2024-04-25T00:45:09.718 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:09 smithi121 ceph-mon[32231]: pgmap v370: 129 pgs: 2 active+clean+snaptrim, 8 active+clean+snaptrim_wait, 119 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 26 op/s 2024-04-25T00:45:09.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:09 smithi032 ceph-mon[29357]: pgmap v370: 129 pgs: 2 active+clean+snaptrim, 8 active+clean+snaptrim_wait, 119 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 26 op/s 2024-04-25T00:45:09.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:09 smithi130 ceph-mon[32525]: pgmap v370: 129 pgs: 2 active+clean+snaptrim, 8 active+clean+snaptrim_wait, 119 active+clean; 22 GiB data, 54 GiB used, 1019 GiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 26 op/s 2024-04-25T00:45:09.909 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.b/config 2024-04-25T00:45:11.677 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:11 smithi121 ceph-mon[32231]: pgmap v371: 129 pgs: 7 active+clean+snaptrim, 13 active+clean+snaptrim_wait, 109 active+clean; 19 GiB data, 49 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 26 op/s 2024-04-25T00:45:11.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:11 smithi032 ceph-mon[29357]: pgmap v371: 129 pgs: 7 active+clean+snaptrim, 13 active+clean+snaptrim_wait, 109 active+clean; 19 GiB data, 49 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 26 op/s 2024-04-25T00:45:11.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:11 smithi130 ceph-mon[32525]: pgmap v371: 129 pgs: 7 active+clean+snaptrim, 13 active+clean+snaptrim_wait, 109 active+clean; 19 GiB data, 49 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 26 op/s 2024-04-25T00:45:11.957 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-25T00:45:11.958 INFO:teuthology.orchestra.run.smithi121.stdout: "AsyncMessenger::Worker-0": { 2024-04-25T00:45:11.958 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_messages": 13949, 2024-04-25T00:45:11.958 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_messages": 13954, 2024-04-25T00:45:11.958 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_bytes": 3546332, 2024-04-25T00:45:11.958 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_bytes": 20848429, 2024-04-25T00:45:11.958 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_created_connections": 9, 2024-04-25T00:45:11.958 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_active_connections": 2, 2024-04-25T00:45:11.958 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_total_time": 1.426630167, 2024-04-25T00:45:11.958 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_send_time": 0.368176959, 2024-04-25T00:45:11.958 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_recv_time": 0.255561301, 2024-04-25T00:45:11.958 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_fast_dispatch_time": 0.624570810, 2024-04-25T00:45:11.959 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:11.959 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 13958, 2024-04-25T00:45:11.959 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 4318.261945924, 2024-04-25T00:45:11.959 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.309375408 2024-04-25T00:45:11.959 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.959 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:11.959 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 1703, 2024-04-25T00:45:11.959 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000542252, 2024-04-25T00:45:11.959 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000318 2024-04-25T00:45:11.959 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.959 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_encrypted_bytes": 0, 2024-04-25T00:45:11.959 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_encrypted_bytes": 0 2024-04-25T00:45:11.959 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.960 INFO:teuthology.orchestra.run.smithi121.stdout: "AsyncMessenger::Worker-1": { 2024-04-25T00:45:11.960 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_messages": 10607, 2024-04-25T00:45:11.960 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_messages": 10734, 2024-04-25T00:45:11.960 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_bytes": 4197056, 2024-04-25T00:45:11.960 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_bytes": 17362236, 2024-04-25T00:45:11.960 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_created_connections": 7, 2024-04-25T00:45:11.960 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_active_connections": 6, 2024-04-25T00:45:11.960 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_total_time": 0.954372888, 2024-04-25T00:45:11.960 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_send_time": 0.255984124, 2024-04-25T00:45:11.960 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_recv_time": 0.182362023, 2024-04-25T00:45:11.960 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_fast_dispatch_time": 0.389966899, 2024-04-25T00:45:11.960 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:11.960 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 10734, 2024-04-25T00:45:11.960 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 4433.079719419, 2024-04-25T00:45:11.961 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.412994197 2024-04-25T00:45:11.961 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.961 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:11.961 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.961 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.961 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.961 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.961 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_encrypted_bytes": 1663776, 2024-04-25T00:45:11.961 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_encrypted_bytes": 500096 2024-04-25T00:45:11.961 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.961 INFO:teuthology.orchestra.run.smithi121.stdout: "AsyncMessenger::Worker-2": { 2024-04-25T00:45:11.961 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_messages": 32853, 2024-04-25T00:45:11.961 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_messages": 32949, 2024-04-25T00:45:11.962 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_bytes": 10410249, 2024-04-25T00:45:11.962 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_bytes": 15934564, 2024-04-25T00:45:11.962 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_created_connections": 33, 2024-04-25T00:45:11.962 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_active_connections": 9, 2024-04-25T00:45:11.962 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_total_time": 2.096111439, 2024-04-25T00:45:11.962 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_send_time": 0.745531552, 2024-04-25T00:45:11.962 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_recv_time": 1.899179705, 2024-04-25T00:45:11.962 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_fast_dispatch_time": 0.089079447, 2024-04-25T00:45:11.962 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:11.962 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 32940, 2024-04-25T00:45:11.962 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 6.729190115, 2024-04-25T00:45:11.962 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000204286 2024-04-25T00:45:11.962 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.962 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:11.963 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 49662, 2024-04-25T00:45:11.963 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.017846426, 2024-04-25T00:45:11.963 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000359 2024-04-25T00:45:11.963 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.963 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_encrypted_bytes": 5200, 2024-04-25T00:45:11.963 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_encrypted_bytes": 135744 2024-04-25T00:45:11.963 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.963 INFO:teuthology.orchestra.run.smithi121.stdout: "cct": { 2024-04-25T00:45:11.963 INFO:teuthology.orchestra.run.smithi121.stdout: "total_workers": 1, 2024-04-25T00:45:11.963 INFO:teuthology.orchestra.run.smithi121.stdout: "unhealthy_workers": 0 2024-04-25T00:45:11.963 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.963 INFO:teuthology.orchestra.run.smithi121.stdout: "finisher-MDSRank": { 2024-04-25T00:45:11.963 INFO:teuthology.orchestra.run.smithi121.stdout: "queue_len": 0, 2024-04-25T00:45:11.963 INFO:teuthology.orchestra.run.smithi121.stdout: "complete_latency": { 2024-04-25T00:45:11.963 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 28449, 2024-04-25T00:45:11.964 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 6.448279629, 2024-04-25T00:45:11.964 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000226661 2024-04-25T00:45:11.964 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:11.964 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.964 INFO:teuthology.orchestra.run.smithi121.stdout: "finisher-PurgeQueue": { 2024-04-25T00:45:11.964 INFO:teuthology.orchestra.run.smithi121.stdout: "queue_len": 0, 2024-04-25T00:45:11.964 INFO:teuthology.orchestra.run.smithi121.stdout: "complete_latency": { 2024-04-25T00:45:11.964 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 513, 2024-04-25T00:45:11.964 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.022991939, 2024-04-25T00:45:11.964 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000044818 2024-04-25T00:45:11.964 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:11.964 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.964 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-25T00:45:11.964 INFO:teuthology.orchestra.run.smithi121.stdout: "request": 16, 2024-04-25T00:45:11.964 INFO:teuthology.orchestra.run.smithi121.stdout: "reply": 12, 2024-04-25T00:45:11.965 INFO:teuthology.orchestra.run.smithi121.stdout: "reply_latency": { 2024-04-25T00:45:11.965 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 12, 2024-04-25T00:45:11.965 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.009151069, 2024-04-25T00:45:11.965 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000762589 2024-04-25T00:45:11.965 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.965 INFO:teuthology.orchestra.run.smithi121.stdout: "slow_reply": 0, 2024-04-25T00:45:11.965 INFO:teuthology.orchestra.run.smithi121.stdout: "forward": 4, 2024-04-25T00:45:11.965 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_fetch_complete": 0, 2024-04-25T00:45:11.965 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_fetch_keys": 0, 2024-04-25T00:45:11.965 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_commit": 11, 2024-04-25T00:45:11.965 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_split": 0, 2024-04-25T00:45:11.965 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_merge": 0, 2024-04-25T00:45:11.965 INFO:teuthology.orchestra.run.smithi121.stdout: "inodes": 15, 2024-04-25T00:45:11.965 INFO:teuthology.orchestra.run.smithi121.stdout: "inodes_top": 0, 2024-04-25T00:45:11.965 INFO:teuthology.orchestra.run.smithi121.stdout: "inodes_bottom": 1, 2024-04-25T00:45:11.966 INFO:teuthology.orchestra.run.smithi121.stdout: "inodes_pin_tail": 14, 2024-04-25T00:45:11.966 INFO:teuthology.orchestra.run.smithi121.stdout: "inodes_pinned": 14, 2024-04-25T00:45:11.966 INFO:teuthology.orchestra.run.smithi121.stdout: "inodes_expired": 31, 2024-04-25T00:45:11.966 INFO:teuthology.orchestra.run.smithi121.stdout: "inodes_with_caps": 1, 2024-04-25T00:45:11.966 INFO:teuthology.orchestra.run.smithi121.stdout: "caps": 1, 2024-04-25T00:45:11.966 INFO:teuthology.orchestra.run.smithi121.stdout: "subtrees": 3, 2024-04-25T00:45:11.966 INFO:teuthology.orchestra.run.smithi121.stdout: "traverse": 31, 2024-04-25T00:45:11.966 INFO:teuthology.orchestra.run.smithi121.stdout: "traverse_hit": 18, 2024-04-25T00:45:11.966 INFO:teuthology.orchestra.run.smithi121.stdout: "traverse_forward": 1, 2024-04-25T00:45:11.966 INFO:teuthology.orchestra.run.smithi121.stdout: "traverse_discover": 3, 2024-04-25T00:45:11.966 INFO:teuthology.orchestra.run.smithi121.stdout: "traverse_dir_fetch": 0, 2024-04-25T00:45:11.966 INFO:teuthology.orchestra.run.smithi121.stdout: "traverse_remote_ino": 0, 2024-04-25T00:45:11.966 INFO:teuthology.orchestra.run.smithi121.stdout: "traverse_lock": 0, 2024-04-25T00:45:11.966 INFO:teuthology.orchestra.run.smithi121.stdout: "load_cent": 0, 2024-04-25T00:45:11.967 INFO:teuthology.orchestra.run.smithi121.stdout: "q": 0, 2024-04-25T00:45:11.967 INFO:teuthology.orchestra.run.smithi121.stdout: "exported": 1, 2024-04-25T00:45:11.967 INFO:teuthology.orchestra.run.smithi121.stdout: "exported_inodes": 24, 2024-04-25T00:45:11.967 INFO:teuthology.orchestra.run.smithi121.stdout: "imported": 1, 2024-04-25T00:45:11.967 INFO:teuthology.orchestra.run.smithi121.stdout: "imported_inodes": 7, 2024-04-25T00:45:11.967 INFO:teuthology.orchestra.run.smithi121.stdout: "openino_dir_fetch": 0, 2024-04-25T00:45:11.967 INFO:teuthology.orchestra.run.smithi121.stdout: "openino_backtrace_fetch": 6, 2024-04-25T00:45:11.967 INFO:teuthology.orchestra.run.smithi121.stdout: "openino_peer_discover": 0, 2024-04-25T00:45:11.967 INFO:teuthology.orchestra.run.smithi121.stdout: "root_rfiles": 22, 2024-04-25T00:45:11.967 INFO:teuthology.orchestra.run.smithi121.stdout: "root_rbytes": 10738142561, 2024-04-25T00:45:11.967 INFO:teuthology.orchestra.run.smithi121.stdout: "root_rsnaps": 6, 2024-04-25T00:45:11.967 INFO:teuthology.orchestra.run.smithi121.stdout: "scrub_backtrace_fetch": 6, 2024-04-25T00:45:11.967 INFO:teuthology.orchestra.run.smithi121.stdout: "scrub_set_tag": 0, 2024-04-25T00:45:11.967 INFO:teuthology.orchestra.run.smithi121.stdout: "scrub_backtrace_repaired": 0, 2024-04-25T00:45:11.967 INFO:teuthology.orchestra.run.smithi121.stdout: "scrub_inotable_repaired": 0, 2024-04-25T00:45:11.968 INFO:teuthology.orchestra.run.smithi121.stdout: "scrub_dir_inodes": 0, 2024-04-25T00:45:11.968 INFO:teuthology.orchestra.run.smithi121.stdout: "scrub_dir_base_inodes": 0, 2024-04-25T00:45:11.968 INFO:teuthology.orchestra.run.smithi121.stdout: "scrub_dirfrag_rstats": 0, 2024-04-25T00:45:11.968 INFO:teuthology.orchestra.run.smithi121.stdout: "scrub_file_inodes": 6, 2024-04-25T00:45:11.968 INFO:teuthology.orchestra.run.smithi121.stdout: "handle_inode_file_caps": 0, 2024-04-25T00:45:11.968 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph_cap_op_revoke": 12, 2024-04-25T00:45:11.968 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph_cap_op_grant": 14, 2024-04-25T00:45:11.968 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph_cap_op_trunc": 0, 2024-04-25T00:45:11.968 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-04-25T00:45:11.968 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph_cap_op_flush_ack": 0, 2024-04-25T00:45:11.968 INFO:teuthology.orchestra.run.smithi121.stdout: "handle_client_caps": 29113, 2024-04-25T00:45:11.968 INFO:teuthology.orchestra.run.smithi121.stdout: "handle_client_caps_dirty": 29100, 2024-04-25T00:45:11.968 INFO:teuthology.orchestra.run.smithi121.stdout: "handle_client_cap_release": 3, 2024-04-25T00:45:11.968 INFO:teuthology.orchestra.run.smithi121.stdout: "process_request_cap_release": 9 2024-04-25T00:45:11.968 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.969 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_cache": { 2024-04-25T00:45:11.969 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_update": 0, 2024-04-25T00:45:11.969 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_update_receipt": 0, 2024-04-25T00:45:11.969 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_try_discover": 0, 2024-04-25T00:45:11.969 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_send_discover": 4, 2024-04-25T00:45:11.969 INFO:teuthology.orchestra.run.smithi121.stdout: "dir_handle_discover": 0, 2024-04-25T00:45:11.969 INFO:teuthology.orchestra.run.smithi121.stdout: "num_strays": 0, 2024-04-25T00:45:11.969 INFO:teuthology.orchestra.run.smithi121.stdout: "num_strays_delayed": 0, 2024-04-25T00:45:11.969 INFO:teuthology.orchestra.run.smithi121.stdout: "num_strays_enqueuing": 0, 2024-04-25T00:45:11.969 INFO:teuthology.orchestra.run.smithi121.stdout: "strays_created": 3, 2024-04-25T00:45:11.969 INFO:teuthology.orchestra.run.smithi121.stdout: "strays_enqueued": 3, 2024-04-25T00:45:11.969 INFO:teuthology.orchestra.run.smithi121.stdout: "strays_reintegrated": 0, 2024-04-25T00:45:11.969 INFO:teuthology.orchestra.run.smithi121.stdout: "strays_migrated": 0, 2024-04-25T00:45:11.969 INFO:teuthology.orchestra.run.smithi121.stdout: "num_recovering_processing": 0, 2024-04-25T00:45:11.970 INFO:teuthology.orchestra.run.smithi121.stdout: "num_recovering_enqueued": 0, 2024-04-25T00:45:11.970 INFO:teuthology.orchestra.run.smithi121.stdout: "num_recovering_prioritized": 0, 2024-04-25T00:45:11.970 INFO:teuthology.orchestra.run.smithi121.stdout: "recovery_started": 0, 2024-04-25T00:45:11.970 INFO:teuthology.orchestra.run.smithi121.stdout: "recovery_completed": 0, 2024-04-25T00:45:11.970 INFO:teuthology.orchestra.run.smithi121.stdout: "ireq_quiesce_path": 0, 2024-04-25T00:45:11.970 INFO:teuthology.orchestra.run.smithi121.stdout: "ireq_quiesce_inode": 0, 2024-04-25T00:45:11.970 INFO:teuthology.orchestra.run.smithi121.stdout: "ireq_enqueue_scrub": 0, 2024-04-25T00:45:11.970 INFO:teuthology.orchestra.run.smithi121.stdout: "ireq_exportdir": 1, 2024-04-25T00:45:11.970 INFO:teuthology.orchestra.run.smithi121.stdout: "ireq_flush": 0, 2024-04-25T00:45:11.970 INFO:teuthology.orchestra.run.smithi121.stdout: "ireq_fragmentdir": 0, 2024-04-25T00:45:11.970 INFO:teuthology.orchestra.run.smithi121.stdout: "ireq_fragstats": 0, 2024-04-25T00:45:11.970 INFO:teuthology.orchestra.run.smithi121.stdout: "ireq_inodestats": 0 2024-04-25T00:45:11.970 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.970 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_log": { 2024-04-25T00:45:11.971 INFO:teuthology.orchestra.run.smithi121.stdout: "evlrg": 0, 2024-04-25T00:45:11.971 INFO:teuthology.orchestra.run.smithi121.stdout: "evadd": 29191, 2024-04-25T00:45:11.971 INFO:teuthology.orchestra.run.smithi121.stdout: "evex": 0, 2024-04-25T00:45:11.971 INFO:teuthology.orchestra.run.smithi121.stdout: "evtrm": 0, 2024-04-25T00:45:11.971 INFO:teuthology.orchestra.run.smithi121.stdout: "ev": 29191, 2024-04-25T00:45:11.971 INFO:teuthology.orchestra.run.smithi121.stdout: "evexg": 0, 2024-04-25T00:45:11.971 INFO:teuthology.orchestra.run.smithi121.stdout: "evexd": 0, 2024-04-25T00:45:11.971 INFO:teuthology.orchestra.run.smithi121.stdout: "segadd": 36, 2024-04-25T00:45:11.971 INFO:teuthology.orchestra.run.smithi121.stdout: "segex": 0, 2024-04-25T00:45:11.971 INFO:teuthology.orchestra.run.smithi121.stdout: "segtrm": 0, 2024-04-25T00:45:11.971 INFO:teuthology.orchestra.run.smithi121.stdout: "seg": 36, 2024-04-25T00:45:11.971 INFO:teuthology.orchestra.run.smithi121.stdout: "segmjr": 4, 2024-04-25T00:45:11.971 INFO:teuthology.orchestra.run.smithi121.stdout: "segexg": 0, 2024-04-25T00:45:11.971 INFO:teuthology.orchestra.run.smithi121.stdout: "segexd": 0, 2024-04-25T00:45:11.971 INFO:teuthology.orchestra.run.smithi121.stdout: "expos": 4194304, 2024-04-25T00:45:11.972 INFO:teuthology.orchestra.run.smithi121.stdout: "wrpos": 39026770, 2024-04-25T00:45:11.972 INFO:teuthology.orchestra.run.smithi121.stdout: "rdpos": 0, 2024-04-25T00:45:11.972 INFO:teuthology.orchestra.run.smithi121.stdout: "jlat": { 2024-04-25T00:45:11.972 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 26391, 2024-04-25T00:45:11.972 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 116628.425724761, 2024-04-25T00:45:11.972 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 4.419249961 2024-04-25T00:45:11.972 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.972 INFO:teuthology.orchestra.run.smithi121.stdout: "replayed": 0 2024-04-25T00:45:11.972 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.972 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_mem": { 2024-04-25T00:45:11.972 INFO:teuthology.orchestra.run.smithi121.stdout: "ino": 19, 2024-04-25T00:45:11.972 INFO:teuthology.orchestra.run.smithi121.stdout: "ino+": 46, 2024-04-25T00:45:11.972 INFO:teuthology.orchestra.run.smithi121.stdout: "ino-": 27, 2024-04-25T00:45:11.972 INFO:teuthology.orchestra.run.smithi121.stdout: "dir": 18, 2024-04-25T00:45:11.973 INFO:teuthology.orchestra.run.smithi121.stdout: "dir+": 22, 2024-04-25T00:45:11.973 INFO:teuthology.orchestra.run.smithi121.stdout: "dir-": 4, 2024-04-25T00:45:11.973 INFO:teuthology.orchestra.run.smithi121.stdout: "dn": 15, 2024-04-25T00:45:11.973 INFO:teuthology.orchestra.run.smithi121.stdout: "dn+": 49, 2024-04-25T00:45:11.973 INFO:teuthology.orchestra.run.smithi121.stdout: "dn-": 34, 2024-04-25T00:45:11.973 INFO:teuthology.orchestra.run.smithi121.stdout: "cap": 1, 2024-04-25T00:45:11.973 INFO:teuthology.orchestra.run.smithi121.stdout: "cap+": 11, 2024-04-25T00:45:11.973 INFO:teuthology.orchestra.run.smithi121.stdout: "cap-": 10, 2024-04-25T00:45:11.973 INFO:teuthology.orchestra.run.smithi121.stdout: "rss": 70808, 2024-04-25T00:45:11.973 INFO:teuthology.orchestra.run.smithi121.stdout: "heap": 223612 2024-04-25T00:45:11.973 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.973 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_server": { 2024-04-25T00:45:11.973 INFO:teuthology.orchestra.run.smithi121.stdout: "dispatch_client_request": 27, 2024-04-25T00:45:11.973 INFO:teuthology.orchestra.run.smithi121.stdout: "dispatch_server_request": 1, 2024-04-25T00:45:11.973 INFO:teuthology.orchestra.run.smithi121.stdout: "handle_client_request": 16, 2024-04-25T00:45:11.974 INFO:teuthology.orchestra.run.smithi121.stdout: "handle_client_session": 22, 2024-04-25T00:45:11.974 INFO:teuthology.orchestra.run.smithi121.stdout: "handle_peer_request": 7, 2024-04-25T00:45:11.974 INFO:teuthology.orchestra.run.smithi121.stdout: "req_create_latency": { 2024-04-25T00:45:11.974 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.974 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.974 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.974 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.974 INFO:teuthology.orchestra.run.smithi121.stdout: "req_getattr_latency": { 2024-04-25T00:45:11.974 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 3, 2024-04-25T00:45:11.974 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.001931003, 2024-04-25T00:45:11.974 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000643667 2024-04-25T00:45:11.974 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.974 INFO:teuthology.orchestra.run.smithi121.stdout: "req_getfilelock_latency": { 2024-04-25T00:45:11.974 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.975 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.975 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.975 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.975 INFO:teuthology.orchestra.run.smithi121.stdout: "req_link_latency": { 2024-04-25T00:45:11.975 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.975 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.975 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.975 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.975 INFO:teuthology.orchestra.run.smithi121.stdout: "req_lookup_latency": { 2024-04-25T00:45:11.975 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 5, 2024-04-25T00:45:11.975 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.001554572, 2024-04-25T00:45:11.975 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000310914 2024-04-25T00:45:11.975 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.975 INFO:teuthology.orchestra.run.smithi121.stdout: "req_lookuphash_latency": { 2024-04-25T00:45:11.975 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.976 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.976 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.976 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.976 INFO:teuthology.orchestra.run.smithi121.stdout: "req_lookupino_latency": { 2024-04-25T00:45:11.976 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.976 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.976 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.976 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.976 INFO:teuthology.orchestra.run.smithi121.stdout: "req_lookupname_latency": { 2024-04-25T00:45:11.976 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.976 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.976 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.976 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.976 INFO:teuthology.orchestra.run.smithi121.stdout: "req_lookupparent_latency": { 2024-04-25T00:45:11.977 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.977 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.977 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.977 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.977 INFO:teuthology.orchestra.run.smithi121.stdout: "req_lookupsnap_latency": { 2024-04-25T00:45:11.977 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.977 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.977 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.977 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.977 INFO:teuthology.orchestra.run.smithi121.stdout: "req_lssnap_latency": { 2024-04-25T00:45:11.977 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.977 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.978 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.978 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.978 INFO:teuthology.orchestra.run.smithi121.stdout: "req_mkdir_latency": { 2024-04-25T00:45:11.978 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.978 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.978 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.978 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.979 INFO:teuthology.orchestra.run.smithi121.stdout: "req_mknod_latency": { 2024-04-25T00:45:11.979 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.979 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.979 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.979 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.979 INFO:teuthology.orchestra.run.smithi121.stdout: "req_mksnap_latency": { 2024-04-25T00:45:11.979 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.979 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.979 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.979 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.979 INFO:teuthology.orchestra.run.smithi121.stdout: "req_open_latency": { 2024-04-25T00:45:11.979 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.979 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.979 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.979 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.980 INFO:teuthology.orchestra.run.smithi121.stdout: "req_readdir_latency": { 2024-04-25T00:45:11.980 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 1, 2024-04-25T00:45:11.980 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000475899, 2024-04-25T00:45:11.980 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000475899 2024-04-25T00:45:11.980 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.980 INFO:teuthology.orchestra.run.smithi121.stdout: "req_rename_latency": { 2024-04-25T00:45:11.980 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.980 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.980 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.980 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.980 INFO:teuthology.orchestra.run.smithi121.stdout: "req_renamesnap_latency": { 2024-04-25T00:45:11.980 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.980 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.981 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.981 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.981 INFO:teuthology.orchestra.run.smithi121.stdout: "req_snapdiff_latency": { 2024-04-25T00:45:11.981 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.981 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.981 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.981 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.981 INFO:teuthology.orchestra.run.smithi121.stdout: "req_rmdir_latency": { 2024-04-25T00:45:11.981 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.981 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.981 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.981 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.981 INFO:teuthology.orchestra.run.smithi121.stdout: "req_rmsnap_latency": { 2024-04-25T00:45:11.981 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.981 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.982 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.982 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.982 INFO:teuthology.orchestra.run.smithi121.stdout: "req_rmxattr_latency": { 2024-04-25T00:45:11.982 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.982 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.982 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.982 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.982 INFO:teuthology.orchestra.run.smithi121.stdout: "req_setattr_latency": { 2024-04-25T00:45:11.982 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.982 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.982 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.982 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.982 INFO:teuthology.orchestra.run.smithi121.stdout: "req_setdirlayout_latency": { 2024-04-25T00:45:11.982 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.983 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.983 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.983 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.983 INFO:teuthology.orchestra.run.smithi121.stdout: "req_setfilelock_latency": { 2024-04-25T00:45:11.983 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.983 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.983 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.983 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.983 INFO:teuthology.orchestra.run.smithi121.stdout: "req_setlayout_latency": { 2024-04-25T00:45:11.983 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.983 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.983 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.983 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.983 INFO:teuthology.orchestra.run.smithi121.stdout: "req_setxattr_latency": { 2024-04-25T00:45:11.984 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.984 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.984 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.984 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.984 INFO:teuthology.orchestra.run.smithi121.stdout: "req_symlink_latency": { 2024-04-25T00:45:11.984 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.984 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.984 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.984 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.984 INFO:teuthology.orchestra.run.smithi121.stdout: "req_unlink_latency": { 2024-04-25T00:45:11.984 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 3, 2024-04-25T00:45:11.984 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.005189595, 2024-04-25T00:45:11.984 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.001729865 2024-04-25T00:45:11.984 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.984 INFO:teuthology.orchestra.run.smithi121.stdout: "cap_revoke_eviction": 0, 2024-04-25T00:45:11.985 INFO:teuthology.orchestra.run.smithi121.stdout: "cap_acquisition_throttle": 0, 2024-04-25T00:45:11.985 INFO:teuthology.orchestra.run.smithi121.stdout: "req_getvxattr_latency": { 2024-04-25T00:45:11.985 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.985 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.985 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.985 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:11.985 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.985 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_sessions": { 2024-04-25T00:45:11.985 INFO:teuthology.orchestra.run.smithi121.stdout: "session_count": 1, 2024-04-25T00:45:11.985 INFO:teuthology.orchestra.run.smithi121.stdout: "session_add": 4, 2024-04-25T00:45:11.985 INFO:teuthology.orchestra.run.smithi121.stdout: "session_remove": 3, 2024-04-25T00:45:11.985 INFO:teuthology.orchestra.run.smithi121.stdout: "sessions_open": 1, 2024-04-25T00:45:11.985 INFO:teuthology.orchestra.run.smithi121.stdout: "sessions_stale": 0, 2024-04-25T00:45:11.985 INFO:teuthology.orchestra.run.smithi121.stdout: "total_load": 7, 2024-04-25T00:45:11.986 INFO:teuthology.orchestra.run.smithi121.stdout: "average_load": 7, 2024-04-25T00:45:11.986 INFO:teuthology.orchestra.run.smithi121.stdout: "avg_session_uptime": 714, 2024-04-25T00:45:11.986 INFO:teuthology.orchestra.run.smithi121.stdout: "mdthresh_evicted": 0 2024-04-25T00:45:11.986 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.986 INFO:teuthology.orchestra.run.smithi121.stdout: "mempool": { 2024-04-25T00:45:11.986 INFO:teuthology.orchestra.run.smithi121.stdout: "bloom_filter_bytes": 0, 2024-04-25T00:45:11.986 INFO:teuthology.orchestra.run.smithi121.stdout: "bloom_filter_items": 0, 2024-04-25T00:45:11.986 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_alloc_bytes": 0, 2024-04-25T00:45:11.986 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_alloc_items": 0, 2024-04-25T00:45:11.986 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_data_bytes": 0, 2024-04-25T00:45:11.986 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_data_items": 0, 2024-04-25T00:45:11.986 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_onode_bytes": 0, 2024-04-25T00:45:11.986 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_onode_items": 0, 2024-04-25T00:45:11.986 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_meta_bytes": 0, 2024-04-25T00:45:11.987 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_meta_items": 0, 2024-04-25T00:45:11.987 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_other_bytes": 0, 2024-04-25T00:45:11.987 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_other_items": 0, 2024-04-25T00:45:11.987 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_buffer_bytes": 0, 2024-04-25T00:45:11.987 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_buffer_items": 0, 2024-04-25T00:45:11.987 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_extent_bytes": 0, 2024-04-25T00:45:11.987 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_extent_items": 0, 2024-04-25T00:45:11.987 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_blob_bytes": 0, 2024-04-25T00:45:11.987 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_blob_items": 0, 2024-04-25T00:45:11.987 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_shared_blob_bytes": 0, 2024-04-25T00:45:11.987 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_shared_blob_items": 0, 2024-04-25T00:45:11.987 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_inline_bl_bytes": 0, 2024-04-25T00:45:11.987 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_inline_bl_items": 0, 2024-04-25T00:45:11.987 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_fsck_bytes": 0, 2024-04-25T00:45:11.988 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_fsck_items": 0, 2024-04-25T00:45:11.988 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_txc_bytes": 0, 2024-04-25T00:45:11.988 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_txc_items": 0, 2024-04-25T00:45:11.988 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_writing_deferred_bytes": 0, 2024-04-25T00:45:11.988 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_writing_deferred_items": 0, 2024-04-25T00:45:11.988 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_writing_bytes": 0, 2024-04-25T00:45:11.988 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_writing_items": 0, 2024-04-25T00:45:11.988 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_bytes": 0, 2024-04-25T00:45:11.988 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_items": 0, 2024-04-25T00:45:11.988 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_file_reader_bytes": 0, 2024-04-25T00:45:11.988 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_file_reader_items": 0, 2024-04-25T00:45:11.988 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_file_writer_bytes": 0, 2024-04-25T00:45:11.988 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_file_writer_items": 0, 2024-04-25T00:45:11.988 INFO:teuthology.orchestra.run.smithi121.stdout: "buffer_anon_bytes": 296539, 2024-04-25T00:45:11.988 INFO:teuthology.orchestra.run.smithi121.stdout: "buffer_anon_items": 104, 2024-04-25T00:45:11.989 INFO:teuthology.orchestra.run.smithi121.stdout: "buffer_meta_bytes": 0, 2024-04-25T00:45:11.989 INFO:teuthology.orchestra.run.smithi121.stdout: "buffer_meta_items": 0, 2024-04-25T00:45:11.989 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_bytes": 0, 2024-04-25T00:45:11.989 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_items": 0, 2024-04-25T00:45:11.989 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mapbl_bytes": 0, 2024-04-25T00:45:11.989 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mapbl_items": 0, 2024-04-25T00:45:11.989 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pglog_bytes": 0, 2024-04-25T00:45:11.989 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pglog_items": 0, 2024-04-25T00:45:11.989 INFO:teuthology.orchestra.run.smithi121.stdout: "osdmap_bytes": 9944, 2024-04-25T00:45:11.989 INFO:teuthology.orchestra.run.smithi121.stdout: "osdmap_items": 192, 2024-04-25T00:45:11.989 INFO:teuthology.orchestra.run.smithi121.stdout: "osdmap_mapping_bytes": 0, 2024-04-25T00:45:11.989 INFO:teuthology.orchestra.run.smithi121.stdout: "osdmap_mapping_items": 0, 2024-04-25T00:45:11.989 INFO:teuthology.orchestra.run.smithi121.stdout: "pgmap_bytes": 0, 2024-04-25T00:45:11.989 INFO:teuthology.orchestra.run.smithi121.stdout: "pgmap_items": 0, 2024-04-25T00:45:11.989 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_co_bytes": 79813, 2024-04-25T00:45:11.990 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_co_items": 355, 2024-04-25T00:45:11.990 INFO:teuthology.orchestra.run.smithi121.stdout: "unittest_1_bytes": 0, 2024-04-25T00:45:11.990 INFO:teuthology.orchestra.run.smithi121.stdout: "unittest_1_items": 0, 2024-04-25T00:45:11.990 INFO:teuthology.orchestra.run.smithi121.stdout: "unittest_2_bytes": 0, 2024-04-25T00:45:11.990 INFO:teuthology.orchestra.run.smithi121.stdout: "unittest_2_items": 0 2024-04-25T00:45:11.990 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.990 INFO:teuthology.orchestra.run.smithi121.stdout: "objecter": { 2024-04-25T00:45:11.990 INFO:teuthology.orchestra.run.smithi121.stdout: "op_active": 0, 2024-04-25T00:45:11.990 INFO:teuthology.orchestra.run.smithi121.stdout: "op_laggy": 0, 2024-04-25T00:45:11.990 INFO:teuthology.orchestra.run.smithi121.stdout: "op_send": 27030, 2024-04-25T00:45:11.990 INFO:teuthology.orchestra.run.smithi121.stdout: "op_send_bytes": 34848122, 2024-04-25T00:45:11.990 INFO:teuthology.orchestra.run.smithi121.stdout: "op_resend": 33, 2024-04-25T00:45:11.990 INFO:teuthology.orchestra.run.smithi121.stdout: "op_reply": 27030, 2024-04-25T00:45:11.990 INFO:teuthology.orchestra.run.smithi121.stdout: "op_latency": { 2024-04-25T00:45:11.991 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 27030, 2024-04-25T00:45:11.991 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 116589.699221561, 2024-04-25T00:45:11.991 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 4.313344403 2024-04-25T00:45:11.991 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.991 INFO:teuthology.orchestra.run.smithi121.stdout: "op_inflight": 0, 2024-04-25T00:45:11.991 INFO:teuthology.orchestra.run.smithi121.stdout: "oplen_avg": { 2024-04-25T00:45:11.991 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 27030, 2024-04-25T00:45:11.991 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 27062 2024-04-25T00:45:11.991 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.991 INFO:teuthology.orchestra.run.smithi121.stdout: "op": 27030, 2024-04-25T00:45:11.991 INFO:teuthology.orchestra.run.smithi121.stdout: "op_r": 7, 2024-04-25T00:45:11.991 INFO:teuthology.orchestra.run.smithi121.stdout: "op_w": 27023, 2024-04-25T00:45:11.991 INFO:teuthology.orchestra.run.smithi121.stdout: "op_rmw": 0, 2024-04-25T00:45:11.991 INFO:teuthology.orchestra.run.smithi121.stdout: "op_pg": 0, 2024-04-25T00:45:11.991 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_stat": 0, 2024-04-25T00:45:11.991 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_create": 10, 2024-04-25T00:45:11.992 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_read": 1, 2024-04-25T00:45:11.992 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_write": 26392, 2024-04-25T00:45:11.992 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_writefull": 27, 2024-04-25T00:45:11.992 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_writesame": 0, 2024-04-25T00:45:11.992 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_append": 0, 2024-04-25T00:45:11.992 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_zero": 0, 2024-04-25T00:45:11.992 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_truncate": 0, 2024-04-25T00:45:11.992 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_delete": 538, 2024-04-25T00:45:11.992 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_mapext": 0, 2024-04-25T00:45:11.992 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_sparse_read": 0, 2024-04-25T00:45:11.992 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_clonerange": 0, 2024-04-25T00:45:11.992 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_getxattr": 6, 2024-04-25T00:45:11.992 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_setxattr": 20, 2024-04-25T00:45:11.992 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_cmpxattr": 0, 2024-04-25T00:45:11.992 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_rmxattr": 0, 2024-04-25T00:45:11.992 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_resetxattrs": 0, 2024-04-25T00:45:11.992 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_call": 0, 2024-04-25T00:45:11.993 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_watch": 0, 2024-04-25T00:45:11.993 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_notify": 0, 2024-04-25T00:45:11.993 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_src_cmpxattr": 0, 2024-04-25T00:45:11.993 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_pgls": 0, 2024-04-25T00:45:11.993 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_pgls_filter": 0, 2024-04-25T00:45:11.993 INFO:teuthology.orchestra.run.smithi121.stdout: "osdop_other": 0, 2024-04-25T00:45:11.993 INFO:teuthology.orchestra.run.smithi121.stdout: "linger_active": 0, 2024-04-25T00:45:11.993 INFO:teuthology.orchestra.run.smithi121.stdout: "linger_send": 0, 2024-04-25T00:45:11.993 INFO:teuthology.orchestra.run.smithi121.stdout: "linger_resend": 0, 2024-04-25T00:45:11.993 INFO:teuthology.orchestra.run.smithi121.stdout: "linger_ping": 0, 2024-04-25T00:45:11.993 INFO:teuthology.orchestra.run.smithi121.stdout: "poolop_active": 0, 2024-04-25T00:45:11.993 INFO:teuthology.orchestra.run.smithi121.stdout: "poolop_send": 0, 2024-04-25T00:45:11.993 INFO:teuthology.orchestra.run.smithi121.stdout: "poolop_resend": 0, 2024-04-25T00:45:11.993 INFO:teuthology.orchestra.run.smithi121.stdout: "poolstat_active": 0, 2024-04-25T00:45:11.993 INFO:teuthology.orchestra.run.smithi121.stdout: "poolstat_send": 0, 2024-04-25T00:45:11.993 INFO:teuthology.orchestra.run.smithi121.stdout: "poolstat_resend": 0, 2024-04-25T00:45:11.994 INFO:teuthology.orchestra.run.smithi121.stdout: "statfs_active": 0, 2024-04-25T00:45:11.994 INFO:teuthology.orchestra.run.smithi121.stdout: "statfs_send": 0, 2024-04-25T00:45:11.994 INFO:teuthology.orchestra.run.smithi121.stdout: "statfs_resend": 0, 2024-04-25T00:45:11.994 INFO:teuthology.orchestra.run.smithi121.stdout: "command_active": 0, 2024-04-25T00:45:11.994 INFO:teuthology.orchestra.run.smithi121.stdout: "command_send": 0, 2024-04-25T00:45:11.994 INFO:teuthology.orchestra.run.smithi121.stdout: "command_resend": 0, 2024-04-25T00:45:11.994 INFO:teuthology.orchestra.run.smithi121.stdout: "map_epoch": 106, 2024-04-25T00:45:11.994 INFO:teuthology.orchestra.run.smithi121.stdout: "map_full": 0, 2024-04-25T00:45:11.994 INFO:teuthology.orchestra.run.smithi121.stdout: "map_inc": 17, 2024-04-25T00:45:11.994 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_sessions": 12, 2024-04-25T00:45:11.994 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_session_open": 12, 2024-04-25T00:45:11.994 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_session_close": 0, 2024-04-25T00:45:11.994 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_laggy": 0, 2024-04-25T00:45:11.994 INFO:teuthology.orchestra.run.smithi121.stdout: "omap_wr": 63, 2024-04-25T00:45:11.995 INFO:teuthology.orchestra.run.smithi121.stdout: "omap_rd": 0, 2024-04-25T00:45:11.995 INFO:teuthology.orchestra.run.smithi121.stdout: "omap_del": 5 2024-04-25T00:45:11.995 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.995 INFO:teuthology.orchestra.run.smithi121.stdout: "oft": { 2024-04-25T00:45:11.995 INFO:teuthology.orchestra.run.smithi121.stdout: "omap_total_objs": 1, 2024-04-25T00:45:11.995 INFO:teuthology.orchestra.run.smithi121.stdout: "omap_total_kv_pairs": 0, 2024-04-25T00:45:11.995 INFO:teuthology.orchestra.run.smithi121.stdout: "omap_total_updates": 0, 2024-04-25T00:45:11.995 INFO:teuthology.orchestra.run.smithi121.stdout: "omap_total_removes": 0 2024-04-25T00:45:11.995 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.995 INFO:teuthology.orchestra.run.smithi121.stdout: "osd-slow-ops": { 2024-04-25T00:45:11.995 INFO:teuthology.orchestra.run.smithi121.stdout: "slow_ops_count": 0 2024-04-25T00:45:11.995 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.995 INFO:teuthology.orchestra.run.smithi121.stdout: "purge_queue": { 2024-04-25T00:45:11.995 INFO:teuthology.orchestra.run.smithi121.stdout: "pq_executing_ops": 0, 2024-04-25T00:45:11.995 INFO:teuthology.orchestra.run.smithi121.stdout: "pq_executing_ops_high_water": 514, 2024-04-25T00:45:11.995 INFO:teuthology.orchestra.run.smithi121.stdout: "pq_executing": 0, 2024-04-25T00:45:11.996 INFO:teuthology.orchestra.run.smithi121.stdout: "pq_executing_high_water": 2, 2024-04-25T00:45:11.996 INFO:teuthology.orchestra.run.smithi121.stdout: "pq_executed_ops": 518, 2024-04-25T00:45:11.996 INFO:teuthology.orchestra.run.smithi121.stdout: "pq_executed": 3, 2024-04-25T00:45:11.996 INFO:teuthology.orchestra.run.smithi121.stdout: "pq_item_in_journal": 0 2024-04-25T00:45:11.996 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.996 INFO:teuthology.orchestra.run.smithi121.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-04-25T00:45:11.996 INFO:teuthology.orchestra.run.smithi121.stdout: "val": 0, 2024-04-25T00:45:11.996 INFO:teuthology.orchestra.run.smithi121.stdout: "max": 104857600, 2024-04-25T00:45:11.996 INFO:teuthology.orchestra.run.smithi121.stdout: "get_started": 0, 2024-04-25T00:45:11.996 INFO:teuthology.orchestra.run.smithi121.stdout: "get": 57405, 2024-04-25T00:45:11.996 INFO:teuthology.orchestra.run.smithi121.stdout: "get_sum": 12967543, 2024-04-25T00:45:11.996 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:11.996 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_success": 57405, 2024-04-25T00:45:11.996 INFO:teuthology.orchestra.run.smithi121.stdout: "take": 0, 2024-04-25T00:45:11.997 INFO:teuthology.orchestra.run.smithi121.stdout: "take_sum": 0, 2024-04-25T00:45:11.997 INFO:teuthology.orchestra.run.smithi121.stdout: "put": 57405, 2024-04-25T00:45:11.997 INFO:teuthology.orchestra.run.smithi121.stdout: "put_sum": 12967543, 2024-04-25T00:45:11.997 INFO:teuthology.orchestra.run.smithi121.stdout: "wait": { 2024-04-25T00:45:11.997 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.997 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.997 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.997 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:11.997 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.997 INFO:teuthology.orchestra.run.smithi121.stdout: "throttle-objecter_bytes": { 2024-04-25T00:45:11.997 INFO:teuthology.orchestra.run.smithi121.stdout: "val": 0, 2024-04-25T00:45:11.997 INFO:teuthology.orchestra.run.smithi121.stdout: "max": 104857600, 2024-04-25T00:45:11.997 INFO:teuthology.orchestra.run.smithi121.stdout: "get_started": 0, 2024-04-25T00:45:11.997 INFO:teuthology.orchestra.run.smithi121.stdout: "get": 0, 2024-04-25T00:45:11.997 INFO:teuthology.orchestra.run.smithi121.stdout: "get_sum": 0, 2024-04-25T00:45:11.998 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:11.998 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_success": 0, 2024-04-25T00:45:11.998 INFO:teuthology.orchestra.run.smithi121.stdout: "take": 27030, 2024-04-25T00:45:11.998 INFO:teuthology.orchestra.run.smithi121.stdout: "take_sum": 34848593, 2024-04-25T00:45:11.998 INFO:teuthology.orchestra.run.smithi121.stdout: "put": 27030, 2024-04-25T00:45:11.998 INFO:teuthology.orchestra.run.smithi121.stdout: "put_sum": 34848593, 2024-04-25T00:45:11.998 INFO:teuthology.orchestra.run.smithi121.stdout: "wait": { 2024-04-25T00:45:11.998 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.998 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.998 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:11.998 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:11.998 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:11.998 INFO:teuthology.orchestra.run.smithi121.stdout: "throttle-objecter_ops": { 2024-04-25T00:45:11.998 INFO:teuthology.orchestra.run.smithi121.stdout: "val": 0, 2024-04-25T00:45:11.999 INFO:teuthology.orchestra.run.smithi121.stdout: "max": 1024, 2024-04-25T00:45:11.999 INFO:teuthology.orchestra.run.smithi121.stdout: "get_started": 0, 2024-04-25T00:45:11.999 INFO:teuthology.orchestra.run.smithi121.stdout: "get": 0, 2024-04-25T00:45:11.999 INFO:teuthology.orchestra.run.smithi121.stdout: "get_sum": 0, 2024-04-25T00:45:11.999 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:11.999 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_success": 0, 2024-04-25T00:45:11.999 INFO:teuthology.orchestra.run.smithi121.stdout: "take": 27030, 2024-04-25T00:45:11.999 INFO:teuthology.orchestra.run.smithi121.stdout: "take_sum": 27030, 2024-04-25T00:45:11.999 INFO:teuthology.orchestra.run.smithi121.stdout: "put": 27030, 2024-04-25T00:45:11.999 INFO:teuthology.orchestra.run.smithi121.stdout: "put_sum": 27030, 2024-04-25T00:45:11.999 INFO:teuthology.orchestra.run.smithi121.stdout: "wait": { 2024-04-25T00:45:11.999 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:11.999 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:11.999 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:12.001 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:12.001 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:12.002 INFO:teuthology.orchestra.run.smithi121.stdout: "throttle-write_buf_throttle": { 2024-04-25T00:45:12.002 INFO:teuthology.orchestra.run.smithi121.stdout: "val": 0, 2024-04-25T00:45:12.002 INFO:teuthology.orchestra.run.smithi121.stdout: "max": 3758096384, 2024-04-25T00:45:12.002 INFO:teuthology.orchestra.run.smithi121.stdout: "get_started": 0, 2024-04-25T00:45:12.002 INFO:teuthology.orchestra.run.smithi121.stdout: "get": 3, 2024-04-25T00:45:12.002 INFO:teuthology.orchestra.run.smithi121.stdout: "get_sum": 471, 2024-04-25T00:45:12.002 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:12.002 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_success": 3, 2024-04-25T00:45:12.002 INFO:teuthology.orchestra.run.smithi121.stdout: "take": 0, 2024-04-25T00:45:12.002 INFO:teuthology.orchestra.run.smithi121.stdout: "take_sum": 0, 2024-04-25T00:45:12.002 INFO:teuthology.orchestra.run.smithi121.stdout: "put": 1, 2024-04-25T00:45:12.002 INFO:teuthology.orchestra.run.smithi121.stdout: "put_sum": 471, 2024-04-25T00:45:12.002 INFO:teuthology.orchestra.run.smithi121.stdout: "wait": { 2024-04-25T00:45:12.002 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:12.002 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:12.003 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:12.003 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:12.003 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:12.003 INFO:teuthology.orchestra.run.smithi121.stdout: "throttle-write_buf_throttle-0x55d62a586500": { 2024-04-25T00:45:12.003 INFO:teuthology.orchestra.run.smithi121.stdout: "val": 0, 2024-04-25T00:45:12.003 INFO:teuthology.orchestra.run.smithi121.stdout: "max": 3758096384, 2024-04-25T00:45:12.003 INFO:teuthology.orchestra.run.smithi121.stdout: "get_started": 0, 2024-04-25T00:45:12.003 INFO:teuthology.orchestra.run.smithi121.stdout: "get": 29191, 2024-04-25T00:45:12.003 INFO:teuthology.orchestra.run.smithi121.stdout: "get_sum": 34832466, 2024-04-25T00:45:12.003 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:12.003 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_success": 29191, 2024-04-25T00:45:12.003 INFO:teuthology.orchestra.run.smithi121.stdout: "take": 0, 2024-04-25T00:45:12.003 INFO:teuthology.orchestra.run.smithi121.stdout: "take_sum": 0, 2024-04-25T00:45:12.003 INFO:teuthology.orchestra.run.smithi121.stdout: "put": 26391, 2024-04-25T00:45:12.004 INFO:teuthology.orchestra.run.smithi121.stdout: "put_sum": 34832466, 2024-04-25T00:45:12.004 INFO:teuthology.orchestra.run.smithi121.stdout: "wait": { 2024-04-25T00:45:12.004 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:12.004 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:12.004 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:12.004 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:12.004 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:12.004 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-25T00:45:13.102 INFO:tasks.check_counter:Daemon mds.h mds.exported=1 2024-04-25T00:45:13.102 INFO:tasks.check_counter:Daemon mds.h mds.imported=1 2024-04-25T00:45:13.103 INFO:tasks.check_counter:Daemon mds.h mds.root_rsnaps=6 2024-04-25T00:45:13.103 INFO:tasks.check_counter:Daemon mds.h mds_server.req_mksnap_latency.avgcount=0 2024-04-25T00:45:13.103 INFO:tasks.check_counter:Daemon mds.h mds_server.req_rmsnap_latency.avgcount=0 2024-04-25T00:45:13.103 DEBUG:tasks.check_counter:Getting stats from k 2024-04-25T00:45:13.103 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph daemon mds.k perf dump 2024-04-25T00:45:13.346 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.b/config 2024-04-25T00:45:13.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:13 smithi032 ceph-mon[29357]: pgmap v372: 129 pgs: 10 active+clean+snaptrim, 20 active+clean+snaptrim_wait, 99 active+clean; 15 GiB data, 42 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 28 KiB/s wr, 9 op/s 2024-04-25T00:45:13.795 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:13 smithi121 ceph-mon[32231]: pgmap v372: 129 pgs: 10 active+clean+snaptrim, 20 active+clean+snaptrim_wait, 99 active+clean; 15 GiB data, 42 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 28 KiB/s wr, 9 op/s 2024-04-25T00:45:13.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:13 smithi130 ceph-mon[32525]: pgmap v372: 129 pgs: 10 active+clean+snaptrim, 20 active+clean+snaptrim_wait, 99 active+clean; 15 GiB data, 42 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 28 KiB/s wr, 9 op/s 2024-04-25T00:45:15.567 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-25T00:45:15.568 INFO:teuthology.orchestra.run.smithi121.stdout: "AsyncMessenger::Worker-0": { 2024-04-25T00:45:15.568 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_messages": 253, 2024-04-25T00:45:15.568 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_messages": 242, 2024-04-25T00:45:15.568 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_bytes": 1589872, 2024-04-25T00:45:15.568 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_bytes": 148880, 2024-04-25T00:45:15.568 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_created_connections": 4, 2024-04-25T00:45:15.568 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_active_connections": 0, 2024-04-25T00:45:15.568 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_total_time": 0.050892778, 2024-04-25T00:45:15.568 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_send_time": 0.014506437, 2024-04-25T00:45:15.568 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_recv_time": 0.038446467, 2024-04-25T00:45:15.568 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_fast_dispatch_time": 0.006479682, 2024-04-25T00:45:15.568 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:15.568 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 241, 2024-04-25T00:45:15.568 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.014137664, 2024-04-25T00:45:15.568 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000058662 2024-04-25T00:45:15.569 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:15.569 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:15.569 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:15.569 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:15.569 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:15.569 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:15.569 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_encrypted_bytes": 1589872, 2024-04-25T00:45:15.569 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_encrypted_bytes": 148880 2024-04-25T00:45:15.569 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:15.569 INFO:teuthology.orchestra.run.smithi121.stdout: "AsyncMessenger::Worker-1": { 2024-04-25T00:45:15.569 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_messages": 2, 2024-04-25T00:45:15.569 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_messages": 185, 2024-04-25T00:45:15.569 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_bytes": 320, 2024-04-25T00:45:15.569 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_bytes": 187840, 2024-04-25T00:45:15.569 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_created_connections": 5, 2024-04-25T00:45:15.569 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_active_connections": 3, 2024-04-25T00:45:15.570 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_total_time": 0.011959014, 2024-04-25T00:45:15.570 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_send_time": 0.007955788, 2024-04-25T00:45:15.570 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_recv_time": 0.001619162, 2024-04-25T00:45:15.570 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-04-25T00:45:15.570 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:15.570 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 182, 2024-04-25T00:45:15.570 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.017258286, 2024-04-25T00:45:15.570 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000094825 2024-04-25T00:45:15.570 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:15.570 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:15.570 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:15.570 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:15.570 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:15.570 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:15.571 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_encrypted_bytes": 320, 2024-04-25T00:45:15.571 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_encrypted_bytes": 187840 2024-04-25T00:45:15.571 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:15.571 INFO:teuthology.orchestra.run.smithi121.stdout: "AsyncMessenger::Worker-2": { 2024-04-25T00:45:15.571 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_messages": 4, 2024-04-25T00:45:15.571 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_messages": 4, 2024-04-25T00:45:15.571 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_bytes": 2992, 2024-04-25T00:45:15.571 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_bytes": 448, 2024-04-25T00:45:15.571 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_created_connections": 2, 2024-04-25T00:45:15.571 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_active_connections": 1, 2024-04-25T00:45:15.571 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_total_time": 0.000668214, 2024-04-25T00:45:15.571 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_send_time": 0.000034061, 2024-04-25T00:45:15.571 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_recv_time": 0.000624861, 2024-04-25T00:45:15.571 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-04-25T00:45:15.571 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:15.572 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 3, 2024-04-25T00:45:15.572 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000681763, 2024-04-25T00:45:15.572 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000227254 2024-04-25T00:45:15.572 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:15.572 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:15.572 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:15.572 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:15.572 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:15.572 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:15.572 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_recv_encrypted_bytes": 2992, 2024-04-25T00:45:15.572 INFO:teuthology.orchestra.run.smithi121.stdout: "msgr_send_encrypted_bytes": 448 2024-04-25T00:45:15.572 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:15.573 INFO:teuthology.orchestra.run.smithi121.stdout: "cct": { 2024-04-25T00:45:15.573 INFO:teuthology.orchestra.run.smithi121.stdout: "total_workers": 0, 2024-04-25T00:45:15.573 INFO:teuthology.orchestra.run.smithi121.stdout: "unhealthy_workers": 0 2024-04-25T00:45:15.573 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:15.573 INFO:teuthology.orchestra.run.smithi121.stdout: "mempool": { 2024-04-25T00:45:15.573 INFO:teuthology.orchestra.run.smithi121.stdout: "bloom_filter_bytes": 0, 2024-04-25T00:45:15.573 INFO:teuthology.orchestra.run.smithi121.stdout: "bloom_filter_items": 0, 2024-04-25T00:45:15.573 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_alloc_bytes": 0, 2024-04-25T00:45:15.573 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_alloc_items": 0, 2024-04-25T00:45:15.573 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_data_bytes": 0, 2024-04-25T00:45:15.573 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_data_items": 0, 2024-04-25T00:45:15.573 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_onode_bytes": 0, 2024-04-25T00:45:15.573 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_onode_items": 0, 2024-04-25T00:45:15.573 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_meta_bytes": 0, 2024-04-25T00:45:15.573 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_meta_items": 0, 2024-04-25T00:45:15.574 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_other_bytes": 0, 2024-04-25T00:45:15.574 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_other_items": 0, 2024-04-25T00:45:15.574 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_buffer_bytes": 0, 2024-04-25T00:45:15.574 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_buffer_items": 0, 2024-04-25T00:45:15.574 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_extent_bytes": 0, 2024-04-25T00:45:15.574 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_extent_items": 0, 2024-04-25T00:45:15.574 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_blob_bytes": 0, 2024-04-25T00:45:15.574 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_blob_items": 0, 2024-04-25T00:45:15.574 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_shared_blob_bytes": 0, 2024-04-25T00:45:15.574 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_shared_blob_items": 0, 2024-04-25T00:45:15.574 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_inline_bl_bytes": 0, 2024-04-25T00:45:15.574 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_inline_bl_items": 0, 2024-04-25T00:45:15.574 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_fsck_bytes": 0, 2024-04-25T00:45:15.574 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_fsck_items": 0, 2024-04-25T00:45:15.575 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_txc_bytes": 0, 2024-04-25T00:45:15.575 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_txc_items": 0, 2024-04-25T00:45:15.575 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_writing_deferred_bytes": 0, 2024-04-25T00:45:15.575 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_writing_deferred_items": 0, 2024-04-25T00:45:15.575 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_writing_bytes": 0, 2024-04-25T00:45:15.575 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_writing_items": 0, 2024-04-25T00:45:15.575 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_bytes": 0, 2024-04-25T00:45:15.575 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_items": 0, 2024-04-25T00:45:15.575 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_file_reader_bytes": 0, 2024-04-25T00:45:15.575 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_file_reader_items": 0, 2024-04-25T00:45:15.575 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_file_writer_bytes": 0, 2024-04-25T00:45:15.575 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_file_writer_items": 0, 2024-04-25T00:45:15.575 INFO:teuthology.orchestra.run.smithi121.stdout: "buffer_anon_bytes": 148714, 2024-04-25T00:45:15.575 INFO:teuthology.orchestra.run.smithi121.stdout: "buffer_anon_items": 30, 2024-04-25T00:45:15.576 INFO:teuthology.orchestra.run.smithi121.stdout: "buffer_meta_bytes": 0, 2024-04-25T00:45:15.576 INFO:teuthology.orchestra.run.smithi121.stdout: "buffer_meta_items": 0, 2024-04-25T00:45:15.576 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_bytes": 0, 2024-04-25T00:45:15.576 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_items": 0, 2024-04-25T00:45:15.576 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mapbl_bytes": 0, 2024-04-25T00:45:15.576 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mapbl_items": 0, 2024-04-25T00:45:15.576 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pglog_bytes": 0, 2024-04-25T00:45:15.576 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pglog_items": 0, 2024-04-25T00:45:15.576 INFO:teuthology.orchestra.run.smithi121.stdout: "osdmap_bytes": 0, 2024-04-25T00:45:15.576 INFO:teuthology.orchestra.run.smithi121.stdout: "osdmap_items": 0, 2024-04-25T00:45:15.576 INFO:teuthology.orchestra.run.smithi121.stdout: "osdmap_mapping_bytes": 0, 2024-04-25T00:45:15.576 INFO:teuthology.orchestra.run.smithi121.stdout: "osdmap_mapping_items": 0, 2024-04-25T00:45:15.576 INFO:teuthology.orchestra.run.smithi121.stdout: "pgmap_bytes": 0, 2024-04-25T00:45:15.576 INFO:teuthology.orchestra.run.smithi121.stdout: "pgmap_items": 0, 2024-04-25T00:45:15.576 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_co_bytes": 928, 2024-04-25T00:45:15.577 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_co_items": 2, 2024-04-25T00:45:15.577 INFO:teuthology.orchestra.run.smithi121.stdout: "unittest_1_bytes": 0, 2024-04-25T00:45:15.577 INFO:teuthology.orchestra.run.smithi121.stdout: "unittest_1_items": 0, 2024-04-25T00:45:15.577 INFO:teuthology.orchestra.run.smithi121.stdout: "unittest_2_bytes": 0, 2024-04-25T00:45:15.577 INFO:teuthology.orchestra.run.smithi121.stdout: "unittest_2_items": 0 2024-04-25T00:45:15.577 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-25T00:45:15.577 INFO:teuthology.orchestra.run.smithi121.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-04-25T00:45:15.577 INFO:teuthology.orchestra.run.smithi121.stdout: "val": 0, 2024-04-25T00:45:15.577 INFO:teuthology.orchestra.run.smithi121.stdout: "max": 104857600, 2024-04-25T00:45:15.577 INFO:teuthology.orchestra.run.smithi121.stdout: "get_started": 0, 2024-04-25T00:45:15.577 INFO:teuthology.orchestra.run.smithi121.stdout: "get": 255, 2024-04-25T00:45:15.577 INFO:teuthology.orchestra.run.smithi121.stdout: "get_sum": 1554187, 2024-04-25T00:45:15.577 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:15.577 INFO:teuthology.orchestra.run.smithi121.stdout: "get_or_fail_success": 255, 2024-04-25T00:45:15.577 INFO:teuthology.orchestra.run.smithi121.stdout: "take": 0, 2024-04-25T00:45:15.578 INFO:teuthology.orchestra.run.smithi121.stdout: "take_sum": 0, 2024-04-25T00:45:15.578 INFO:teuthology.orchestra.run.smithi121.stdout: "put": 255, 2024-04-25T00:45:15.578 INFO:teuthology.orchestra.run.smithi121.stdout: "put_sum": 1554187, 2024-04-25T00:45:15.578 INFO:teuthology.orchestra.run.smithi121.stdout: "wait": { 2024-04-25T00:45:15.578 INFO:teuthology.orchestra.run.smithi121.stdout: "avgcount": 0, 2024-04-25T00:45:15.578 INFO:teuthology.orchestra.run.smithi121.stdout: "sum": 0.000000000, 2024-04-25T00:45:15.578 INFO:teuthology.orchestra.run.smithi121.stdout: "avgtime": 0.000000000 2024-04-25T00:45:15.578 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:15.578 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-25T00:45:15.578 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-25T00:45:15.579 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:15 smithi121 ceph-mon[32231]: pgmap v373: 129 pgs: 8 active+clean+snaptrim, 12 active+clean+snaptrim_wait, 109 active+clean; 14 GiB data, 40 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 KiB/s rd, 27 KiB/s wr, 9 op/s 2024-04-25T00:45:15.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:15 smithi032 ceph-mon[29357]: pgmap v373: 129 pgs: 8 active+clean+snaptrim, 12 active+clean+snaptrim_wait, 109 active+clean; 14 GiB data, 40 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 KiB/s rd, 27 KiB/s wr, 9 op/s 2024-04-25T00:45:15.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:15 smithi130 ceph-mon[32525]: pgmap v373: 129 pgs: 8 active+clean+snaptrim, 12 active+clean+snaptrim_wait, 109 active+clean; 14 GiB data, 40 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 KiB/s rd, 27 KiB/s wr, 9 op/s 2024-04-25T00:45:16.490 WARNING:tasks.check_counter:Counter 'mds.exported' not found on daemon mds.k 2024-04-25T00:45:16.491 WARNING:tasks.check_counter:Counter 'mds.imported' not found on daemon mds.k 2024-04-25T00:45:16.491 WARNING:tasks.check_counter:Counter 'mds.root_rsnaps' not found on daemon mds.k 2024-04-25T00:45:16.491 WARNING:tasks.check_counter:Counter 'mds_server.req_mksnap_latency.avgcount' not found on daemon mds.k 2024-04-25T00:45:16.491 WARNING:tasks.check_counter:Counter 'mds_server.req_rmsnap_latency.avgcount' not found on daemon mds.k 2024-04-25T00:45:16.491 DEBUG:tasks.check_counter:Getting stats from c 2024-04-25T00:45:16.491 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph daemon mds.c perf dump 2024-04-25T00:45:16.789 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.c/config 2024-04-25T00:45:17.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:17 smithi032 ceph-mon[29357]: pgmap v374: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 204 B/s wr, 6 op/s 2024-04-25T00:45:17.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:17 smithi032 ceph-mon[29357]: osdmap e107: 12 total, 12 up, 12 in 2024-04-25T00:45:17.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:17 smithi121 ceph-mon[32231]: pgmap v374: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 204 B/s wr, 6 op/s 2024-04-25T00:45:17.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:17 smithi121 ceph-mon[32231]: osdmap e107: 12 total, 12 up, 12 in 2024-04-25T00:45:17.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:17 smithi130 ceph-mon[32525]: pgmap v374: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 204 B/s wr, 6 op/s 2024-04-25T00:45:17.895 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:17 smithi130 ceph-mon[32525]: osdmap e107: 12 total, 12 up, 12 in 2024-04-25T00:45:19.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:19 smithi032 ceph-mon[29357]: pgmap v376: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:45:19.763 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:19 smithi130 ceph-mon[32525]: pgmap v376: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:45:19.763 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-25T00:45:19.764 INFO:teuthology.orchestra.run.smithi130.stdout: "AsyncMessenger::Worker-0": { 2024-04-25T00:45:19.764 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_messages": 1908, 2024-04-25T00:45:19.764 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages": 1887, 2024-04-25T00:45:19.764 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_bytes": 109503406, 2024-04-25T00:45:19.764 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_bytes": 667350, 2024-04-25T00:45:19.764 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_created_connections": 7, 2024-04-25T00:45:19.764 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_active_connections": 3, 2024-04-25T00:45:19.764 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_total_time": 0.306185884, 2024-04-25T00:45:19.764 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_send_time": 0.063033508, 2024-04-25T00:45:19.764 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_recv_time": 0.171077323, 2024-04-25T00:45:19.764 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_fast_dispatch_time": 0.066590507, 2024-04-25T00:45:19.764 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:19.764 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 1887, 2024-04-25T00:45:19.764 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.145684999, 2024-04-25T00:45:19.765 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000077204 2024-04-25T00:45:19.765 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.765 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:19.765 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.765 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.765 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.765 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.765 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_encrypted_bytes": 1654176, 2024-04-25T00:45:19.765 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_encrypted_bytes": 151232 2024-04-25T00:45:19.765 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.765 INFO:teuthology.orchestra.run.smithi130.stdout: "AsyncMessenger::Worker-1": { 2024-04-25T00:45:19.765 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_messages": 634, 2024-04-25T00:45:19.765 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages": 819, 2024-04-25T00:45:19.765 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_bytes": 126013483, 2024-04-25T00:45:19.765 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_bytes": 678886, 2024-04-25T00:45:19.766 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_created_connections": 9, 2024-04-25T00:45:19.766 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_active_connections": 7, 2024-04-25T00:45:19.766 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_total_time": 0.146959663, 2024-04-25T00:45:19.766 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_send_time": 0.022611579, 2024-04-25T00:45:19.766 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_recv_time": 0.092289662, 2024-04-25T00:45:19.766 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_fast_dispatch_time": 0.024536358, 2024-04-25T00:45:19.766 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:19.766 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 816, 2024-04-25T00:45:19.766 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.045113966, 2024-04-25T00:45:19.766 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000055286 2024-04-25T00:45:19.766 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.766 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:19.766 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.766 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.767 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.767 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.767 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_encrypted_bytes": 320, 2024-04-25T00:45:19.767 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_encrypted_bytes": 477536 2024-04-25T00:45:19.767 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.767 INFO:teuthology.orchestra.run.smithi130.stdout: "AsyncMessenger::Worker-2": { 2024-04-25T00:45:19.767 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_messages": 605, 2024-04-25T00:45:19.767 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages": 607, 2024-04-25T00:45:19.767 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_bytes": 125997665, 2024-04-25T00:45:19.767 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_bytes": 192397, 2024-04-25T00:45:19.767 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_created_connections": 6, 2024-04-25T00:45:19.767 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_active_connections": 5, 2024-04-25T00:45:19.767 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_total_time": 0.121289230, 2024-04-25T00:45:19.767 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_send_time": 0.017185050, 2024-04-25T00:45:19.768 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_recv_time": 0.078297803, 2024-04-25T00:45:19.768 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_fast_dispatch_time": 0.021525181, 2024-04-25T00:45:19.768 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:19.768 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 605, 2024-04-25T00:45:19.768 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.029470917, 2024-04-25T00:45:19.768 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000048712 2024-04-25T00:45:19.768 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.768 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:19.768 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.768 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.768 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.768 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.768 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_encrypted_bytes": 0, 2024-04-25T00:45:19.768 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_encrypted_bytes": 0 2024-04-25T00:45:19.769 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.769 INFO:teuthology.orchestra.run.smithi130.stdout: "cct": { 2024-04-25T00:45:19.769 INFO:teuthology.orchestra.run.smithi130.stdout: "total_workers": 1, 2024-04-25T00:45:19.769 INFO:teuthology.orchestra.run.smithi130.stdout: "unhealthy_workers": 0 2024-04-25T00:45:19.769 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.769 INFO:teuthology.orchestra.run.smithi130.stdout: "finisher-MDSRank": { 2024-04-25T00:45:19.769 INFO:teuthology.orchestra.run.smithi130.stdout: "queue_len": 0, 2024-04-25T00:45:19.769 INFO:teuthology.orchestra.run.smithi130.stdout: "complete_latency": { 2024-04-25T00:45:19.769 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 4525, 2024-04-25T00:45:19.769 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.132782876, 2024-04-25T00:45:19.769 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000029344 2024-04-25T00:45:19.769 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:19.769 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.769 INFO:teuthology.orchestra.run.smithi130.stdout: "finisher-PurgeQueue": { 2024-04-25T00:45:19.769 INFO:teuthology.orchestra.run.smithi130.stdout: "queue_len": 0, 2024-04-25T00:45:19.770 INFO:teuthology.orchestra.run.smithi130.stdout: "complete_latency": { 2024-04-25T00:45:19.770 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.770 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.770 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.770 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:19.770 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.770 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-04-25T00:45:19.770 INFO:teuthology.orchestra.run.smithi130.stdout: "request": 0, 2024-04-25T00:45:19.770 INFO:teuthology.orchestra.run.smithi130.stdout: "reply": 0, 2024-04-25T00:45:19.770 INFO:teuthology.orchestra.run.smithi130.stdout: "reply_latency": { 2024-04-25T00:45:19.770 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.770 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.770 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.770 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.770 INFO:teuthology.orchestra.run.smithi130.stdout: "slow_reply": 0, 2024-04-25T00:45:19.771 INFO:teuthology.orchestra.run.smithi130.stdout: "forward": 0, 2024-04-25T00:45:19.771 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_fetch_complete": 0, 2024-04-25T00:45:19.771 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_fetch_keys": 0, 2024-04-25T00:45:19.771 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_commit": 0, 2024-04-25T00:45:19.771 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_split": 0, 2024-04-25T00:45:19.771 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_merge": 0, 2024-04-25T00:45:19.771 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes": 119, 2024-04-25T00:45:19.771 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_top": 9, 2024-04-25T00:45:19.771 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_bottom": 110, 2024-04-25T00:45:19.771 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_pin_tail": 0, 2024-04-25T00:45:19.771 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_pinned": 106, 2024-04-25T00:45:19.771 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_expired": 0, 2024-04-25T00:45:19.771 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_with_caps": 0, 2024-04-25T00:45:19.771 INFO:teuthology.orchestra.run.smithi130.stdout: "caps": 0, 2024-04-25T00:45:19.771 INFO:teuthology.orchestra.run.smithi130.stdout: "subtrees": 2, 2024-04-25T00:45:19.771 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse": 0, 2024-04-25T00:45:19.772 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_hit": 0, 2024-04-25T00:45:19.772 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_forward": 0, 2024-04-25T00:45:19.772 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_discover": 0, 2024-04-25T00:45:19.772 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_dir_fetch": 0, 2024-04-25T00:45:19.772 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_remote_ino": 0, 2024-04-25T00:45:19.772 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_lock": 0, 2024-04-25T00:45:19.772 INFO:teuthology.orchestra.run.smithi130.stdout: "load_cent": 0, 2024-04-25T00:45:19.772 INFO:teuthology.orchestra.run.smithi130.stdout: "q": 0, 2024-04-25T00:45:19.772 INFO:teuthology.orchestra.run.smithi130.stdout: "exported": 0, 2024-04-25T00:45:19.772 INFO:teuthology.orchestra.run.smithi130.stdout: "exported_inodes": 0, 2024-04-25T00:45:19.772 INFO:teuthology.orchestra.run.smithi130.stdout: "imported": 0, 2024-04-25T00:45:19.772 INFO:teuthology.orchestra.run.smithi130.stdout: "imported_inodes": 0, 2024-04-25T00:45:19.772 INFO:teuthology.orchestra.run.smithi130.stdout: "openino_dir_fetch": 0, 2024-04-25T00:45:19.772 INFO:teuthology.orchestra.run.smithi130.stdout: "openino_backtrace_fetch": 0, 2024-04-25T00:45:19.772 INFO:teuthology.orchestra.run.smithi130.stdout: "openino_peer_discover": 0, 2024-04-25T00:45:19.773 INFO:teuthology.orchestra.run.smithi130.stdout: "root_rfiles": 4, 2024-04-25T00:45:19.773 INFO:teuthology.orchestra.run.smithi130.stdout: "root_rbytes": 234, 2024-04-25T00:45:19.773 INFO:teuthology.orchestra.run.smithi130.stdout: "root_rsnaps": 6, 2024-04-25T00:45:19.773 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_backtrace_fetch": 0, 2024-04-25T00:45:19.773 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_set_tag": 0, 2024-04-25T00:45:19.773 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_backtrace_repaired": 0, 2024-04-25T00:45:19.773 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_inotable_repaired": 0, 2024-04-25T00:45:19.773 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_dir_inodes": 0, 2024-04-25T00:45:19.773 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_dir_base_inodes": 0, 2024-04-25T00:45:19.773 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_dirfrag_rstats": 0, 2024-04-25T00:45:19.773 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_file_inodes": 0, 2024-04-25T00:45:19.773 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_inode_file_caps": 0, 2024-04-25T00:45:19.773 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_revoke": 0, 2024-04-25T00:45:19.773 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_grant": 0, 2024-04-25T00:45:19.773 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_trunc": 0, 2024-04-25T00:45:19.774 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-04-25T00:45:19.774 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_flush_ack": 0, 2024-04-25T00:45:19.774 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_caps": 0, 2024-04-25T00:45:19.774 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_caps_dirty": 0, 2024-04-25T00:45:19.774 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_cap_release": 0, 2024-04-25T00:45:19.774 INFO:teuthology.orchestra.run.smithi130.stdout: "process_request_cap_release": 0 2024-04-25T00:45:19.774 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.774 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_cache": { 2024-04-25T00:45:19.774 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_update": 0, 2024-04-25T00:45:19.774 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_update_receipt": 0, 2024-04-25T00:45:19.774 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_try_discover": 0, 2024-04-25T00:45:19.774 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_send_discover": 0, 2024-04-25T00:45:19.774 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_handle_discover": 0, 2024-04-25T00:45:19.774 INFO:teuthology.orchestra.run.smithi130.stdout: "num_strays": 0, 2024-04-25T00:45:19.775 INFO:teuthology.orchestra.run.smithi130.stdout: "num_strays_delayed": 0, 2024-04-25T00:45:19.775 INFO:teuthology.orchestra.run.smithi130.stdout: "num_strays_enqueuing": 0, 2024-04-25T00:45:19.775 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_created": 0, 2024-04-25T00:45:19.775 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_enqueued": 0, 2024-04-25T00:45:19.775 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_reintegrated": 0, 2024-04-25T00:45:19.775 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_migrated": 0, 2024-04-25T00:45:19.775 INFO:teuthology.orchestra.run.smithi130.stdout: "num_recovering_processing": 0, 2024-04-25T00:45:19.775 INFO:teuthology.orchestra.run.smithi130.stdout: "num_recovering_enqueued": 0, 2024-04-25T00:45:19.775 INFO:teuthology.orchestra.run.smithi130.stdout: "num_recovering_prioritized": 0, 2024-04-25T00:45:19.775 INFO:teuthology.orchestra.run.smithi130.stdout: "recovery_started": 0, 2024-04-25T00:45:19.775 INFO:teuthology.orchestra.run.smithi130.stdout: "recovery_completed": 0, 2024-04-25T00:45:19.775 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_quiesce_path": 0, 2024-04-25T00:45:19.775 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_quiesce_inode": 0, 2024-04-25T00:45:19.775 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_enqueue_scrub": 0, 2024-04-25T00:45:19.775 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_exportdir": 0, 2024-04-25T00:45:19.775 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_flush": 0, 2024-04-25T00:45:19.776 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_fragmentdir": 0, 2024-04-25T00:45:19.776 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_fragstats": 0, 2024-04-25T00:45:19.776 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_inodestats": 0 2024-04-25T00:45:19.776 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.776 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_log": { 2024-04-25T00:45:19.776 INFO:teuthology.orchestra.run.smithi130.stdout: "evlrg": 0, 2024-04-25T00:45:19.776 INFO:teuthology.orchestra.run.smithi130.stdout: "evadd": 0, 2024-04-25T00:45:19.776 INFO:teuthology.orchestra.run.smithi130.stdout: "evex": 0, 2024-04-25T00:45:19.776 INFO:teuthology.orchestra.run.smithi130.stdout: "evtrm": 0, 2024-04-25T00:45:19.776 INFO:teuthology.orchestra.run.smithi130.stdout: "ev": 161830, 2024-04-25T00:45:19.776 INFO:teuthology.orchestra.run.smithi130.stdout: "evexg": 0, 2024-04-25T00:45:19.776 INFO:teuthology.orchestra.run.smithi130.stdout: "evexd": 0, 2024-04-25T00:45:19.776 INFO:teuthology.orchestra.run.smithi130.stdout: "segadd": 0, 2024-04-25T00:45:19.776 INFO:teuthology.orchestra.run.smithi130.stdout: "segex": 0, 2024-04-25T00:45:19.776 INFO:teuthology.orchestra.run.smithi130.stdout: "segtrm": 0, 2024-04-25T00:45:19.776 INFO:teuthology.orchestra.run.smithi130.stdout: "seg": 135, 2024-04-25T00:45:19.777 INFO:teuthology.orchestra.run.smithi130.stdout: "segmjr": 15, 2024-04-25T00:45:19.777 INFO:teuthology.orchestra.run.smithi130.stdout: "segexg": 0, 2024-04-25T00:45:19.777 INFO:teuthology.orchestra.run.smithi130.stdout: "segexd": 0, 2024-04-25T00:45:19.777 INFO:teuthology.orchestra.run.smithi130.stdout: "expos": 114541499, 2024-04-25T00:45:19.777 INFO:teuthology.orchestra.run.smithi130.stdout: "wrpos": 363198262, 2024-04-25T00:45:19.777 INFO:teuthology.orchestra.run.smithi130.stdout: "rdpos": 363198126, 2024-04-25T00:45:19.777 INFO:teuthology.orchestra.run.smithi130.stdout: "jlat": { 2024-04-25T00:45:19.777 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.777 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.777 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.777 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.777 INFO:teuthology.orchestra.run.smithi130.stdout: "replayed": 161830 2024-04-25T00:45:19.777 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.778 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_mem": { 2024-04-25T00:45:19.778 INFO:teuthology.orchestra.run.smithi130.stdout: "ino": 91, 2024-04-25T00:45:19.778 INFO:teuthology.orchestra.run.smithi130.stdout: "ino+": 119, 2024-04-25T00:45:19.778 INFO:teuthology.orchestra.run.smithi130.stdout: "ino-": 28, 2024-04-25T00:45:19.778 INFO:teuthology.orchestra.run.smithi130.stdout: "dir": 14, 2024-04-25T00:45:19.778 INFO:teuthology.orchestra.run.smithi130.stdout: "dir+": 23, 2024-04-25T00:45:19.778 INFO:teuthology.orchestra.run.smithi130.stdout: "dir-": 9, 2024-04-25T00:45:19.778 INFO:teuthology.orchestra.run.smithi130.stdout: "dn": 119, 2024-04-25T00:45:19.778 INFO:teuthology.orchestra.run.smithi130.stdout: "dn+": 142, 2024-04-25T00:45:19.778 INFO:teuthology.orchestra.run.smithi130.stdout: "dn-": 23, 2024-04-25T00:45:19.778 INFO:teuthology.orchestra.run.smithi130.stdout: "cap": 0, 2024-04-25T00:45:19.778 INFO:teuthology.orchestra.run.smithi130.stdout: "cap+": 0, 2024-04-25T00:45:19.778 INFO:teuthology.orchestra.run.smithi130.stdout: "cap-": 0, 2024-04-25T00:45:19.778 INFO:teuthology.orchestra.run.smithi130.stdout: "rss": 54432, 2024-04-25T00:45:19.778 INFO:teuthology.orchestra.run.smithi130.stdout: "heap": 215420 2024-04-25T00:45:19.779 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.779 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_server": { 2024-04-25T00:45:19.779 INFO:teuthology.orchestra.run.smithi130.stdout: "dispatch_client_request": 0, 2024-04-25T00:45:19.779 INFO:teuthology.orchestra.run.smithi130.stdout: "dispatch_server_request": 0, 2024-04-25T00:45:19.779 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_request": 0, 2024-04-25T00:45:19.779 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_session": 0, 2024-04-25T00:45:19.779 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_peer_request": 0, 2024-04-25T00:45:19.779 INFO:teuthology.orchestra.run.smithi130.stdout: "req_create_latency": { 2024-04-25T00:45:19.779 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.779 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.779 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.779 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.779 INFO:teuthology.orchestra.run.smithi130.stdout: "req_getattr_latency": { 2024-04-25T00:45:19.779 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.779 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.780 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.780 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.780 INFO:teuthology.orchestra.run.smithi130.stdout: "req_getfilelock_latency": { 2024-04-25T00:45:19.780 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.780 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.780 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.780 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.780 INFO:teuthology.orchestra.run.smithi130.stdout: "req_link_latency": { 2024-04-25T00:45:19.780 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.780 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.780 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.780 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.780 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookup_latency": { 2024-04-25T00:45:19.780 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.780 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.781 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.781 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.781 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookuphash_latency": { 2024-04-25T00:45:19.781 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.781 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.781 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.781 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.781 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupino_latency": { 2024-04-25T00:45:19.781 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.781 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.781 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.781 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.781 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupname_latency": { 2024-04-25T00:45:19.781 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.781 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.781 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.782 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.782 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupparent_latency": { 2024-04-25T00:45:19.782 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.782 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.782 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.782 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.782 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupsnap_latency": { 2024-04-25T00:45:19.782 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.782 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.782 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.782 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.782 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lssnap_latency": { 2024-04-25T00:45:19.782 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.782 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.782 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.783 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.783 INFO:teuthology.orchestra.run.smithi130.stdout: "req_mkdir_latency": { 2024-04-25T00:45:19.783 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.783 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.783 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.783 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.783 INFO:teuthology.orchestra.run.smithi130.stdout: "req_mknod_latency": { 2024-04-25T00:45:19.783 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.783 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.783 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.783 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.783 INFO:teuthology.orchestra.run.smithi130.stdout: "req_mksnap_latency": { 2024-04-25T00:45:19.783 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.783 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.783 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.784 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.784 INFO:teuthology.orchestra.run.smithi130.stdout: "req_open_latency": { 2024-04-25T00:45:19.784 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.784 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.784 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.784 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.784 INFO:teuthology.orchestra.run.smithi130.stdout: "req_readdir_latency": { 2024-04-25T00:45:19.784 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.784 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.784 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.784 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.784 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rename_latency": { 2024-04-25T00:45:19.784 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.784 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.784 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.785 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.785 INFO:teuthology.orchestra.run.smithi130.stdout: "req_renamesnap_latency": { 2024-04-25T00:45:19.785 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.785 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.785 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.785 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.785 INFO:teuthology.orchestra.run.smithi130.stdout: "req_snapdiff_latency": { 2024-04-25T00:45:19.785 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.785 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.785 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.785 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.785 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rmdir_latency": { 2024-04-25T00:45:19.785 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.785 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.785 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.786 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.786 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rmsnap_latency": { 2024-04-25T00:45:19.786 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.786 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.786 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.786 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.786 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rmxattr_latency": { 2024-04-25T00:45:19.786 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.786 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.786 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.786 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.786 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setattr_latency": { 2024-04-25T00:45:19.786 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.786 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.787 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.787 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.787 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setdirlayout_latency": { 2024-04-25T00:45:19.787 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.787 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.787 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.787 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.787 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setfilelock_latency": { 2024-04-25T00:45:19.787 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.787 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.787 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.787 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.787 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setlayout_latency": { 2024-04-25T00:45:19.787 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.787 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.788 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.788 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.788 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setxattr_latency": { 2024-04-25T00:45:19.788 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.788 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.788 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.788 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.788 INFO:teuthology.orchestra.run.smithi130.stdout: "req_symlink_latency": { 2024-04-25T00:45:19.788 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.788 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.788 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.788 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.788 INFO:teuthology.orchestra.run.smithi130.stdout: "req_unlink_latency": { 2024-04-25T00:45:19.788 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.788 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.789 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.789 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.789 INFO:teuthology.orchestra.run.smithi130.stdout: "cap_revoke_eviction": 0, 2024-04-25T00:45:19.789 INFO:teuthology.orchestra.run.smithi130.stdout: "cap_acquisition_throttle": 0, 2024-04-25T00:45:19.789 INFO:teuthology.orchestra.run.smithi130.stdout: "req_getvxattr_latency": { 2024-04-25T00:45:19.789 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.789 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.789 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.789 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:19.789 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.789 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_sessions": { 2024-04-25T00:45:19.789 INFO:teuthology.orchestra.run.smithi130.stdout: "session_count": 2, 2024-04-25T00:45:19.789 INFO:teuthology.orchestra.run.smithi130.stdout: "session_add": 6, 2024-04-25T00:45:19.789 INFO:teuthology.orchestra.run.smithi130.stdout: "session_remove": 4, 2024-04-25T00:45:19.789 INFO:teuthology.orchestra.run.smithi130.stdout: "sessions_open": 4, 2024-04-25T00:45:19.790 INFO:teuthology.orchestra.run.smithi130.stdout: "sessions_stale": 0, 2024-04-25T00:45:19.790 INFO:teuthology.orchestra.run.smithi130.stdout: "total_load": 0, 2024-04-25T00:45:19.790 INFO:teuthology.orchestra.run.smithi130.stdout: "average_load": 0, 2024-04-25T00:45:19.790 INFO:teuthology.orchestra.run.smithi130.stdout: "avg_session_uptime": 2975, 2024-04-25T00:45:19.790 INFO:teuthology.orchestra.run.smithi130.stdout: "mdthresh_evicted": 0 2024-04-25T00:45:19.790 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.790 INFO:teuthology.orchestra.run.smithi130.stdout: "mempool": { 2024-04-25T00:45:19.790 INFO:teuthology.orchestra.run.smithi130.stdout: "bloom_filter_bytes": 0, 2024-04-25T00:45:19.790 INFO:teuthology.orchestra.run.smithi130.stdout: "bloom_filter_items": 0, 2024-04-25T00:45:19.790 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_alloc_bytes": 0, 2024-04-25T00:45:19.790 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_alloc_items": 0, 2024-04-25T00:45:19.790 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_data_bytes": 0, 2024-04-25T00:45:19.791 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_data_items": 0, 2024-04-25T00:45:19.791 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_onode_bytes": 0, 2024-04-25T00:45:19.791 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_onode_items": 0, 2024-04-25T00:45:19.791 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_meta_bytes": 0, 2024-04-25T00:45:19.791 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_meta_items": 0, 2024-04-25T00:45:19.791 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_other_bytes": 0, 2024-04-25T00:45:19.791 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_other_items": 0, 2024-04-25T00:45:19.791 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_buffer_bytes": 0, 2024-04-25T00:45:19.791 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_buffer_items": 0, 2024-04-25T00:45:19.791 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_extent_bytes": 0, 2024-04-25T00:45:19.791 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_extent_items": 0, 2024-04-25T00:45:19.791 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_blob_bytes": 0, 2024-04-25T00:45:19.791 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_blob_items": 0, 2024-04-25T00:45:19.791 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_shared_blob_bytes": 0, 2024-04-25T00:45:19.792 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_shared_blob_items": 0, 2024-04-25T00:45:19.792 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_inline_bl_bytes": 0, 2024-04-25T00:45:19.792 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_inline_bl_items": 0, 2024-04-25T00:45:19.792 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_bytes": 0, 2024-04-25T00:45:19.792 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_items": 0, 2024-04-25T00:45:19.792 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_txc_bytes": 0, 2024-04-25T00:45:19.792 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_txc_items": 0, 2024-04-25T00:45:19.792 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_deferred_bytes": 0, 2024-04-25T00:45:19.792 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_deferred_items": 0, 2024-04-25T00:45:19.792 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_bytes": 0, 2024-04-25T00:45:19.792 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_items": 0, 2024-04-25T00:45:19.792 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_bytes": 0, 2024-04-25T00:45:19.792 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_items": 0, 2024-04-25T00:45:19.792 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_reader_bytes": 0, 2024-04-25T00:45:19.792 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_reader_items": 0, 2024-04-25T00:45:19.793 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_writer_bytes": 0, 2024-04-25T00:45:19.793 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_writer_items": 0, 2024-04-25T00:45:19.793 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_anon_bytes": 241930, 2024-04-25T00:45:19.793 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_anon_items": 166, 2024-04-25T00:45:19.793 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_meta_bytes": 0, 2024-04-25T00:45:19.793 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_meta_items": 0, 2024-04-25T00:45:19.793 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_bytes": 0, 2024-04-25T00:45:19.793 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_items": 0, 2024-04-25T00:45:19.793 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mapbl_bytes": 0, 2024-04-25T00:45:19.793 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mapbl_items": 0, 2024-04-25T00:45:19.793 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pglog_bytes": 0, 2024-04-25T00:45:19.793 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pglog_items": 0, 2024-04-25T00:45:19.793 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_bytes": 9704, 2024-04-25T00:45:19.793 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_items": 186, 2024-04-25T00:45:19.793 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_mapping_bytes": 0, 2024-04-25T00:45:19.794 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_mapping_items": 0, 2024-04-25T00:45:19.794 INFO:teuthology.orchestra.run.smithi130.stdout: "pgmap_bytes": 0, 2024-04-25T00:45:19.794 INFO:teuthology.orchestra.run.smithi130.stdout: "pgmap_items": 0, 2024-04-25T00:45:19.794 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_co_bytes": 350438, 2024-04-25T00:45:19.794 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_co_items": 2883, 2024-04-25T00:45:19.794 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_1_bytes": 0, 2024-04-25T00:45:19.794 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_1_items": 0, 2024-04-25T00:45:19.794 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_2_bytes": 0, 2024-04-25T00:45:19.794 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_2_items": 0 2024-04-25T00:45:19.794 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.794 INFO:teuthology.orchestra.run.smithi130.stdout: "objecter": { 2024-04-25T00:45:19.794 INFO:teuthology.orchestra.run.smithi130.stdout: "op_active": 0, 2024-04-25T00:45:19.794 INFO:teuthology.orchestra.run.smithi130.stdout: "op_laggy": 0, 2024-04-25T00:45:19.794 INFO:teuthology.orchestra.run.smithi130.stdout: "op_send": 2860, 2024-04-25T00:45:19.795 INFO:teuthology.orchestra.run.smithi130.stdout: "op_send_bytes": 26, 2024-04-25T00:45:19.795 INFO:teuthology.orchestra.run.smithi130.stdout: "op_resend": 4, 2024-04-25T00:45:19.795 INFO:teuthology.orchestra.run.smithi130.stdout: "op_reply": 2860, 2024-04-25T00:45:19.795 INFO:teuthology.orchestra.run.smithi130.stdout: "op_latency": { 2024-04-25T00:45:19.795 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 2860, 2024-04-25T00:45:19.795 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 100.086145654, 2024-04-25T00:45:19.795 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.034995155 2024-04-25T00:45:19.795 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.795 INFO:teuthology.orchestra.run.smithi130.stdout: "op_inflight": 0, 2024-04-25T00:45:19.795 INFO:teuthology.orchestra.run.smithi130.stdout: "oplen_avg": { 2024-04-25T00:45:19.795 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 2860, 2024-04-25T00:45:19.795 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 2861 2024-04-25T00:45:19.795 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.795 INFO:teuthology.orchestra.run.smithi130.stdout: "op": 2860, 2024-04-25T00:45:19.795 INFO:teuthology.orchestra.run.smithi130.stdout: "op_r": 2860, 2024-04-25T00:45:19.796 INFO:teuthology.orchestra.run.smithi130.stdout: "op_w": 0, 2024-04-25T00:45:19.796 INFO:teuthology.orchestra.run.smithi130.stdout: "op_rmw": 0, 2024-04-25T00:45:19.796 INFO:teuthology.orchestra.run.smithi130.stdout: "op_pg": 0, 2024-04-25T00:45:19.796 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_stat": 1788, 2024-04-25T00:45:19.796 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_create": 0, 2024-04-25T00:45:19.796 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_read": 1069, 2024-04-25T00:45:19.796 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_write": 0, 2024-04-25T00:45:19.796 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_writefull": 0, 2024-04-25T00:45:19.796 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_writesame": 0, 2024-04-25T00:45:19.796 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_append": 0, 2024-04-25T00:45:19.796 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_zero": 0, 2024-04-25T00:45:19.796 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_truncate": 0, 2024-04-25T00:45:19.796 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_delete": 0, 2024-04-25T00:45:19.796 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_mapext": 0, 2024-04-25T00:45:19.796 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_sparse_read": 0, 2024-04-25T00:45:19.797 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_clonerange": 0, 2024-04-25T00:45:19.797 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_getxattr": 2, 2024-04-25T00:45:19.797 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_setxattr": 0, 2024-04-25T00:45:19.797 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_cmpxattr": 0, 2024-04-25T00:45:19.797 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_rmxattr": 0, 2024-04-25T00:45:19.797 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_resetxattrs": 0, 2024-04-25T00:45:19.797 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_call": 0, 2024-04-25T00:45:19.797 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_watch": 0, 2024-04-25T00:45:19.797 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_notify": 0, 2024-04-25T00:45:19.797 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_src_cmpxattr": 0, 2024-04-25T00:45:19.797 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_pgls": 0, 2024-04-25T00:45:19.797 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_pgls_filter": 0, 2024-04-25T00:45:19.797 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_other": 0, 2024-04-25T00:45:19.797 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_active": 0, 2024-04-25T00:45:19.797 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_send": 0, 2024-04-25T00:45:19.797 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_resend": 0, 2024-04-25T00:45:19.797 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_ping": 0, 2024-04-25T00:45:19.798 INFO:teuthology.orchestra.run.smithi130.stdout: "poolop_active": 0, 2024-04-25T00:45:19.798 INFO:teuthology.orchestra.run.smithi130.stdout: "poolop_send": 0, 2024-04-25T00:45:19.798 INFO:teuthology.orchestra.run.smithi130.stdout: "poolop_resend": 0, 2024-04-25T00:45:19.798 INFO:teuthology.orchestra.run.smithi130.stdout: "poolstat_active": 0, 2024-04-25T00:45:19.798 INFO:teuthology.orchestra.run.smithi130.stdout: "poolstat_send": 0, 2024-04-25T00:45:19.798 INFO:teuthology.orchestra.run.smithi130.stdout: "poolstat_resend": 0, 2024-04-25T00:45:19.798 INFO:teuthology.orchestra.run.smithi130.stdout: "statfs_active": 0, 2024-04-25T00:45:19.798 INFO:teuthology.orchestra.run.smithi130.stdout: "statfs_send": 0, 2024-04-25T00:45:19.798 INFO:teuthology.orchestra.run.smithi130.stdout: "statfs_resend": 0, 2024-04-25T00:45:19.798 INFO:teuthology.orchestra.run.smithi130.stdout: "command_active": 0, 2024-04-25T00:45:19.798 INFO:teuthology.orchestra.run.smithi130.stdout: "command_send": 0, 2024-04-25T00:45:19.798 INFO:teuthology.orchestra.run.smithi130.stdout: "command_resend": 0, 2024-04-25T00:45:19.798 INFO:teuthology.orchestra.run.smithi130.stdout: "map_epoch": 107, 2024-04-25T00:45:19.798 INFO:teuthology.orchestra.run.smithi130.stdout: "map_full": 0, 2024-04-25T00:45:19.798 INFO:teuthology.orchestra.run.smithi130.stdout: "map_inc": 18, 2024-04-25T00:45:19.798 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_sessions": 11, 2024-04-25T00:45:19.799 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_session_open": 11, 2024-04-25T00:45:19.799 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_session_close": 0, 2024-04-25T00:45:19.799 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_laggy": 0, 2024-04-25T00:45:19.799 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_wr": 0, 2024-04-25T00:45:19.799 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_rd": 2, 2024-04-25T00:45:19.799 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_del": 0 2024-04-25T00:45:19.799 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.799 INFO:teuthology.orchestra.run.smithi130.stdout: "oft": { 2024-04-25T00:45:19.799 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_objs": 0, 2024-04-25T00:45:19.799 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_kv_pairs": 0, 2024-04-25T00:45:19.799 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_updates": 0, 2024-04-25T00:45:19.799 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_removes": 0 2024-04-25T00:45:19.799 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.799 INFO:teuthology.orchestra.run.smithi130.stdout: "osd-slow-ops": { 2024-04-25T00:45:19.799 INFO:teuthology.orchestra.run.smithi130.stdout: "slow_ops_count": 0 2024-04-25T00:45:19.799 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.800 INFO:teuthology.orchestra.run.smithi130.stdout: "purge_queue": { 2024-04-25T00:45:19.800 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing_ops": 0, 2024-04-25T00:45:19.800 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing_ops_high_water": 0, 2024-04-25T00:45:19.800 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing": 0, 2024-04-25T00:45:19.800 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing_high_water": 0, 2024-04-25T00:45:19.800 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executed_ops": 0, 2024-04-25T00:45:19.800 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executed": 0, 2024-04-25T00:45:19.800 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_item_in_journal": 0 2024-04-25T00:45:19.800 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.800 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-04-25T00:45:19.800 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2024-04-25T00:45:19.800 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 104857600, 2024-04-25T00:45:19.800 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2024-04-25T00:45:19.800 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 3143, 2024-04-25T00:45:19.801 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 361214327, 2024-04-25T00:45:19.801 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:19.801 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 3143, 2024-04-25T00:45:19.801 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 0, 2024-04-25T00:45:19.801 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 0, 2024-04-25T00:45:19.801 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 3143, 2024-04-25T00:45:19.801 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 361214327, 2024-04-25T00:45:19.801 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2024-04-25T00:45:19.801 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.801 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.801 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.801 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:19.801 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.801 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-objecter_bytes": { 2024-04-25T00:45:19.802 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2024-04-25T00:45:19.802 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 104857600, 2024-04-25T00:45:19.802 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2024-04-25T00:45:19.802 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 0, 2024-04-25T00:45:19.802 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 0, 2024-04-25T00:45:19.802 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:19.802 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 0, 2024-04-25T00:45:19.802 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 2860, 2024-04-25T00:45:19.802 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 359069628, 2024-04-25T00:45:19.802 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 2860, 2024-04-25T00:45:19.802 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 359069628, 2024-04-25T00:45:19.802 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2024-04-25T00:45:19.802 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.802 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.803 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.803 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:19.803 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.803 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-objecter_ops": { 2024-04-25T00:45:19.803 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2024-04-25T00:45:19.803 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 1024, 2024-04-25T00:45:19.803 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2024-04-25T00:45:19.803 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 0, 2024-04-25T00:45:19.803 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 0, 2024-04-25T00:45:19.803 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:19.803 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 0, 2024-04-25T00:45:19.803 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 2860, 2024-04-25T00:45:19.803 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 2860, 2024-04-25T00:45:19.803 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 2860, 2024-04-25T00:45:19.803 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 2860, 2024-04-25T00:45:19.803 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2024-04-25T00:45:19.804 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.804 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.804 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.804 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:19.804 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.804 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-write_buf_throttle": { 2024-04-25T00:45:19.804 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2024-04-25T00:45:19.804 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 3758096384, 2024-04-25T00:45:19.804 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2024-04-25T00:45:19.804 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 0, 2024-04-25T00:45:19.804 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 0, 2024-04-25T00:45:19.804 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:19.804 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 0, 2024-04-25T00:45:19.804 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 0, 2024-04-25T00:45:19.804 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 0, 2024-04-25T00:45:19.804 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 0, 2024-04-25T00:45:19.805 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 0, 2024-04-25T00:45:19.805 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2024-04-25T00:45:19.805 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.805 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.805 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.805 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:19.805 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:19.805 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-write_buf_throttle-0x560e2e2c66e0": { 2024-04-25T00:45:19.805 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2024-04-25T00:45:19.805 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 3758096384, 2024-04-25T00:45:19.805 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2024-04-25T00:45:19.805 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 0, 2024-04-25T00:45:19.805 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 0, 2024-04-25T00:45:19.805 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:19.805 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 0, 2024-04-25T00:45:19.806 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 0, 2024-04-25T00:45:19.806 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 0, 2024-04-25T00:45:19.806 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 0, 2024-04-25T00:45:19.806 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 0, 2024-04-25T00:45:19.806 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2024-04-25T00:45:19.806 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:19.806 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:19.806 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:19.806 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:19.806 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:19.806 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-25T00:45:19.810 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:19 smithi121 ceph-mon[32231]: pgmap v376: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:45:21.164 INFO:tasks.check_counter:Daemon mds.c mds.exported=0 2024-04-25T00:45:21.164 INFO:tasks.check_counter:Daemon mds.c mds.imported=0 2024-04-25T00:45:21.164 INFO:tasks.check_counter:Daemon mds.c mds.root_rsnaps=6 2024-04-25T00:45:21.164 INFO:tasks.check_counter:Daemon mds.c mds_server.req_mksnap_latency.avgcount=0 2024-04-25T00:45:21.164 INFO:tasks.check_counter:Daemon mds.c mds_server.req_rmsnap_latency.avgcount=0 2024-04-25T00:45:21.165 DEBUG:tasks.check_counter:Getting stats from f 2024-04-25T00:45:21.165 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph daemon mds.f perf dump 2024-04-25T00:45:21.417 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.c/config 2024-04-25T00:45:21.727 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:21 smithi130 ceph-mon[32525]: pgmap v377: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:45:21.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:21 smithi032 ceph-mon[29357]: pgmap v377: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:45:21.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:21 smithi121 ceph-mon[32231]: pgmap v377: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:45:23.608 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:23 smithi130 ceph-mon[32525]: pgmap v378: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:45:23.636 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-25T00:45:23.636 INFO:teuthology.orchestra.run.smithi130.stdout: "AsyncMessenger::Worker-0": { 2024-04-25T00:45:23.636 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_messages": 1939, 2024-04-25T00:45:23.636 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages": 1941, 2024-04-25T00:45:23.636 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_bytes": 22441200, 2024-04-25T00:45:23.636 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_bytes": 616609, 2024-04-25T00:45:23.636 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_created_connections": 7, 2024-04-25T00:45:23.636 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_active_connections": 3, 2024-04-25T00:45:23.636 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_total_time": 0.207200131, 2024-04-25T00:45:23.636 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_send_time": 0.055765938, 2024-04-25T00:45:23.637 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_recv_time": 0.064365406, 2024-04-25T00:45:23.637 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_fast_dispatch_time": 0.072082922, 2024-04-25T00:45:23.637 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:23.637 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 1939, 2024-04-25T00:45:23.637 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.201306324, 2024-04-25T00:45:23.637 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000103819 2024-04-25T00:45:23.637 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.637 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:23.637 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.638 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.638 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.638 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.638 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_encrypted_bytes": 0, 2024-04-25T00:45:23.638 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_encrypted_bytes": 0 2024-04-25T00:45:23.638 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.638 INFO:teuthology.orchestra.run.smithi130.stdout: "AsyncMessenger::Worker-1": { 2024-04-25T00:45:23.638 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_messages": 417, 2024-04-25T00:45:23.638 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages": 597, 2024-04-25T00:45:23.638 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_bytes": 8493842, 2024-04-25T00:45:23.639 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_bytes": 610536, 2024-04-25T00:45:23.639 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_created_connections": 7, 2024-04-25T00:45:23.639 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_active_connections": 5, 2024-04-25T00:45:23.639 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_total_time": 0.064135413, 2024-04-25T00:45:23.639 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_send_time": 0.018481636, 2024-04-25T00:45:23.639 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_recv_time": 0.021438537, 2024-04-25T00:45:23.639 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_fast_dispatch_time": 0.018764276, 2024-04-25T00:45:23.639 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:23.639 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 595, 2024-04-25T00:45:23.639 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.093833169, 2024-04-25T00:45:23.639 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000157702 2024-04-25T00:45:23.640 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.640 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:23.640 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.640 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.640 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.640 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.640 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_encrypted_bytes": 3312, 2024-04-25T00:45:23.640 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_encrypted_bytes": 479776 2024-04-25T00:45:23.640 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.640 INFO:teuthology.orchestra.run.smithi130.stdout: "AsyncMessenger::Worker-2": { 2024-04-25T00:45:23.640 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_messages": 357, 2024-04-25T00:45:23.640 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages": 338, 2024-04-25T00:45:23.640 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_bytes": 5865028, 2024-04-25T00:45:23.641 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_bytes": 175754, 2024-04-25T00:45:23.641 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_created_connections": 4, 2024-04-25T00:45:23.641 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_active_connections": 3, 2024-04-25T00:45:23.641 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_total_time": 0.063696636, 2024-04-25T00:45:23.641 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_send_time": 0.017752578, 2024-04-25T00:45:23.641 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_recv_time": 0.045404363, 2024-04-25T00:45:23.641 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_fast_dispatch_time": 0.010302930, 2024-04-25T00:45:23.641 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:23.641 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 337, 2024-04-25T00:45:23.642 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.028857636, 2024-04-25T00:45:23.642 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000085630 2024-04-25T00:45:23.642 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.642 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:23.642 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.642 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.642 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.642 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.642 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_encrypted_bytes": 1651568, 2024-04-25T00:45:23.642 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_encrypted_bytes": 151904 2024-04-25T00:45:23.642 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.643 INFO:teuthology.orchestra.run.smithi130.stdout: "cct": { 2024-04-25T00:45:23.643 INFO:teuthology.orchestra.run.smithi130.stdout: "total_workers": 1, 2024-04-25T00:45:23.643 INFO:teuthology.orchestra.run.smithi130.stdout: "unhealthy_workers": 0 2024-04-25T00:45:23.643 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.643 INFO:teuthology.orchestra.run.smithi130.stdout: "finisher-MDSRank": { 2024-04-25T00:45:23.643 INFO:teuthology.orchestra.run.smithi130.stdout: "queue_len": 0, 2024-04-25T00:45:23.643 INFO:teuthology.orchestra.run.smithi130.stdout: "complete_latency": { 2024-04-25T00:45:23.643 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 4083, 2024-04-25T00:45:23.643 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.119286071, 2024-04-25T00:45:23.643 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000029215 2024-04-25T00:45:23.643 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:23.643 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.644 INFO:teuthology.orchestra.run.smithi130.stdout: "finisher-PurgeQueue": { 2024-04-25T00:45:23.644 INFO:teuthology.orchestra.run.smithi130.stdout: "queue_len": 0, 2024-04-25T00:45:23.644 INFO:teuthology.orchestra.run.smithi130.stdout: "complete_latency": { 2024-04-25T00:45:23.644 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.644 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.644 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.644 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:23.644 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.644 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-04-25T00:45:23.644 INFO:teuthology.orchestra.run.smithi130.stdout: "request": 0, 2024-04-25T00:45:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "reply": 0, 2024-04-25T00:45:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "reply_latency": { 2024-04-25T00:45:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "slow_reply": 0, 2024-04-25T00:45:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "forward": 0, 2024-04-25T00:45:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_fetch_complete": 0, 2024-04-25T00:45:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_fetch_keys": 0, 2024-04-25T00:45:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_commit": 0, 2024-04-25T00:45:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_split": 0, 2024-04-25T00:45:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_merge": 0, 2024-04-25T00:45:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes": 17, 2024-04-25T00:45:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_top": 0, 2024-04-25T00:45:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_bottom": 17, 2024-04-25T00:45:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_pin_tail": 0, 2024-04-25T00:45:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_pinned": 17, 2024-04-25T00:45:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_expired": 0, 2024-04-25T00:45:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_with_caps": 0, 2024-04-25T00:45:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "caps": 0, 2024-04-25T00:45:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "subtrees": 1, 2024-04-25T00:45:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse": 0, 2024-04-25T00:45:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_hit": 0, 2024-04-25T00:45:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_forward": 0, 2024-04-25T00:45:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_discover": 0, 2024-04-25T00:45:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_dir_fetch": 0, 2024-04-25T00:45:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_remote_ino": 0, 2024-04-25T00:45:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_lock": 0, 2024-04-25T00:45:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "load_cent": 0, 2024-04-25T00:45:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "q": 0, 2024-04-25T00:45:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "exported": 0, 2024-04-25T00:45:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "exported_inodes": 0, 2024-04-25T00:45:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "imported": 0, 2024-04-25T00:45:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "imported_inodes": 0, 2024-04-25T00:45:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "openino_dir_fetch": 0, 2024-04-25T00:45:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "openino_backtrace_fetch": 0, 2024-04-25T00:45:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "openino_peer_discover": 0, 2024-04-25T00:45:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "root_rfiles": 0, 2024-04-25T00:45:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "root_rbytes": 0, 2024-04-25T00:45:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "root_rsnaps": 0, 2024-04-25T00:45:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_backtrace_fetch": 0, 2024-04-25T00:45:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_set_tag": 0, 2024-04-25T00:45:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_backtrace_repaired": 0, 2024-04-25T00:45:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_inotable_repaired": 0, 2024-04-25T00:45:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_dir_inodes": 0, 2024-04-25T00:45:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_dir_base_inodes": 0, 2024-04-25T00:45:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_dirfrag_rstats": 0, 2024-04-25T00:45:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_file_inodes": 0, 2024-04-25T00:45:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_inode_file_caps": 0, 2024-04-25T00:45:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_revoke": 0, 2024-04-25T00:45:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_grant": 0, 2024-04-25T00:45:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_trunc": 0, 2024-04-25T00:45:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-04-25T00:45:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_flush_ack": 0, 2024-04-25T00:45:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_caps": 0, 2024-04-25T00:45:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_caps_dirty": 0, 2024-04-25T00:45:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_cap_release": 0, 2024-04-25T00:45:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "process_request_cap_release": 0 2024-04-25T00:45:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_cache": { 2024-04-25T00:45:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_update": 0, 2024-04-25T00:45:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_update_receipt": 0, 2024-04-25T00:45:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_try_discover": 0, 2024-04-25T00:45:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_send_discover": 0, 2024-04-25T00:45:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_handle_discover": 0, 2024-04-25T00:45:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "num_strays": 0, 2024-04-25T00:45:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "num_strays_delayed": 0, 2024-04-25T00:45:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "num_strays_enqueuing": 0, 2024-04-25T00:45:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_created": 0, 2024-04-25T00:45:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_enqueued": 0, 2024-04-25T00:45:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_reintegrated": 0, 2024-04-25T00:45:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_migrated": 0, 2024-04-25T00:45:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "num_recovering_processing": 0, 2024-04-25T00:45:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "num_recovering_enqueued": 0, 2024-04-25T00:45:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "num_recovering_prioritized": 0, 2024-04-25T00:45:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "recovery_started": 0, 2024-04-25T00:45:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "recovery_completed": 0, 2024-04-25T00:45:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_quiesce_path": 0, 2024-04-25T00:45:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_quiesce_inode": 0, 2024-04-25T00:45:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_enqueue_scrub": 0, 2024-04-25T00:45:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_exportdir": 0, 2024-04-25T00:45:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_flush": 0, 2024-04-25T00:45:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_fragmentdir": 0, 2024-04-25T00:45:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_fragstats": 0, 2024-04-25T00:45:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_inodestats": 0 2024-04-25T00:45:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_log": { 2024-04-25T00:45:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "evlrg": 0, 2024-04-25T00:45:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "evadd": 0, 2024-04-25T00:45:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "evex": 0, 2024-04-25T00:45:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "evtrm": 0, 2024-04-25T00:45:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "ev": 28857, 2024-04-25T00:45:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "evexg": 0, 2024-04-25T00:45:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "evexd": 0, 2024-04-25T00:45:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "segadd": 0, 2024-04-25T00:45:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "segex": 0, 2024-04-25T00:45:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "segtrm": 0, 2024-04-25T00:45:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "seg": 35, 2024-04-25T00:45:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "segmjr": 4, 2024-04-25T00:45:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "segexg": 0, 2024-04-25T00:45:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "segexd": 0, 2024-04-25T00:45:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "expos": 4194304, 2024-04-25T00:45:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "wrpos": 38656663, 2024-04-25T00:45:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "rdpos": 38654058, 2024-04-25T00:45:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "jlat": { 2024-04-25T00:45:23.653 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.653 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.653 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.653 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.653 INFO:teuthology.orchestra.run.smithi130.stdout: "replayed": 28857 2024-04-25T00:45:23.653 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.653 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_mem": { 2024-04-25T00:45:23.653 INFO:teuthology.orchestra.run.smithi130.stdout: "ino": 9, 2024-04-25T00:45:23.653 INFO:teuthology.orchestra.run.smithi130.stdout: "ino+": 55, 2024-04-25T00:45:23.654 INFO:teuthology.orchestra.run.smithi130.stdout: "ino-": 46, 2024-04-25T00:45:23.654 INFO:teuthology.orchestra.run.smithi130.stdout: "dir": 3, 2024-04-25T00:45:23.654 INFO:teuthology.orchestra.run.smithi130.stdout: "dir+": 25, 2024-04-25T00:45:23.654 INFO:teuthology.orchestra.run.smithi130.stdout: "dir-": 22, 2024-04-25T00:45:23.654 INFO:teuthology.orchestra.run.smithi130.stdout: "dn": 17, 2024-04-25T00:45:23.654 INFO:teuthology.orchestra.run.smithi130.stdout: "dn+": 64, 2024-04-25T00:45:23.655 INFO:teuthology.orchestra.run.smithi130.stdout: "dn-": 47, 2024-04-25T00:45:23.655 INFO:teuthology.orchestra.run.smithi130.stdout: "cap": 0, 2024-04-25T00:45:23.655 INFO:teuthology.orchestra.run.smithi130.stdout: "cap+": 0, 2024-04-25T00:45:23.655 INFO:teuthology.orchestra.run.smithi130.stdout: "cap-": 0, 2024-04-25T00:45:23.655 INFO:teuthology.orchestra.run.smithi130.stdout: "rss": 55868, 2024-04-25T00:45:23.655 INFO:teuthology.orchestra.run.smithi130.stdout: "heap": 215420 2024-04-25T00:45:23.656 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.656 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_server": { 2024-04-25T00:45:23.656 INFO:teuthology.orchestra.run.smithi130.stdout: "dispatch_client_request": 0, 2024-04-25T00:45:23.656 INFO:teuthology.orchestra.run.smithi130.stdout: "dispatch_server_request": 0, 2024-04-25T00:45:23.656 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_request": 0, 2024-04-25T00:45:23.656 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_session": 0, 2024-04-25T00:45:23.657 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_peer_request": 0, 2024-04-25T00:45:23.657 INFO:teuthology.orchestra.run.smithi130.stdout: "req_create_latency": { 2024-04-25T00:45:23.657 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.657 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.657 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.657 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.658 INFO:teuthology.orchestra.run.smithi130.stdout: "req_getattr_latency": { 2024-04-25T00:45:23.658 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.658 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.658 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.658 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.658 INFO:teuthology.orchestra.run.smithi130.stdout: "req_getfilelock_latency": { 2024-04-25T00:45:23.659 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.659 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.659 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.659 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.659 INFO:teuthology.orchestra.run.smithi130.stdout: "req_link_latency": { 2024-04-25T00:45:23.659 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.659 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.660 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.660 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.660 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookup_latency": { 2024-04-25T00:45:23.660 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.660 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.661 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.661 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.661 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookuphash_latency": { 2024-04-25T00:45:23.661 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.661 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.661 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.661 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.662 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupino_latency": { 2024-04-25T00:45:23.662 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.662 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.662 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.662 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.662 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupname_latency": { 2024-04-25T00:45:23.663 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.663 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.663 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.663 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.663 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupparent_latency": { 2024-04-25T00:45:23.663 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.663 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.664 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.664 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.664 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupsnap_latency": { 2024-04-25T00:45:23.664 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.664 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.664 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.665 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.665 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lssnap_latency": { 2024-04-25T00:45:23.665 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.665 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.665 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.665 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.666 INFO:teuthology.orchestra.run.smithi130.stdout: "req_mkdir_latency": { 2024-04-25T00:45:23.667 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.667 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.667 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.668 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.668 INFO:teuthology.orchestra.run.smithi130.stdout: "req_mknod_latency": { 2024-04-25T00:45:23.668 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.668 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.668 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.668 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.669 INFO:teuthology.orchestra.run.smithi130.stdout: "req_mksnap_latency": { 2024-04-25T00:45:23.669 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.669 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.669 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.669 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.669 INFO:teuthology.orchestra.run.smithi130.stdout: "req_open_latency": { 2024-04-25T00:45:23.670 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.670 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.670 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.670 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.670 INFO:teuthology.orchestra.run.smithi130.stdout: "req_readdir_latency": { 2024-04-25T00:45:23.671 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.671 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.671 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.671 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.671 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rename_latency": { 2024-04-25T00:45:23.671 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.672 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.672 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.672 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.672 INFO:teuthology.orchestra.run.smithi130.stdout: "req_renamesnap_latency": { 2024-04-25T00:45:23.672 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.672 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.673 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.673 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.673 INFO:teuthology.orchestra.run.smithi130.stdout: "req_snapdiff_latency": { 2024-04-25T00:45:23.673 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.673 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.674 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.674 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.674 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rmdir_latency": { 2024-04-25T00:45:23.674 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.674 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.674 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.675 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.675 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rmsnap_latency": { 2024-04-25T00:45:23.675 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.675 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.675 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.675 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.675 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rmxattr_latency": { 2024-04-25T00:45:23.675 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.675 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.675 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.676 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.676 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setattr_latency": { 2024-04-25T00:45:23.676 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.676 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.676 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.676 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.676 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setdirlayout_latency": { 2024-04-25T00:45:23.676 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.676 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.676 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.676 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.677 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setfilelock_latency": { 2024-04-25T00:45:23.677 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.677 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.677 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.677 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.678 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setlayout_latency": { 2024-04-25T00:45:23.678 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.678 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.678 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.678 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.678 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setxattr_latency": { 2024-04-25T00:45:23.678 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.678 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.679 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.679 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.679 INFO:teuthology.orchestra.run.smithi130.stdout: "req_symlink_latency": { 2024-04-25T00:45:23.679 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.679 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.679 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.680 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.680 INFO:teuthology.orchestra.run.smithi130.stdout: "req_unlink_latency": { 2024-04-25T00:45:23.680 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.680 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.680 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.681 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.681 INFO:teuthology.orchestra.run.smithi130.stdout: "cap_revoke_eviction": 0, 2024-04-25T00:45:23.681 INFO:teuthology.orchestra.run.smithi130.stdout: "cap_acquisition_throttle": 0, 2024-04-25T00:45:23.681 INFO:teuthology.orchestra.run.smithi130.stdout: "req_getvxattr_latency": { 2024-04-25T00:45:23.681 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.681 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.682 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.682 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:23.682 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.682 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_sessions": { 2024-04-25T00:45:23.682 INFO:teuthology.orchestra.run.smithi130.stdout: "session_count": 1, 2024-04-25T00:45:23.682 INFO:teuthology.orchestra.run.smithi130.stdout: "session_add": 2, 2024-04-25T00:45:23.683 INFO:teuthology.orchestra.run.smithi130.stdout: "session_remove": 1, 2024-04-25T00:45:23.683 INFO:teuthology.orchestra.run.smithi130.stdout: "sessions_open": 2, 2024-04-25T00:45:23.683 INFO:teuthology.orchestra.run.smithi130.stdout: "sessions_stale": 0, 2024-04-25T00:45:23.683 INFO:teuthology.orchestra.run.smithi130.stdout: "total_load": 0, 2024-04-25T00:45:23.683 INFO:teuthology.orchestra.run.smithi130.stdout: "average_load": 0, 2024-04-25T00:45:23.683 INFO:teuthology.orchestra.run.smithi130.stdout: "avg_session_uptime": 2977, 2024-04-25T00:45:23.684 INFO:teuthology.orchestra.run.smithi130.stdout: "mdthresh_evicted": 0 2024-04-25T00:45:23.684 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.684 INFO:teuthology.orchestra.run.smithi130.stdout: "mempool": { 2024-04-25T00:45:23.684 INFO:teuthology.orchestra.run.smithi130.stdout: "bloom_filter_bytes": 0, 2024-04-25T00:45:23.684 INFO:teuthology.orchestra.run.smithi130.stdout: "bloom_filter_items": 0, 2024-04-25T00:45:23.684 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_alloc_bytes": 0, 2024-04-25T00:45:23.685 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_alloc_items": 0, 2024-04-25T00:45:23.685 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_data_bytes": 0, 2024-04-25T00:45:23.685 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_data_items": 0, 2024-04-25T00:45:23.685 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_onode_bytes": 0, 2024-04-25T00:45:23.685 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_onode_items": 0, 2024-04-25T00:45:23.686 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_meta_bytes": 0, 2024-04-25T00:45:23.686 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_meta_items": 0, 2024-04-25T00:45:23.686 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_other_bytes": 0, 2024-04-25T00:45:23.686 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_other_items": 0, 2024-04-25T00:45:23.686 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_buffer_bytes": 0, 2024-04-25T00:45:23.686 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_buffer_items": 0, 2024-04-25T00:45:23.687 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_extent_bytes": 0, 2024-04-25T00:45:23.687 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_extent_items": 0, 2024-04-25T00:45:23.687 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_blob_bytes": 0, 2024-04-25T00:45:23.687 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_blob_items": 0, 2024-04-25T00:45:23.687 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_shared_blob_bytes": 0, 2024-04-25T00:45:23.687 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_shared_blob_items": 0, 2024-04-25T00:45:23.688 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_inline_bl_bytes": 0, 2024-04-25T00:45:23.688 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_inline_bl_items": 0, 2024-04-25T00:45:23.688 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_bytes": 0, 2024-04-25T00:45:23.688 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_items": 0, 2024-04-25T00:45:23.688 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_txc_bytes": 0, 2024-04-25T00:45:23.688 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_txc_items": 0, 2024-04-25T00:45:23.689 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_deferred_bytes": 0, 2024-04-25T00:45:23.689 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_deferred_items": 0, 2024-04-25T00:45:23.689 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_bytes": 0, 2024-04-25T00:45:23.689 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_items": 0, 2024-04-25T00:45:23.689 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_bytes": 0, 2024-04-25T00:45:23.689 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_items": 0, 2024-04-25T00:45:23.690 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_reader_bytes": 0, 2024-04-25T00:45:23.690 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_reader_items": 0, 2024-04-25T00:45:23.690 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_writer_bytes": 0, 2024-04-25T00:45:23.690 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_writer_items": 0, 2024-04-25T00:45:23.690 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_anon_bytes": 207375, 2024-04-25T00:45:23.690 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_anon_items": 71, 2024-04-25T00:45:23.690 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_meta_bytes": 0, 2024-04-25T00:45:23.690 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_meta_items": 0, 2024-04-25T00:45:23.691 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_bytes": 0, 2024-04-25T00:45:23.691 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_items": 0, 2024-04-25T00:45:23.691 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mapbl_bytes": 0, 2024-04-25T00:45:23.691 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mapbl_items": 0, 2024-04-25T00:45:23.691 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pglog_bytes": 0, 2024-04-25T00:45:23.691 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pglog_items": 0, 2024-04-25T00:45:23.691 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_bytes": 9704, 2024-04-25T00:45:23.691 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_items": 186, 2024-04-25T00:45:23.691 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_mapping_bytes": 0, 2024-04-25T00:45:23.692 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_mapping_items": 0, 2024-04-25T00:45:23.692 INFO:teuthology.orchestra.run.smithi130.stdout: "pgmap_bytes": 0, 2024-04-25T00:45:23.692 INFO:teuthology.orchestra.run.smithi130.stdout: "pgmap_items": 0, 2024-04-25T00:45:23.692 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_co_bytes": 40538, 2024-04-25T00:45:23.692 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_co_items": 346, 2024-04-25T00:45:23.692 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_1_bytes": 0, 2024-04-25T00:45:23.692 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_1_items": 0, 2024-04-25T00:45:23.692 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_2_bytes": 0, 2024-04-25T00:45:23.693 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_2_items": 0 2024-04-25T00:45:23.693 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.693 INFO:teuthology.orchestra.run.smithi130.stdout: "objecter": { 2024-04-25T00:45:23.693 INFO:teuthology.orchestra.run.smithi130.stdout: "op_active": 0, 2024-04-25T00:45:23.693 INFO:teuthology.orchestra.run.smithi130.stdout: "op_laggy": 0, 2024-04-25T00:45:23.693 INFO:teuthology.orchestra.run.smithi130.stdout: "op_send": 2425, 2024-04-25T00:45:23.693 INFO:teuthology.orchestra.run.smithi130.stdout: "op_send_bytes": 21, 2024-04-25T00:45:23.694 INFO:teuthology.orchestra.run.smithi130.stdout: "op_resend": 3, 2024-04-25T00:45:23.694 INFO:teuthology.orchestra.run.smithi130.stdout: "op_reply": 2425, 2024-04-25T00:45:23.694 INFO:teuthology.orchestra.run.smithi130.stdout: "op_latency": { 2024-04-25T00:45:23.694 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 2425, 2024-04-25T00:45:23.694 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 53.021048150, 2024-04-25T00:45:23.694 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.021864349 2024-04-25T00:45:23.694 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.694 INFO:teuthology.orchestra.run.smithi130.stdout: "op_inflight": 0, 2024-04-25T00:45:23.694 INFO:teuthology.orchestra.run.smithi130.stdout: "oplen_avg": { 2024-04-25T00:45:23.694 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 2425, 2024-04-25T00:45:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 2426 2024-04-25T00:45:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "op": 2425, 2024-04-25T00:45:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "op_r": 2425, 2024-04-25T00:45:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "op_w": 0, 2024-04-25T00:45:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "op_rmw": 0, 2024-04-25T00:45:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "op_pg": 0, 2024-04-25T00:45:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_stat": 1468, 2024-04-25T00:45:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_create": 0, 2024-04-25T00:45:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_read": 955, 2024-04-25T00:45:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_write": 0, 2024-04-25T00:45:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_writefull": 0, 2024-04-25T00:45:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_writesame": 0, 2024-04-25T00:45:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_append": 0, 2024-04-25T00:45:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_zero": 0, 2024-04-25T00:45:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_truncate": 0, 2024-04-25T00:45:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_delete": 0, 2024-04-25T00:45:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_mapext": 0, 2024-04-25T00:45:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_sparse_read": 0, 2024-04-25T00:45:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_clonerange": 0, 2024-04-25T00:45:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_getxattr": 1, 2024-04-25T00:45:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_setxattr": 0, 2024-04-25T00:45:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_cmpxattr": 0, 2024-04-25T00:45:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_rmxattr": 0, 2024-04-25T00:45:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_resetxattrs": 0, 2024-04-25T00:45:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_call": 0, 2024-04-25T00:45:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_watch": 0, 2024-04-25T00:45:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_notify": 0, 2024-04-25T00:45:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_src_cmpxattr": 0, 2024-04-25T00:45:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_pgls": 0, 2024-04-25T00:45:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_pgls_filter": 0, 2024-04-25T00:45:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_other": 0, 2024-04-25T00:45:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_active": 0, 2024-04-25T00:45:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_send": 0, 2024-04-25T00:45:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_resend": 0, 2024-04-25T00:45:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_ping": 0, 2024-04-25T00:45:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "poolop_active": 0, 2024-04-25T00:45:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "poolop_send": 0, 2024-04-25T00:45:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "poolop_resend": 0, 2024-04-25T00:45:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "poolstat_active": 0, 2024-04-25T00:45:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "poolstat_send": 0, 2024-04-25T00:45:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "poolstat_resend": 0, 2024-04-25T00:45:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "statfs_active": 0, 2024-04-25T00:45:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "statfs_send": 0, 2024-04-25T00:45:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "statfs_resend": 0, 2024-04-25T00:45:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "command_active": 0, 2024-04-25T00:45:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "command_send": 0, 2024-04-25T00:45:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "command_resend": 0, 2024-04-25T00:45:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "map_epoch": 107, 2024-04-25T00:45:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "map_full": 0, 2024-04-25T00:45:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "map_inc": 18, 2024-04-25T00:45:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_sessions": 7, 2024-04-25T00:45:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_session_open": 7, 2024-04-25T00:45:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_session_close": 0, 2024-04-25T00:45:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_laggy": 0, 2024-04-25T00:45:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_wr": 0, 2024-04-25T00:45:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_rd": 2, 2024-04-25T00:45:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_del": 0 2024-04-25T00:45:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "oft": { 2024-04-25T00:45:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_objs": 0, 2024-04-25T00:45:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_kv_pairs": 0, 2024-04-25T00:45:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_updates": 0, 2024-04-25T00:45:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_removes": 0 2024-04-25T00:45:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "osd-slow-ops": { 2024-04-25T00:45:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "slow_ops_count": 0 2024-04-25T00:45:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "purge_queue": { 2024-04-25T00:45:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing_ops": 0, 2024-04-25T00:45:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing_ops_high_water": 0, 2024-04-25T00:45:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing": 0, 2024-04-25T00:45:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing_high_water": 0, 2024-04-25T00:45:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executed_ops": 0, 2024-04-25T00:45:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executed": 0, 2024-04-25T00:45:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_item_in_journal": 0 2024-04-25T00:45:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-04-25T00:45:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2024-04-25T00:45:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 104857600, 2024-04-25T00:45:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2024-04-25T00:45:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 2709, 2024-04-25T00:45:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 36538845, 2024-04-25T00:45:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 2709, 2024-04-25T00:45:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 0, 2024-04-25T00:45:23.701 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 0, 2024-04-25T00:45:23.701 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 2709, 2024-04-25T00:45:23.701 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 36538845, 2024-04-25T00:45:23.701 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2024-04-25T00:45:23.701 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.701 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.701 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.701 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:23.701 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.701 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-objecter_bytes": { 2024-04-25T00:45:23.701 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2024-04-25T00:45:23.701 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 104857600, 2024-04-25T00:45:23.702 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2024-04-25T00:45:23.702 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 0, 2024-04-25T00:45:23.702 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 0, 2024-04-25T00:45:23.702 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:23.702 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 0, 2024-04-25T00:45:23.702 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 2425, 2024-04-25T00:45:23.702 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 34462616, 2024-04-25T00:45:23.702 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 2425, 2024-04-25T00:45:23.702 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 34462616, 2024-04-25T00:45:23.702 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2024-04-25T00:45:23.703 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.703 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.703 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.703 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:23.703 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.703 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-objecter_ops": { 2024-04-25T00:45:23.703 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2024-04-25T00:45:23.703 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 1024, 2024-04-25T00:45:23.703 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2024-04-25T00:45:23.703 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 0, 2024-04-25T00:45:23.703 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 0, 2024-04-25T00:45:23.703 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:23.704 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 0, 2024-04-25T00:45:23.704 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 2425, 2024-04-25T00:45:23.704 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 2425, 2024-04-25T00:45:23.704 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 2425, 2024-04-25T00:45:23.704 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 2425, 2024-04-25T00:45:23.704 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2024-04-25T00:45:23.704 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.704 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.704 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.704 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:23.704 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.704 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-write_buf_throttle": { 2024-04-25T00:45:23.704 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2024-04-25T00:45:23.704 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 3758096384, 2024-04-25T00:45:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2024-04-25T00:45:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 0, 2024-04-25T00:45:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 0, 2024-04-25T00:45:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 0, 2024-04-25T00:45:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 0, 2024-04-25T00:45:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 0, 2024-04-25T00:45:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 0, 2024-04-25T00:45:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 0, 2024-04-25T00:45:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2024-04-25T00:45:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-write_buf_throttle-0x55925f2fa6e0": { 2024-04-25T00:45:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2024-04-25T00:45:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 3758096384, 2024-04-25T00:45:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2024-04-25T00:45:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 0, 2024-04-25T00:45:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 0, 2024-04-25T00:45:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 0, 2024-04-25T00:45:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 0, 2024-04-25T00:45:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 0, 2024-04-25T00:45:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 0, 2024-04-25T00:45:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 0, 2024-04-25T00:45:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2024-04-25T00:45:23.707 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:23.707 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:23.707 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:23.707 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:23.707 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:23.707 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-25T00:45:23.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:23 smithi121 ceph-mon[32231]: pgmap v378: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:45:23.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:23 smithi032 ceph-mon[29357]: pgmap v378: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 102 B/s wr, 6 op/s 2024-04-25T00:45:24.568 INFO:tasks.check_counter:Daemon mds.f mds.exported=0 2024-04-25T00:45:24.568 INFO:tasks.check_counter:Daemon mds.f mds.imported=0 2024-04-25T00:45:24.569 INFO:tasks.check_counter:Daemon mds.f mds.root_rsnaps=0 2024-04-25T00:45:24.569 INFO:tasks.check_counter:Daemon mds.f mds_server.req_mksnap_latency.avgcount=0 2024-04-25T00:45:24.569 INFO:tasks.check_counter:Daemon mds.f mds_server.req_rmsnap_latency.avgcount=0 2024-04-25T00:45:24.569 DEBUG:tasks.check_counter:Getting stats from i 2024-04-25T00:45:24.569 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph daemon mds.i perf dump 2024-04-25T00:45:24.808 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.c/config 2024-04-25T00:45:25.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:25 smithi130 ceph-mon[32525]: pgmap v379: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 6 op/s 2024-04-25T00:45:25.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:25 smithi121 ceph-mon[32231]: pgmap v379: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 6 op/s 2024-04-25T00:45:25.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:25 smithi032 ceph-mon[29357]: pgmap v379: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 6 op/s 2024-04-25T00:45:26.642 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-25T00:45:26.642 INFO:teuthology.orchestra.run.smithi130.stdout: "AsyncMessenger::Worker-0": { 2024-04-25T00:45:26.642 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_messages": 44139, 2024-04-25T00:45:26.642 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages": 44183, 2024-04-25T00:45:26.642 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_bytes": 13252990, 2024-04-25T00:45:26.642 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_bytes": 35770210, 2024-04-25T00:45:26.642 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_created_connections": 9, 2024-04-25T00:45:26.642 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_active_connections": 4, 2024-04-25T00:45:26.642 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_total_time": 3.183311318, 2024-04-25T00:45:26.642 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_send_time": 1.106906505, 2024-04-25T00:45:26.643 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_recv_time": 2.230254886, 2024-04-25T00:45:26.643 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_fast_dispatch_time": 0.396505280, 2024-04-25T00:45:26.643 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:26.643 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 44181, 2024-04-25T00:45:26.643 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 5.896367887, 2024-04-25T00:45:26.643 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000133459 2024-04-25T00:45:26.643 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.643 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:26.643 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 51505, 2024-04-25T00:45:26.643 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.012518184, 2024-04-25T00:45:26.643 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000243 2024-04-25T00:45:26.643 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.643 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_encrypted_bytes": 0, 2024-04-25T00:45:26.643 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_encrypted_bytes": 0 2024-04-25T00:45:26.644 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.644 INFO:teuthology.orchestra.run.smithi130.stdout: "AsyncMessenger::Worker-1": { 2024-04-25T00:45:26.644 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_messages": 658, 2024-04-25T00:45:26.644 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages": 696, 2024-04-25T00:45:26.644 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_bytes": 192978, 2024-04-25T00:45:26.644 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_bytes": 3691258, 2024-04-25T00:45:26.644 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_created_connections": 30, 2024-04-25T00:45:26.644 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_active_connections": 9, 2024-04-25T00:45:26.644 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_total_time": 0.090722872, 2024-04-25T00:45:26.644 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_send_time": 0.034142355, 2024-04-25T00:45:26.644 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_recv_time": 0.026434265, 2024-04-25T00:45:26.644 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_fast_dispatch_time": 0.021230477, 2024-04-25T00:45:26.644 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:26.644 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 693, 2024-04-25T00:45:26.644 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.165189508, 2024-04-25T00:45:26.645 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000238368 2024-04-25T00:45:26.645 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.645 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:26.645 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 155, 2024-04-25T00:45:26.645 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000034401, 2024-04-25T00:45:26.645 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000221 2024-04-25T00:45:26.645 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.645 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_encrypted_bytes": 2000, 2024-04-25T00:45:26.645 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_encrypted_bytes": 134656 2024-04-25T00:45:26.645 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.645 INFO:teuthology.orchestra.run.smithi130.stdout: "AsyncMessenger::Worker-2": { 2024-04-25T00:45:26.645 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_messages": 5211, 2024-04-25T00:45:26.645 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages": 5333, 2024-04-25T00:45:26.645 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_bytes": 2875914, 2024-04-25T00:45:26.646 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_bytes": 9561559, 2024-04-25T00:45:26.646 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_created_connections": 8, 2024-04-25T00:45:26.646 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_active_connections": 5, 2024-04-25T00:45:26.646 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_total_time": 0.444359014, 2024-04-25T00:45:26.646 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_send_time": 0.138917019, 2024-04-25T00:45:26.646 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_recv_time": 0.121445574, 2024-04-25T00:45:26.646 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_fast_dispatch_time": 0.150221631, 2024-04-25T00:45:26.646 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:26.646 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 5333, 2024-04-25T00:45:26.646 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.451010344, 2024-04-25T00:45:26.646 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000084569 2024-04-25T00:45:26.646 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.646 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:26.647 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.647 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.647 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.647 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.647 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_encrypted_bytes": 1669408, 2024-04-25T00:45:26.647 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_encrypted_bytes": 515424 2024-04-25T00:45:26.647 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.647 INFO:teuthology.orchestra.run.smithi130.stdout: "cct": { 2024-04-25T00:45:26.647 INFO:teuthology.orchestra.run.smithi130.stdout: "total_workers": 1, 2024-04-25T00:45:26.647 INFO:teuthology.orchestra.run.smithi130.stdout: "unhealthy_workers": 0 2024-04-25T00:45:26.647 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.647 INFO:teuthology.orchestra.run.smithi130.stdout: "finisher-MDSRank": { 2024-04-25T00:45:26.647 INFO:teuthology.orchestra.run.smithi130.stdout: "queue_len": 0, 2024-04-25T00:45:26.647 INFO:teuthology.orchestra.run.smithi130.stdout: "complete_latency": { 2024-04-25T00:45:26.647 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 26383, 2024-04-25T00:45:26.648 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 6.157607798, 2024-04-25T00:45:26.648 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000233393 2024-04-25T00:45:26.648 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:26.648 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.648 INFO:teuthology.orchestra.run.smithi130.stdout: "finisher-PurgeQueue": { 2024-04-25T00:45:26.648 INFO:teuthology.orchestra.run.smithi130.stdout: "queue_len": 0, 2024-04-25T00:45:26.648 INFO:teuthology.orchestra.run.smithi130.stdout: "complete_latency": { 2024-04-25T00:45:26.648 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 523, 2024-04-25T00:45:26.648 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.024236913, 2024-04-25T00:45:26.648 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000046342 2024-04-25T00:45:26.648 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:26.648 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.648 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-04-25T00:45:26.648 INFO:teuthology.orchestra.run.smithi130.stdout: "request": 27, 2024-04-25T00:45:26.649 INFO:teuthology.orchestra.run.smithi130.stdout: "reply": 26, 2024-04-25T00:45:26.649 INFO:teuthology.orchestra.run.smithi130.stdout: "reply_latency": { 2024-04-25T00:45:26.649 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 26, 2024-04-25T00:45:26.649 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.029046456, 2024-04-25T00:45:26.649 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.001117171 2024-04-25T00:45:26.649 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.649 INFO:teuthology.orchestra.run.smithi130.stdout: "slow_reply": 0, 2024-04-25T00:45:26.649 INFO:teuthology.orchestra.run.smithi130.stdout: "forward": 1, 2024-04-25T00:45:26.649 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_fetch_complete": 0, 2024-04-25T00:45:26.649 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_fetch_keys": 0, 2024-04-25T00:45:26.649 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_commit": 11, 2024-04-25T00:45:26.649 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_split": 0, 2024-04-25T00:45:26.649 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_merge": 0, 2024-04-25T00:45:26.650 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes": 21, 2024-04-25T00:45:26.650 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_top": 1, 2024-04-25T00:45:26.650 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_bottom": 1, 2024-04-25T00:45:26.650 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_pin_tail": 19, 2024-04-25T00:45:26.650 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_pinned": 19, 2024-04-25T00:45:26.650 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_expired": 46, 2024-04-25T00:45:26.650 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_with_caps": 1, 2024-04-25T00:45:26.650 INFO:teuthology.orchestra.run.smithi130.stdout: "caps": 1, 2024-04-25T00:45:26.650 INFO:teuthology.orchestra.run.smithi130.stdout: "subtrees": 3, 2024-04-25T00:45:26.650 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse": 52, 2024-04-25T00:45:26.650 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_hit": 35, 2024-04-25T00:45:26.650 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_forward": 0, 2024-04-25T00:45:26.650 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_discover": 4, 2024-04-25T00:45:26.651 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_dir_fetch": 0, 2024-04-25T00:45:26.651 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_remote_ino": 0, 2024-04-25T00:45:26.651 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_lock": 0, 2024-04-25T00:45:26.651 INFO:teuthology.orchestra.run.smithi130.stdout: "load_cent": 0, 2024-04-25T00:45:26.651 INFO:teuthology.orchestra.run.smithi130.stdout: "q": 0, 2024-04-25T00:45:26.651 INFO:teuthology.orchestra.run.smithi130.stdout: "exported": 3, 2024-04-25T00:45:26.651 INFO:teuthology.orchestra.run.smithi130.stdout: "exported_inodes": 34, 2024-04-25T00:45:26.651 INFO:teuthology.orchestra.run.smithi130.stdout: "imported": 4, 2024-04-25T00:45:26.651 INFO:teuthology.orchestra.run.smithi130.stdout: "imported_inodes": 16, 2024-04-25T00:45:26.651 INFO:teuthology.orchestra.run.smithi130.stdout: "openino_dir_fetch": 0, 2024-04-25T00:45:26.651 INFO:teuthology.orchestra.run.smithi130.stdout: "openino_backtrace_fetch": 34, 2024-04-25T00:45:26.651 INFO:teuthology.orchestra.run.smithi130.stdout: "openino_peer_discover": 1, 2024-04-25T00:45:26.651 INFO:teuthology.orchestra.run.smithi130.stdout: "root_rfiles": 22, 2024-04-25T00:45:26.652 INFO:teuthology.orchestra.run.smithi130.stdout: "root_rbytes": 10738142561, 2024-04-25T00:45:26.652 INFO:teuthology.orchestra.run.smithi130.stdout: "root_rsnaps": 6, 2024-04-25T00:45:26.652 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_backtrace_fetch": 34, 2024-04-25T00:45:26.652 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_set_tag": 0, 2024-04-25T00:45:26.652 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_backtrace_repaired": 0, 2024-04-25T00:45:26.652 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_inotable_repaired": 0, 2024-04-25T00:45:26.652 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_dir_inodes": 4, 2024-04-25T00:45:26.652 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_dir_base_inodes": 0, 2024-04-25T00:45:26.652 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_dirfrag_rstats": 4, 2024-04-25T00:45:26.652 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_file_inodes": 30, 2024-04-25T00:45:26.652 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_inode_file_caps": 0, 2024-04-25T00:45:26.652 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_revoke": 9, 2024-04-25T00:45:26.652 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_grant": 19, 2024-04-25T00:45:26.652 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_trunc": 0, 2024-04-25T00:45:26.652 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-04-25T00:45:26.653 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_flush_ack": 0, 2024-04-25T00:45:26.653 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_caps": 28726, 2024-04-25T00:45:26.653 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_caps_dirty": 28716, 2024-04-25T00:45:26.653 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_cap_release": 5, 2024-04-25T00:45:26.653 INFO:teuthology.orchestra.run.smithi130.stdout: "process_request_cap_release": 22 2024-04-25T00:45:26.653 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.653 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_cache": { 2024-04-25T00:45:26.653 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_update": 0, 2024-04-25T00:45:26.653 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_update_receipt": 0, 2024-04-25T00:45:26.653 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_try_discover": 0, 2024-04-25T00:45:26.653 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_send_discover": 5, 2024-04-25T00:45:26.653 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_handle_discover": 0, 2024-04-25T00:45:26.653 INFO:teuthology.orchestra.run.smithi130.stdout: "num_strays": 1, 2024-04-25T00:45:26.653 INFO:teuthology.orchestra.run.smithi130.stdout: "num_strays_delayed": 0, 2024-04-25T00:45:26.654 INFO:teuthology.orchestra.run.smithi130.stdout: "num_strays_enqueuing": 0, 2024-04-25T00:45:26.654 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_created": 10, 2024-04-25T00:45:26.654 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_enqueued": 9, 2024-04-25T00:45:26.654 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_reintegrated": 0, 2024-04-25T00:45:26.654 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_migrated": 0, 2024-04-25T00:45:26.654 INFO:teuthology.orchestra.run.smithi130.stdout: "num_recovering_processing": 0, 2024-04-25T00:45:26.654 INFO:teuthology.orchestra.run.smithi130.stdout: "num_recovering_enqueued": 0, 2024-04-25T00:45:26.654 INFO:teuthology.orchestra.run.smithi130.stdout: "num_recovering_prioritized": 0, 2024-04-25T00:45:26.654 INFO:teuthology.orchestra.run.smithi130.stdout: "recovery_started": 0, 2024-04-25T00:45:26.654 INFO:teuthology.orchestra.run.smithi130.stdout: "recovery_completed": 0, 2024-04-25T00:45:26.654 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_quiesce_path": 0, 2024-04-25T00:45:26.654 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_quiesce_inode": 0, 2024-04-25T00:45:26.654 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_enqueue_scrub": 0, 2024-04-25T00:45:26.654 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_exportdir": 3, 2024-04-25T00:45:26.655 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_flush": 0, 2024-04-25T00:45:26.655 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_fragmentdir": 0, 2024-04-25T00:45:26.655 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_fragstats": 0, 2024-04-25T00:45:26.655 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_inodestats": 0 2024-04-25T00:45:26.655 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.655 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_log": { 2024-04-25T00:45:26.655 INFO:teuthology.orchestra.run.smithi130.stdout: "evlrg": 0, 2024-04-25T00:45:26.655 INFO:teuthology.orchestra.run.smithi130.stdout: "evadd": 28857, 2024-04-25T00:45:26.655 INFO:teuthology.orchestra.run.smithi130.stdout: "evex": 0, 2024-04-25T00:45:26.655 INFO:teuthology.orchestra.run.smithi130.stdout: "evtrm": 0, 2024-04-25T00:45:26.655 INFO:teuthology.orchestra.run.smithi130.stdout: "ev": 28857, 2024-04-25T00:45:26.655 INFO:teuthology.orchestra.run.smithi130.stdout: "evexg": 0, 2024-04-25T00:45:26.655 INFO:teuthology.orchestra.run.smithi130.stdout: "evexd": 0, 2024-04-25T00:45:26.655 INFO:teuthology.orchestra.run.smithi130.stdout: "segadd": 35, 2024-04-25T00:45:26.656 INFO:teuthology.orchestra.run.smithi130.stdout: "segex": 0, 2024-04-25T00:45:26.656 INFO:teuthology.orchestra.run.smithi130.stdout: "segtrm": 0, 2024-04-25T00:45:26.656 INFO:teuthology.orchestra.run.smithi130.stdout: "seg": 35, 2024-04-25T00:45:26.656 INFO:teuthology.orchestra.run.smithi130.stdout: "segmjr": 4, 2024-04-25T00:45:26.656 INFO:teuthology.orchestra.run.smithi130.stdout: "segexg": 0, 2024-04-25T00:45:26.656 INFO:teuthology.orchestra.run.smithi130.stdout: "segexd": 0, 2024-04-25T00:45:26.656 INFO:teuthology.orchestra.run.smithi130.stdout: "expos": 4194304, 2024-04-25T00:45:26.656 INFO:teuthology.orchestra.run.smithi130.stdout: "wrpos": 38656663, 2024-04-25T00:45:26.656 INFO:teuthology.orchestra.run.smithi130.stdout: "rdpos": 0, 2024-04-25T00:45:26.656 INFO:teuthology.orchestra.run.smithi130.stdout: "jlat": { 2024-04-25T00:45:26.657 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 19211, 2024-04-25T00:45:26.657 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 31350.133440607, 2024-04-25T00:45:26.657 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 1.631884516 2024-04-25T00:45:26.657 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.657 INFO:teuthology.orchestra.run.smithi130.stdout: "replayed": 0 2024-04-25T00:45:26.657 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.657 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_mem": { 2024-04-25T00:45:26.657 INFO:teuthology.orchestra.run.smithi130.stdout: "ino": 23, 2024-04-25T00:45:26.657 INFO:teuthology.orchestra.run.smithi130.stdout: "ino+": 64, 2024-04-25T00:45:26.657 INFO:teuthology.orchestra.run.smithi130.stdout: "ino-": 41, 2024-04-25T00:45:26.658 INFO:teuthology.orchestra.run.smithi130.stdout: "dir": 18, 2024-04-25T00:45:26.658 INFO:teuthology.orchestra.run.smithi130.stdout: "dir+": 35, 2024-04-25T00:45:26.658 INFO:teuthology.orchestra.run.smithi130.stdout: "dir-": 17, 2024-04-25T00:45:26.658 INFO:teuthology.orchestra.run.smithi130.stdout: "dn": 21, 2024-04-25T00:45:26.658 INFO:teuthology.orchestra.run.smithi130.stdout: "dn+": 76, 2024-04-25T00:45:26.658 INFO:teuthology.orchestra.run.smithi130.stdout: "dn-": 55, 2024-04-25T00:45:26.658 INFO:teuthology.orchestra.run.smithi130.stdout: "cap": 1, 2024-04-25T00:45:26.658 INFO:teuthology.orchestra.run.smithi130.stdout: "cap+": 23, 2024-04-25T00:45:26.658 INFO:teuthology.orchestra.run.smithi130.stdout: "cap-": 22, 2024-04-25T00:45:26.658 INFO:teuthology.orchestra.run.smithi130.stdout: "rss": 68048, 2024-04-25T00:45:26.658 INFO:teuthology.orchestra.run.smithi130.stdout: "heap": 223612 2024-04-25T00:45:26.658 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.658 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_server": { 2024-04-25T00:45:26.659 INFO:teuthology.orchestra.run.smithi130.stdout: "dispatch_client_request": 45, 2024-04-25T00:45:26.659 INFO:teuthology.orchestra.run.smithi130.stdout: "dispatch_server_request": 1, 2024-04-25T00:45:26.659 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_request": 27, 2024-04-25T00:45:26.659 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_session": 14, 2024-04-25T00:45:26.659 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_peer_request": 13, 2024-04-25T00:45:26.659 INFO:teuthology.orchestra.run.smithi130.stdout: "req_create_latency": { 2024-04-25T00:45:26.659 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.659 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.659 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.659 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.660 INFO:teuthology.orchestra.run.smithi130.stdout: "req_getattr_latency": { 2024-04-25T00:45:26.660 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 1, 2024-04-25T00:45:26.660 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000214519, 2024-04-25T00:45:26.660 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000214519 2024-04-25T00:45:26.660 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.660 INFO:teuthology.orchestra.run.smithi130.stdout: "req_getfilelock_latency": { 2024-04-25T00:45:26.660 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.660 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.660 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.660 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.661 INFO:teuthology.orchestra.run.smithi130.stdout: "req_link_latency": { 2024-04-25T00:45:26.661 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.661 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.661 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.661 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.661 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookup_latency": { 2024-04-25T00:45:26.661 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 10, 2024-04-25T00:45:26.661 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.003063397, 2024-04-25T00:45:26.661 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000306339 2024-04-25T00:45:26.661 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.661 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookuphash_latency": { 2024-04-25T00:45:26.661 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.661 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.662 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.662 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.662 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupino_latency": { 2024-04-25T00:45:26.662 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 1, 2024-04-25T00:45:26.662 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000086078, 2024-04-25T00:45:26.662 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000086078 2024-04-25T00:45:26.662 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.662 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupname_latency": { 2024-04-25T00:45:26.662 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.662 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.662 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.662 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.662 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupparent_latency": { 2024-04-25T00:45:26.662 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.662 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.663 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.663 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.663 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupsnap_latency": { 2024-04-25T00:45:26.663 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.663 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.663 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.663 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.663 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lssnap_latency": { 2024-04-25T00:45:26.663 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.663 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.663 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.664 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.664 INFO:teuthology.orchestra.run.smithi130.stdout: "req_mkdir_latency": { 2024-04-25T00:45:26.664 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.664 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.664 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.664 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.664 INFO:teuthology.orchestra.run.smithi130.stdout: "req_mknod_latency": { 2024-04-25T00:45:26.664 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.665 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.665 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.665 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.665 INFO:teuthology.orchestra.run.smithi130.stdout: "req_mksnap_latency": { 2024-04-25T00:45:26.665 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.665 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.665 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.665 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.665 INFO:teuthology.orchestra.run.smithi130.stdout: "req_open_latency": { 2024-04-25T00:45:26.665 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 1, 2024-04-25T00:45:26.665 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000464773, 2024-04-25T00:45:26.665 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000464773 2024-04-25T00:45:26.665 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.665 INFO:teuthology.orchestra.run.smithi130.stdout: "req_readdir_latency": { 2024-04-25T00:45:26.666 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 3, 2024-04-25T00:45:26.666 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.001462048, 2024-04-25T00:45:26.666 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000487349 2024-04-25T00:45:26.666 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.666 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rename_latency": { 2024-04-25T00:45:26.666 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.666 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.666 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.666 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.666 INFO:teuthology.orchestra.run.smithi130.stdout: "req_renamesnap_latency": { 2024-04-25T00:45:26.666 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.666 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.666 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.666 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.667 INFO:teuthology.orchestra.run.smithi130.stdout: "req_snapdiff_latency": { 2024-04-25T00:45:26.667 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.667 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.667 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.667 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.667 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rmdir_latency": { 2024-04-25T00:45:26.667 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 1, 2024-04-25T00:45:26.667 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.008560934, 2024-04-25T00:45:26.667 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.008560934 2024-04-25T00:45:26.667 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.667 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rmsnap_latency": { 2024-04-25T00:45:26.667 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.667 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.667 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.668 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.668 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rmxattr_latency": { 2024-04-25T00:45:26.668 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.668 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.668 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.668 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.668 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setattr_latency": { 2024-04-25T00:45:26.668 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.668 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.668 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.668 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.668 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setdirlayout_latency": { 2024-04-25T00:45:26.668 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.668 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.668 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.669 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.669 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setfilelock_latency": { 2024-04-25T00:45:26.669 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.669 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.669 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.669 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.669 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setlayout_latency": { 2024-04-25T00:45:26.669 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.669 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.669 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.669 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.669 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setxattr_latency": { 2024-04-25T00:45:26.669 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.669 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.669 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.670 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.670 INFO:teuthology.orchestra.run.smithi130.stdout: "req_symlink_latency": { 2024-04-25T00:45:26.670 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.670 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.670 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.670 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.670 INFO:teuthology.orchestra.run.smithi130.stdout: "req_unlink_latency": { 2024-04-25T00:45:26.670 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 9, 2024-04-25T00:45:26.670 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.015194707, 2024-04-25T00:45:26.670 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.001688300 2024-04-25T00:45:26.670 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.670 INFO:teuthology.orchestra.run.smithi130.stdout: "cap_revoke_eviction": 0, 2024-04-25T00:45:26.670 INFO:teuthology.orchestra.run.smithi130.stdout: "cap_acquisition_throttle": 0, 2024-04-25T00:45:26.670 INFO:teuthology.orchestra.run.smithi130.stdout: "req_getvxattr_latency": { 2024-04-25T00:45:26.671 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.671 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.671 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.671 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:26.671 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.671 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_sessions": { 2024-04-25T00:45:26.671 INFO:teuthology.orchestra.run.smithi130.stdout: "session_count": 1, 2024-04-25T00:45:26.671 INFO:teuthology.orchestra.run.smithi130.stdout: "session_add": 2, 2024-04-25T00:45:26.671 INFO:teuthology.orchestra.run.smithi130.stdout: "session_remove": 1, 2024-04-25T00:45:26.671 INFO:teuthology.orchestra.run.smithi130.stdout: "sessions_open": 1, 2024-04-25T00:45:26.671 INFO:teuthology.orchestra.run.smithi130.stdout: "sessions_stale": 0, 2024-04-25T00:45:26.671 INFO:teuthology.orchestra.run.smithi130.stdout: "total_load": 21, 2024-04-25T00:45:26.671 INFO:teuthology.orchestra.run.smithi130.stdout: "average_load": 21, 2024-04-25T00:45:26.671 INFO:teuthology.orchestra.run.smithi130.stdout: "avg_session_uptime": 717, 2024-04-25T00:45:26.671 INFO:teuthology.orchestra.run.smithi130.stdout: "mdthresh_evicted": 0 2024-04-25T00:45:26.671 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.672 INFO:teuthology.orchestra.run.smithi130.stdout: "mempool": { 2024-04-25T00:45:26.672 INFO:teuthology.orchestra.run.smithi130.stdout: "bloom_filter_bytes": 0, 2024-04-25T00:45:26.672 INFO:teuthology.orchestra.run.smithi130.stdout: "bloom_filter_items": 0, 2024-04-25T00:45:26.672 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_alloc_bytes": 0, 2024-04-25T00:45:26.672 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_alloc_items": 0, 2024-04-25T00:45:26.672 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_data_bytes": 0, 2024-04-25T00:45:26.672 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_data_items": 0, 2024-04-25T00:45:26.672 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_onode_bytes": 0, 2024-04-25T00:45:26.672 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_onode_items": 0, 2024-04-25T00:45:26.672 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_meta_bytes": 0, 2024-04-25T00:45:26.672 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_meta_items": 0, 2024-04-25T00:45:26.672 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_other_bytes": 0, 2024-04-25T00:45:26.672 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_other_items": 0, 2024-04-25T00:45:26.672 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_buffer_bytes": 0, 2024-04-25T00:45:26.672 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_buffer_items": 0, 2024-04-25T00:45:26.673 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_extent_bytes": 0, 2024-04-25T00:45:26.673 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_extent_items": 0, 2024-04-25T00:45:26.673 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_blob_bytes": 0, 2024-04-25T00:45:26.673 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_blob_items": 0, 2024-04-25T00:45:26.673 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_shared_blob_bytes": 0, 2024-04-25T00:45:26.673 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_shared_blob_items": 0, 2024-04-25T00:45:26.673 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_inline_bl_bytes": 0, 2024-04-25T00:45:26.673 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_inline_bl_items": 0, 2024-04-25T00:45:26.673 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_bytes": 0, 2024-04-25T00:45:26.673 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_items": 0, 2024-04-25T00:45:26.673 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_txc_bytes": 0, 2024-04-25T00:45:26.673 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_txc_items": 0, 2024-04-25T00:45:26.673 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_deferred_bytes": 0, 2024-04-25T00:45:26.673 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_deferred_items": 0, 2024-04-25T00:45:26.673 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_bytes": 0, 2024-04-25T00:45:26.673 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_items": 0, 2024-04-25T00:45:26.674 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_bytes": 0, 2024-04-25T00:45:26.674 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_items": 0, 2024-04-25T00:45:26.674 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_reader_bytes": 0, 2024-04-25T00:45:26.674 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_reader_items": 0, 2024-04-25T00:45:26.674 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_writer_bytes": 0, 2024-04-25T00:45:26.674 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_writer_items": 0, 2024-04-25T00:45:26.674 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_anon_bytes": 324038, 2024-04-25T00:45:26.674 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_anon_items": 133, 2024-04-25T00:45:26.674 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_meta_bytes": 0, 2024-04-25T00:45:26.674 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_meta_items": 0, 2024-04-25T00:45:26.674 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_bytes": 0, 2024-04-25T00:45:26.674 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_items": 0, 2024-04-25T00:45:26.674 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mapbl_bytes": 0, 2024-04-25T00:45:26.674 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mapbl_items": 0, 2024-04-25T00:45:26.675 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pglog_bytes": 0, 2024-04-25T00:45:26.675 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pglog_items": 0, 2024-04-25T00:45:26.675 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_bytes": 9704, 2024-04-25T00:45:26.675 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_items": 186, 2024-04-25T00:45:26.675 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_mapping_bytes": 0, 2024-04-25T00:45:26.675 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_mapping_items": 0, 2024-04-25T00:45:26.675 INFO:teuthology.orchestra.run.smithi130.stdout: "pgmap_bytes": 0, 2024-04-25T00:45:26.675 INFO:teuthology.orchestra.run.smithi130.stdout: "pgmap_items": 0, 2024-04-25T00:45:26.675 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_co_bytes": 95479, 2024-04-25T00:45:26.675 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_co_items": 578, 2024-04-25T00:45:26.675 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_1_bytes": 0, 2024-04-25T00:45:26.675 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_1_items": 0, 2024-04-25T00:45:26.675 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_2_bytes": 0, 2024-04-25T00:45:26.676 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_2_items": 0 2024-04-25T00:45:26.676 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.676 INFO:teuthology.orchestra.run.smithi130.stdout: "objecter": { 2024-04-25T00:45:26.676 INFO:teuthology.orchestra.run.smithi130.stdout: "op_active": 0, 2024-04-25T00:45:26.676 INFO:teuthology.orchestra.run.smithi130.stdout: "op_laggy": 0, 2024-04-25T00:45:26.676 INFO:teuthology.orchestra.run.smithi130.stdout: "op_send": 19894, 2024-04-25T00:45:26.676 INFO:teuthology.orchestra.run.smithi130.stdout: "op_send_bytes": 34480819, 2024-04-25T00:45:26.676 INFO:teuthology.orchestra.run.smithi130.stdout: "op_resend": 28, 2024-04-25T00:45:26.676 INFO:teuthology.orchestra.run.smithi130.stdout: "op_reply": 19894, 2024-04-25T00:45:26.676 INFO:teuthology.orchestra.run.smithi130.stdout: "op_latency": { 2024-04-25T00:45:26.676 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 19894, 2024-04-25T00:45:26.676 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 31392.888067446, 2024-04-25T00:45:26.676 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 1.578007844 2024-04-25T00:45:26.676 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.676 INFO:teuthology.orchestra.run.smithi130.stdout: "op_inflight": 0, 2024-04-25T00:45:26.677 INFO:teuthology.orchestra.run.smithi130.stdout: "oplen_avg": { 2024-04-25T00:45:26.677 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 19894, 2024-04-25T00:45:26.677 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 19932 2024-04-25T00:45:26.677 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.677 INFO:teuthology.orchestra.run.smithi130.stdout: "op": 19894, 2024-04-25T00:45:26.677 INFO:teuthology.orchestra.run.smithi130.stdout: "op_r": 35, 2024-04-25T00:45:26.677 INFO:teuthology.orchestra.run.smithi130.stdout: "op_w": 19859, 2024-04-25T00:45:26.677 INFO:teuthology.orchestra.run.smithi130.stdout: "op_rmw": 0, 2024-04-25T00:45:26.677 INFO:teuthology.orchestra.run.smithi130.stdout: "op_pg": 0, 2024-04-25T00:45:26.677 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_stat": 0, 2024-04-25T00:45:26.677 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_create": 10, 2024-04-25T00:45:26.677 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_read": 1, 2024-04-25T00:45:26.677 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_write": 19212, 2024-04-25T00:45:26.677 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_writefull": 29, 2024-04-25T00:45:26.677 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_writesame": 0, 2024-04-25T00:45:26.678 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_append": 0, 2024-04-25T00:45:26.678 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_zero": 0, 2024-04-25T00:45:26.678 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_truncate": 0, 2024-04-25T00:45:26.678 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_delete": 550, 2024-04-25T00:45:26.678 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_mapext": 0, 2024-04-25T00:45:26.678 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_sparse_read": 0, 2024-04-25T00:45:26.678 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_clonerange": 0, 2024-04-25T00:45:26.678 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_getxattr": 34, 2024-04-25T00:45:26.678 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_setxattr": 20, 2024-04-25T00:45:26.678 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_cmpxattr": 0, 2024-04-25T00:45:26.678 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_rmxattr": 0, 2024-04-25T00:45:26.678 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_resetxattrs": 0, 2024-04-25T00:45:26.678 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_call": 0, 2024-04-25T00:45:26.678 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_watch": 0, 2024-04-25T00:45:26.679 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_notify": 0, 2024-04-25T00:45:26.679 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_src_cmpxattr": 0, 2024-04-25T00:45:26.679 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_pgls": 0, 2024-04-25T00:45:26.679 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_pgls_filter": 0, 2024-04-25T00:45:26.679 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_other": 0, 2024-04-25T00:45:26.679 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_active": 0, 2024-04-25T00:45:26.679 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_send": 0, 2024-04-25T00:45:26.679 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_resend": 0, 2024-04-25T00:45:26.679 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_ping": 0, 2024-04-25T00:45:26.679 INFO:teuthology.orchestra.run.smithi130.stdout: "poolop_active": 0, 2024-04-25T00:45:26.679 INFO:teuthology.orchestra.run.smithi130.stdout: "poolop_send": 0, 2024-04-25T00:45:26.679 INFO:teuthology.orchestra.run.smithi130.stdout: "poolop_resend": 0, 2024-04-25T00:45:26.679 INFO:teuthology.orchestra.run.smithi130.stdout: "poolstat_active": 0, 2024-04-25T00:45:26.679 INFO:teuthology.orchestra.run.smithi130.stdout: "poolstat_send": 0, 2024-04-25T00:45:26.679 INFO:teuthology.orchestra.run.smithi130.stdout: "poolstat_resend": 0, 2024-04-25T00:45:26.679 INFO:teuthology.orchestra.run.smithi130.stdout: "statfs_active": 0, 2024-04-25T00:45:26.680 INFO:teuthology.orchestra.run.smithi130.stdout: "statfs_send": 0, 2024-04-25T00:45:26.680 INFO:teuthology.orchestra.run.smithi130.stdout: "statfs_resend": 0, 2024-04-25T00:45:26.680 INFO:teuthology.orchestra.run.smithi130.stdout: "command_active": 0, 2024-04-25T00:45:26.680 INFO:teuthology.orchestra.run.smithi130.stdout: "command_send": 0, 2024-04-25T00:45:26.680 INFO:teuthology.orchestra.run.smithi130.stdout: "command_resend": 0, 2024-04-25T00:45:26.680 INFO:teuthology.orchestra.run.smithi130.stdout: "map_epoch": 107, 2024-04-25T00:45:26.680 INFO:teuthology.orchestra.run.smithi130.stdout: "map_full": 0, 2024-04-25T00:45:26.680 INFO:teuthology.orchestra.run.smithi130.stdout: "map_inc": 18, 2024-04-25T00:45:26.680 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_sessions": 12, 2024-04-25T00:45:26.680 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_session_open": 12, 2024-04-25T00:45:26.680 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_session_close": 0, 2024-04-25T00:45:26.680 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_laggy": 0, 2024-04-25T00:45:26.680 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_wr": 68, 2024-04-25T00:45:26.680 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_rd": 0, 2024-04-25T00:45:26.681 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_del": 8 2024-04-25T00:45:26.681 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.681 INFO:teuthology.orchestra.run.smithi130.stdout: "oft": { 2024-04-25T00:45:26.681 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_objs": 1, 2024-04-25T00:45:26.681 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_kv_pairs": 0, 2024-04-25T00:45:26.681 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_updates": 0, 2024-04-25T00:45:26.681 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_removes": 0 2024-04-25T00:45:26.681 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.681 INFO:teuthology.orchestra.run.smithi130.stdout: "osd-slow-ops": { 2024-04-25T00:45:26.681 INFO:teuthology.orchestra.run.smithi130.stdout: "slow_ops_count": 0 2024-04-25T00:45:26.681 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.681 INFO:teuthology.orchestra.run.smithi130.stdout: "purge_queue": { 2024-04-25T00:45:26.681 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing_ops": 0, 2024-04-25T00:45:26.681 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing_ops_high_water": 526, 2024-04-25T00:45:26.681 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing": 0, 2024-04-25T00:45:26.682 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing_high_water": 7, 2024-04-25T00:45:26.682 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executed_ops": 530, 2024-04-25T00:45:26.682 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executed": 9, 2024-04-25T00:45:26.682 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_item_in_journal": 0 2024-04-25T00:45:26.682 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.682 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-04-25T00:45:26.682 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2024-04-25T00:45:26.682 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 104857600, 2024-04-25T00:45:26.682 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2024-04-25T00:45:26.682 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 50004, 2024-04-25T00:45:26.682 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 11802064, 2024-04-25T00:45:26.682 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:26.682 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 50004, 2024-04-25T00:45:26.682 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 0, 2024-04-25T00:45:26.683 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 0, 2024-04-25T00:45:26.683 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 50004, 2024-04-25T00:45:26.683 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 11802064, 2024-04-25T00:45:26.683 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2024-04-25T00:45:26.683 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.683 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.683 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.683 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:26.683 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.683 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-objecter_bytes": { 2024-04-25T00:45:26.683 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2024-04-25T00:45:26.683 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 104857600, 2024-04-25T00:45:26.684 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2024-04-25T00:45:26.684 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 0, 2024-04-25T00:45:26.684 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 0, 2024-04-25T00:45:26.684 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:26.684 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 0, 2024-04-25T00:45:26.684 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 19894, 2024-04-25T00:45:26.684 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 34482232, 2024-04-25T00:45:26.684 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 19894, 2024-04-25T00:45:26.684 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 34482232, 2024-04-25T00:45:26.684 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2024-04-25T00:45:26.685 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.685 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.685 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.685 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:26.685 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.685 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-objecter_ops": { 2024-04-25T00:45:26.685 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2024-04-25T00:45:26.685 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 1024, 2024-04-25T00:45:26.685 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2024-04-25T00:45:26.685 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 0, 2024-04-25T00:45:26.685 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 0, 2024-04-25T00:45:26.685 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:26.686 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 0, 2024-04-25T00:45:26.686 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 19894, 2024-04-25T00:45:26.686 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 19894, 2024-04-25T00:45:26.686 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 19894, 2024-04-25T00:45:26.686 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 19894, 2024-04-25T00:45:26.686 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2024-04-25T00:45:26.686 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.686 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.686 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.686 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:26.686 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.686 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-write_buf_throttle": { 2024-04-25T00:45:26.686 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2024-04-25T00:45:26.686 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 3758096384, 2024-04-25T00:45:26.687 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2024-04-25T00:45:26.687 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 9, 2024-04-25T00:45:26.687 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 1413, 2024-04-25T00:45:26.687 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:26.687 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 9, 2024-04-25T00:45:26.687 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 0, 2024-04-25T00:45:26.687 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 0, 2024-04-25T00:45:26.687 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 1, 2024-04-25T00:45:26.687 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 1413, 2024-04-25T00:45:26.687 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2024-04-25T00:45:26.687 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.687 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.687 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.687 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:26.687 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:26.688 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-write_buf_throttle-0x5621f5100500": { 2024-04-25T00:45:26.688 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2024-04-25T00:45:26.688 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 3758096384, 2024-04-25T00:45:26.688 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2024-04-25T00:45:26.688 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 28857, 2024-04-25T00:45:26.688 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 34462359, 2024-04-25T00:45:26.688 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:26.688 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 28857, 2024-04-25T00:45:26.688 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 0, 2024-04-25T00:45:26.688 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 0, 2024-04-25T00:45:26.688 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 19211, 2024-04-25T00:45:26.688 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 34462359, 2024-04-25T00:45:26.688 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2024-04-25T00:45:26.689 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:26.689 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:26.689 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:26.689 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:26.689 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:26.689 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-25T00:45:27.464 INFO:tasks.check_counter:Daemon mds.i mds.exported=3 2024-04-25T00:45:27.464 INFO:tasks.check_counter:Daemon mds.i mds.imported=4 2024-04-25T00:45:27.464 INFO:tasks.check_counter:Daemon mds.i mds.root_rsnaps=6 2024-04-25T00:45:27.464 INFO:tasks.check_counter:Daemon mds.i mds_server.req_mksnap_latency.avgcount=0 2024-04-25T00:45:27.464 INFO:tasks.check_counter:Daemon mds.i mds_server.req_rmsnap_latency.avgcount=0 2024-04-25T00:45:27.464 DEBUG:tasks.check_counter:Getting stats from l 2024-04-25T00:45:27.465 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551 shell --fsid cb997156-0299-11ef-bc93-c7b262605968 -- ceph daemon mds.l perf dump 2024-04-25T00:45:27.705 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/mon.c/config 2024-04-25T00:45:27.828 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:27 smithi130 ceph-mon[32525]: pgmap v380: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 6 op/s 2024-04-25T00:45:27.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:27 smithi121 ceph-mon[32231]: pgmap v380: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 6 op/s 2024-04-25T00:45:27.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:27 smithi032 ceph-mon[29357]: pgmap v380: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 6 op/s 2024-04-25T00:45:29.499 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-25T00:45:29.499 INFO:teuthology.orchestra.run.smithi130.stdout: "AsyncMessenger::Worker-0": { 2024-04-25T00:45:29.499 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_messages": 0, 2024-04-25T00:45:29.499 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages": 2, 2024-04-25T00:45:29.499 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_bytes": 0, 2024-04-25T00:45:29.499 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_bytes": 0, 2024-04-25T00:45:29.499 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_created_connections": 4, 2024-04-25T00:45:29.499 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_active_connections": 0, 2024-04-25T00:45:29.499 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_total_time": 0.000935441, 2024-04-25T00:45:29.499 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_send_time": 0.000000000, 2024-04-25T00:45:29.499 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_recv_time": 0.000131103, 2024-04-25T00:45:29.499 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-04-25T00:45:29.499 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:29.500 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:29.500 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:29.500 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:29.500 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:29.500 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:29.500 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:29.500 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:29.500 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:29.500 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:29.500 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_encrypted_bytes": 0, 2024-04-25T00:45:29.500 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_encrypted_bytes": 0 2024-04-25T00:45:29.500 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:29.500 INFO:teuthology.orchestra.run.smithi130.stdout: "AsyncMessenger::Worker-1": { 2024-04-25T00:45:29.500 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_messages": 261, 2024-04-25T00:45:29.500 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages": 248, 2024-04-25T00:45:29.500 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_bytes": 1593952, 2024-04-25T00:45:29.501 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_bytes": 151888, 2024-04-25T00:45:29.501 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_created_connections": 2, 2024-04-25T00:45:29.501 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_active_connections": 1, 2024-04-25T00:45:29.501 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_total_time": 0.053028681, 2024-04-25T00:45:29.501 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_send_time": 0.014931339, 2024-04-25T00:45:29.501 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_recv_time": 0.040226433, 2024-04-25T00:45:29.501 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_fast_dispatch_time": 0.007416526, 2024-04-25T00:45:29.501 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:29.501 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 248, 2024-04-25T00:45:29.501 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.013393996, 2024-04-25T00:45:29.501 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000054008 2024-04-25T00:45:29.501 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:29.501 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:29.501 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:29.501 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:29.502 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:29.502 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:29.502 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_encrypted_bytes": 1593952, 2024-04-25T00:45:29.502 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_encrypted_bytes": 151888 2024-04-25T00:45:29.502 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:29.502 INFO:teuthology.orchestra.run.smithi130.stdout: "AsyncMessenger::Worker-2": { 2024-04-25T00:45:29.502 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_messages": 2, 2024-04-25T00:45:29.502 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages": 189, 2024-04-25T00:45:29.502 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_bytes": 320, 2024-04-25T00:45:29.502 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_bytes": 188544, 2024-04-25T00:45:29.502 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_created_connections": 5, 2024-04-25T00:45:29.502 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_active_connections": 3, 2024-04-25T00:45:29.502 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_total_time": 0.012057004, 2024-04-25T00:45:29.502 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_send_time": 0.008087921, 2024-04-25T00:45:29.503 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_recv_time": 0.001532523, 2024-04-25T00:45:29.503 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-04-25T00:45:29.503 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages_queue_lat": { 2024-04-25T00:45:29.503 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 186, 2024-04-25T00:45:29.503 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.020199080, 2024-04-25T00:45:29.503 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000108597 2024-04-25T00:45:29.503 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:29.503 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_handle_ack_lat": { 2024-04-25T00:45:29.503 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:29.503 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:29.503 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:29.503 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:29.503 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_encrypted_bytes": 320, 2024-04-25T00:45:29.503 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_encrypted_bytes": 188544 2024-04-25T00:45:29.504 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:29.504 INFO:teuthology.orchestra.run.smithi130.stdout: "cct": { 2024-04-25T00:45:29.504 INFO:teuthology.orchestra.run.smithi130.stdout: "total_workers": 0, 2024-04-25T00:45:29.504 INFO:teuthology.orchestra.run.smithi130.stdout: "unhealthy_workers": 0 2024-04-25T00:45:29.504 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:29.504 INFO:teuthology.orchestra.run.smithi130.stdout: "mempool": { 2024-04-25T00:45:29.504 INFO:teuthology.orchestra.run.smithi130.stdout: "bloom_filter_bytes": 0, 2024-04-25T00:45:29.504 INFO:teuthology.orchestra.run.smithi130.stdout: "bloom_filter_items": 0, 2024-04-25T00:45:29.504 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_alloc_bytes": 0, 2024-04-25T00:45:29.504 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_alloc_items": 0, 2024-04-25T00:45:29.504 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_data_bytes": 0, 2024-04-25T00:45:29.504 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_data_items": 0, 2024-04-25T00:45:29.504 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_onode_bytes": 0, 2024-04-25T00:45:29.504 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_onode_items": 0, 2024-04-25T00:45:29.504 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_meta_bytes": 0, 2024-04-25T00:45:29.505 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_meta_items": 0, 2024-04-25T00:45:29.505 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_other_bytes": 0, 2024-04-25T00:45:29.505 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_other_items": 0, 2024-04-25T00:45:29.505 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_buffer_bytes": 0, 2024-04-25T00:45:29.505 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_buffer_items": 0, 2024-04-25T00:45:29.505 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_extent_bytes": 0, 2024-04-25T00:45:29.505 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_extent_items": 0, 2024-04-25T00:45:29.505 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_blob_bytes": 0, 2024-04-25T00:45:29.505 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_blob_items": 0, 2024-04-25T00:45:29.505 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_shared_blob_bytes": 0, 2024-04-25T00:45:29.505 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_shared_blob_items": 0, 2024-04-25T00:45:29.505 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_inline_bl_bytes": 0, 2024-04-25T00:45:29.505 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_inline_bl_items": 0, 2024-04-25T00:45:29.505 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_bytes": 0, 2024-04-25T00:45:29.506 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_items": 0, 2024-04-25T00:45:29.506 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_txc_bytes": 0, 2024-04-25T00:45:29.506 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_txc_items": 0, 2024-04-25T00:45:29.506 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_deferred_bytes": 0, 2024-04-25T00:45:29.506 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_deferred_items": 0, 2024-04-25T00:45:29.506 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_bytes": 0, 2024-04-25T00:45:29.506 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_items": 0, 2024-04-25T00:45:29.506 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_bytes": 0, 2024-04-25T00:45:29.506 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_items": 0, 2024-04-25T00:45:29.506 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_reader_bytes": 0, 2024-04-25T00:45:29.506 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_reader_items": 0, 2024-04-25T00:45:29.506 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_writer_bytes": 0, 2024-04-25T00:45:29.506 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_writer_items": 0, 2024-04-25T00:45:29.506 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_anon_bytes": 148670, 2024-04-25T00:45:29.507 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_anon_items": 30, 2024-04-25T00:45:29.507 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_meta_bytes": 0, 2024-04-25T00:45:29.507 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_meta_items": 0, 2024-04-25T00:45:29.507 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_bytes": 0, 2024-04-25T00:45:29.507 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_items": 0, 2024-04-25T00:45:29.507 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mapbl_bytes": 0, 2024-04-25T00:45:29.507 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mapbl_items": 0, 2024-04-25T00:45:29.508 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pglog_bytes": 0, 2024-04-25T00:45:29.508 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pglog_items": 0, 2024-04-25T00:45:29.508 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_bytes": 0, 2024-04-25T00:45:29.508 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_items": 0, 2024-04-25T00:45:29.508 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_mapping_bytes": 0, 2024-04-25T00:45:29.508 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_mapping_items": 0, 2024-04-25T00:45:29.508 INFO:teuthology.orchestra.run.smithi130.stdout: "pgmap_bytes": 0, 2024-04-25T00:45:29.508 INFO:teuthology.orchestra.run.smithi130.stdout: "pgmap_items": 0, 2024-04-25T00:45:29.508 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_co_bytes": 928, 2024-04-25T00:45:29.508 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_co_items": 2, 2024-04-25T00:45:29.508 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_1_bytes": 0, 2024-04-25T00:45:29.508 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_1_items": 0, 2024-04-25T00:45:29.508 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_2_bytes": 0, 2024-04-25T00:45:29.508 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_2_items": 0 2024-04-25T00:45:29.508 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-25T00:45:29.509 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-04-25T00:45:29.509 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2024-04-25T00:45:29.509 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 104857600, 2024-04-25T00:45:29.509 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2024-04-25T00:45:29.509 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 259, 2024-04-25T00:45:29.509 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 1554707, 2024-04-25T00:45:29.509 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2024-04-25T00:45:29.509 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 259, 2024-04-25T00:45:29.509 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 0, 2024-04-25T00:45:29.509 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 0, 2024-04-25T00:45:29.509 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 259, 2024-04-25T00:45:29.509 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 1554707, 2024-04-25T00:45:29.509 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2024-04-25T00:45:29.510 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2024-04-25T00:45:29.510 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2024-04-25T00:45:29.510 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2024-04-25T00:45:29.510 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:29.510 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-25T00:45:29.510 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-25T00:45:29.813 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:29 smithi130 ceph-mon[32525]: pgmap v381: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 6 op/s 2024-04-25T00:45:29.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:29 smithi121 ceph-mon[32231]: pgmap v381: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 6 op/s 2024-04-25T00:45:29.998 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:29 smithi032 ceph-mon[29357]: pgmap v381: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 6 op/s 2024-04-25T00:45:30.143 WARNING:tasks.check_counter:Counter 'mds.exported' not found on daemon mds.l 2024-04-25T00:45:30.143 WARNING:tasks.check_counter:Counter 'mds.imported' not found on daemon mds.l 2024-04-25T00:45:30.143 WARNING:tasks.check_counter:Counter 'mds.root_rsnaps' not found on daemon mds.l 2024-04-25T00:45:30.143 WARNING:tasks.check_counter:Counter 'mds_server.req_mksnap_latency.avgcount' not found on daemon mds.l 2024-04-25T00:45:30.143 WARNING:tasks.check_counter:Counter 'mds_server.req_rmsnap_latency.avgcount' not found on daemon mds.l 2024-04-25T00:45:30.143 DEBUG:teuthology.run_tasks:Unwinding manager kclient 2024-04-25T00:45:30.154 INFO:tasks.kclient:Unmounting kernel clients... 2024-04-25T00:45:30.154 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-04-25T00:45:30.154 DEBUG:teuthology.orchestra.run.smithi032:> dd if=/proc/self/mounts of=/dev/stdout 2024-04-25T00:45:30.193 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-04-25T00:45:30.193 DEBUG:teuthology.orchestra.run.smithi032:> dd if=/proc/self/mounts of=/dev/stdout 2024-04-25T00:45:30.253 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-25T00:45:30.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.411+0000 7f3144b9d640 1 -- 172.21.15.32:0/3272578603 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f31401020e0 msgr2=0x7f314010e5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:30.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.411+0000 7f3144b9d640 1 --2- 172.21.15.32:0/3272578603 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f31401020e0 0x7f314010e5f0 secure :-1 s=READY pgs=658 cs=0 l=1 rev1=1 crypto rx=0x7f3138009cd0 tx=0x7f313802a8e0 comp rx=0 tx=0).stop 2024-04-25T00:45:30.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.411+0000 7f3144b9d640 1 -- 172.21.15.32:0/3272578603 shutdown_connections 2024-04-25T00:45:30.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.411+0000 7f3144b9d640 1 --2- 172.21.15.32:0/3272578603 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f31401020e0 0x7f314010e5f0 unknown :-1 s=CLOSED pgs=658 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:30.412 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.411+0000 7f3144b9d640 1 --2- 172.21.15.32:0/3272578603 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3140101720 0x7f3140101ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:30.413 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.411+0000 7f3144b9d640 1 --2- 172.21.15.32:0/3272578603 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3140107730 0x7f3140107b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:30.413 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.411+0000 7f3144b9d640 1 -- 172.21.15.32:0/3272578603 >> 172.21.15.32:0/3272578603 conn(0x7f31400fd1d0 msgr2=0x7f31400ff5f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:30.413 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.411+0000 7f3144b9d640 1 -- 172.21.15.32:0/3272578603 shutdown_connections 2024-04-25T00:45:30.413 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.411+0000 7f3144b9d640 1 -- 172.21.15.32:0/3272578603 wait complete. 2024-04-25T00:45:30.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.412+0000 7f3144b9d640 1 Processor -- start 2024-04-25T00:45:30.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.412+0000 7f3144b9d640 1 -- start start 2024-04-25T00:45:30.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.413+0000 7f3144b9d640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3140101720 0x7f3140073a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:30.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.413+0000 7f3144b9d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f31401020e0 0x7f3140073f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:30.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.413+0000 7f3144b9d640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3140107730 0x7f31400744c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:30.415 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.413+0000 7f3144b9d640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f3140111b00 con 0x7f31401020e0 2024-04-25T00:45:30.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.413+0000 7f3144b9d640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f3140111980 con 0x7f3140101720 2024-04-25T00:45:30.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.413+0000 7f3144b9d640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f3140111c80 con 0x7f3140107730 2024-04-25T00:45:30.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.413+0000 7f3135d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f31401020e0 0x7f3140073f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:30.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.413+0000 7f3135d9b640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f31401020e0 0x7f3140073f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:56700/0 (socket says 172.21.15.32:56700) 2024-04-25T00:45:30.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.413+0000 7f3135d9b640 1 -- 172.21.15.32:0/1728972212 learned_addr learned my addr 172.21.15.32:0/1728972212 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:45:30.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.413+0000 7f313e59c640 1 --2- 172.21.15.32:0/1728972212 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3140101720 0x7f3140073a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:30.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.413+0000 7f313ed9d640 1 --2- 172.21.15.32:0/1728972212 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3140107730 0x7f31400744c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:30.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.413+0000 7f3135d9b640 1 -- 172.21.15.32:0/1728972212 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3140107730 msgr2=0x7f31400744c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:30.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.413+0000 7f3135d9b640 1 --2- 172.21.15.32:0/1728972212 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3140107730 0x7f31400744c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:30.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.413+0000 7f3135d9b640 1 -- 172.21.15.32:0/1728972212 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3140101720 msgr2=0x7f3140073a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:30.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.413+0000 7f3135d9b640 1 --2- 172.21.15.32:0/1728972212 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3140101720 0x7f3140073a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:30.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.413+0000 7f3135d9b640 1 -- 172.21.15.32:0/1728972212 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f31401aa920 con 0x7f31401020e0 2024-04-25T00:45:30.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.413+0000 7f3135d9b640 1 --2- 172.21.15.32:0/1728972212 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f31401020e0 0x7f3140073f80 secure :-1 s=READY pgs=659 cs=0 l=1 rev1=1 crypto rx=0x7f312800d950 tx=0x7f312800dd50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:30.416 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.413+0000 7f3137fff640 1 -- 172.21.15.32:0/1728972212 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3128017070 con 0x7f31401020e0 2024-04-25T00:45:30.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.413+0000 7f3144b9d640 1 -- 172.21.15.32:0/1728972212 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f31401aab50 con 0x7f31401020e0 2024-04-25T00:45:30.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.413+0000 7f3137fff640 1 -- 172.21.15.32:0/1728972212 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f312800de80 con 0x7f31401020e0 2024-04-25T00:45:30.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.413+0000 7f3144b9d640 1 -- 172.21.15.32:0/1728972212 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f31401ab0f0 con 0x7f31401020e0 2024-04-25T00:45:30.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.413+0000 7f3137fff640 1 -- 172.21.15.32:0/1728972212 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3128020060 con 0x7f31401020e0 2024-04-25T00:45:30.417 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.414+0000 7f3144b9d640 1 -- 172.21.15.32:0/1728972212 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f3140102e60 con 0x7f31401020e0 2024-04-25T00:45:30.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.416+0000 7f3137fff640 1 -- 172.21.15.32:0/1728972212 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f3128009c30 con 0x7f31401020e0 2024-04-25T00:45:30.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.416+0000 7f3137fff640 1 --2- 172.21.15.32:0/1728972212 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f312c0a2d00 0x7f312c0a51c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:30.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.417+0000 7f3137fff640 1 -- 172.21.15.32:0/1728972212 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(107..107 src has 1..107) ==== 9554+0+0 (secure 0 0 0) 0x7f31280cfae0 con 0x7f31401020e0 2024-04-25T00:45:30.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.417+0000 7f313e59c640 1 --2- 172.21.15.32:0/1728972212 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f312c0a2d00 0x7f312c0a51c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:30.418 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.417+0000 7f313e59c640 1 --2- 172.21.15.32:0/1728972212 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f312c0a2d00 0x7f312c0a51c0 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f313000ab60 tx=0x7f313000a2d0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:30.420 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.420+0000 7f3137fff640 1 -- 172.21.15.32:0/1728972212 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f312809c2f0 con 0x7f31401020e0 2024-04-25T00:45:30.609 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.609+0000 7f3144b9d640 1 -- 172.21.15.32:0/1728972212 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f3140068560 con 0x7f31401020e0 2024-04-25T00:45:30.611 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.610+0000 7f3137fff640 1 -- 172.21.15.32:0/1728972212 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v107) ==== 74+0+25405 (secure 0 0 0) 0x7f312809bbf0 con 0x7f31401020e0 2024-04-25T00:45:30.611 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:45:30.611 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":107,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:45:17.009986+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-25T00:30:46.239556+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-25T00:30:47.257182+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"106","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":19,"snap_epoch":106,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2024-04-25T00:30:51.193975+0000","flags":8197,"flags_names":"hashpspool,ec_overwrites,selfmanaged_snaps","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"106","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":19,"snap_epoch":106,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.b","mappings":[{"from":8,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":1}]},{"pgid":"2.f","mappings":[{"from":8,"to":11}]},{"pgid":"2.10","mappings":[{"from":2,"to":4}]},{"pgid":"3.6","mappings":[{"from":3,"to":6}]},{"pgid":"3.17","mappings":[{"from":4,"to":7}]},{"pgid":"3.1b","mappings":[{"from":5,"to":9}]},{"pgid":"3.20","mappings":[{"from":5,"to":9}]},{"pgid":"3.23","mappings":[{"from":4,"to":8}]},{"pgid":"3.2c","mappings":[{"from":4,"to":9}]},{"pgid":"4.c","mappings":[{"from":9,"to":10}]},{"pgid":"4.12","mappings":[{"from":9,"to":10}]},{"pgid":"4.1c","mappings":[{"from":9,"to":8}]},{"pgid":"4.1d","mappings":[{"from":5,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:6801/585810931":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/1096527590":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/1767429535":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/3915353181":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6800/585810931":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/1694990206":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-num-failure-domains":"0","crush-osds-per-failure-domain":"0","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":3,"snaps":[{"begin":7,"length":1},{"begin":19,"length":1}]},{"pool":4,"snaps":[{"begin":7,"length":1},{"begin":19,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:45:30.613 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.612+0000 7f3144b9d640 1 -- 172.21.15.32:0/1728972212 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f312c0a2d00 msgr2=0x7f312c0a51c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:30.613 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.612+0000 7f3144b9d640 1 --2- 172.21.15.32:0/1728972212 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f312c0a2d00 0x7f312c0a51c0 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f313000ab60 tx=0x7f313000a2d0 comp rx=0 tx=0).stop 2024-04-25T00:45:30.613 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.612+0000 7f3144b9d640 1 -- 172.21.15.32:0/1728972212 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f31401020e0 msgr2=0x7f3140073f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:30.613 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.612+0000 7f3144b9d640 1 --2- 172.21.15.32:0/1728972212 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f31401020e0 0x7f3140073f80 secure :-1 s=READY pgs=659 cs=0 l=1 rev1=1 crypto rx=0x7f312800d950 tx=0x7f312800dd50 comp rx=0 tx=0).stop 2024-04-25T00:45:30.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.612+0000 7f3144b9d640 1 -- 172.21.15.32:0/1728972212 shutdown_connections 2024-04-25T00:45:30.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.612+0000 7f3144b9d640 1 --2- 172.21.15.32:0/1728972212 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f312c0a2d00 0x7f312c0a51c0 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:30.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.612+0000 7f3144b9d640 1 --2- 172.21.15.32:0/1728972212 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3140107730 0x7f31400744c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:30.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.612+0000 7f3144b9d640 1 --2- 172.21.15.32:0/1728972212 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f31401020e0 0x7f3140073f80 unknown :-1 s=CLOSED pgs=659 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:30.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.612+0000 7f3144b9d640 1 --2- 172.21.15.32:0/1728972212 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f3140101720 0x7f3140073a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:30.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.612+0000 7f3144b9d640 1 -- 172.21.15.32:0/1728972212 >> 172.21.15.32:0/1728972212 conn(0x7f31400fd1d0 msgr2=0x7f31400fd710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:30.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.613+0000 7f3144b9d640 1 -- 172.21.15.32:0/1728972212 shutdown_connections 2024-04-25T00:45:30.614 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.613+0000 7f3144b9d640 1 -- 172.21.15.32:0/1728972212 wait complete. 2024-04-25T00:45:30.624 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-25T00:26:07.306839+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-04-25T00:30:46.239556+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '84', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.8799999952316284, 'score_stable': 1.8799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.8799999952316284, 'raw_score_stable': 1.8799999952316284, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-04-25T00:30:47.257182+0000', 'flags': 8193, 'flags_names': 'hashpspool,selfmanaged_snaps', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '106', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 19, 'snap_epoch': 106, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.05000000074505806}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_data_ec', 'create_time': '2024-04-25T00:30:51.193975+0000', 'flags': 8197, 'flags_names': 'hashpspool,ec_overwrites,selfmanaged_snaps', 'type': 3, 'size': 4, 'min_size': 3, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '106', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 19, 'snap_epoch': 106, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'cephfs_data_ec', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 8192, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}] 2024-04-25T00:45:30.624 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-25T00:45:30.732 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.729+0000 7fec558da640 1 -- 172.21.15.32:0/2507423917 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fec501002e0 msgr2=0x7fec501006c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:30.732 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.729+0000 7fec558da640 1 --2- 172.21.15.32:0/2507423917 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fec501002e0 0x7fec501006c0 secure :-1 s=READY pgs=660 cs=0 l=1 rev1=1 crypto rx=0x7fec40009900 tx=0x7fec4002a410 comp rx=0 tx=0).stop 2024-04-25T00:45:30.732 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.729+0000 7fec558da640 1 -- 172.21.15.32:0/2507423917 shutdown_connections 2024-04-25T00:45:30.732 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.729+0000 7fec558da640 1 --2- 172.21.15.32:0/2507423917 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fec50106aa0 0x7fec5010e6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:30.732 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.729+0000 7fec558da640 1 --2- 172.21.15.32:0/2507423917 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fec50100c90 0x7fec501063c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:30.732 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.729+0000 7fec558da640 1 --2- 172.21.15.32:0/2507423917 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fec501002e0 0x7fec501006c0 unknown :-1 s=CLOSED pgs=660 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:30.732 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.729+0000 7fec558da640 1 -- 172.21.15.32:0/2507423917 >> 172.21.15.32:0/2507423917 conn(0x7fec500758d0 msgr2=0x7fec50075ce0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:30.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.729+0000 7fec558da640 1 -- 172.21.15.32:0/2507423917 shutdown_connections 2024-04-25T00:45:30.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.729+0000 7fec558da640 1 -- 172.21.15.32:0/2507423917 wait complete. 2024-04-25T00:45:30.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.730+0000 7fec558da640 1 Processor -- start 2024-04-25T00:45:30.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.730+0000 7fec558da640 1 -- start start 2024-04-25T00:45:30.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.730+0000 7fec558da640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fec501002e0 0x7fec50101250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:30.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.730+0000 7fec558da640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fec50106aa0 0x7fec50101790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:30.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.730+0000 7fec558da640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fec50101cd0 0x7fec501a6300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:30.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.730+0000 7fec558da640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fec50111bf0 con 0x7fec50101cd0 2024-04-25T00:45:30.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.730+0000 7fec558da640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fec50111a70 con 0x7fec501002e0 2024-04-25T00:45:30.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.730+0000 7fec558da640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fec50111d70 con 0x7fec50106aa0 2024-04-25T00:45:30.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.730+0000 7fec4f7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fec50101cd0 0x7fec501a6300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:30.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.730+0000 7fec4f7fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fec50101cd0 0x7fec501a6300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:56716/0 (socket says 172.21.15.32:56716) 2024-04-25T00:45:30.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.730+0000 7fec4f7fe640 1 -- 172.21.15.32:0/896577190 learned_addr learned my addr 172.21.15.32:0/896577190 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:45:30.733 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.730+0000 7fec4effd640 1 --2- 172.21.15.32:0/896577190 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fec501002e0 0x7fec50101250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:30.734 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.731+0000 7fec4e7fc640 1 --2- 172.21.15.32:0/896577190 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fec50106aa0 0x7fec50101790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:30.734 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.731+0000 7fec4f7fe640 1 -- 172.21.15.32:0/896577190 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fec50106aa0 msgr2=0x7fec50101790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:30.734 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.731+0000 7fec4f7fe640 1 --2- 172.21.15.32:0/896577190 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fec50106aa0 0x7fec50101790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:30.734 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.731+0000 7fec4f7fe640 1 -- 172.21.15.32:0/896577190 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fec501002e0 msgr2=0x7fec50101250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:30.734 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.731+0000 7fec4f7fe640 1 --2- 172.21.15.32:0/896577190 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fec501002e0 0x7fec50101250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:30.734 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.731+0000 7fec4f7fe640 1 -- 172.21.15.32:0/896577190 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fec501a6a00 con 0x7fec50101cd0 2024-04-25T00:45:30.734 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.731+0000 7fec4f7fe640 1 --2- 172.21.15.32:0/896577190 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fec50101cd0 0x7fec501a6300 secure :-1 s=READY pgs=661 cs=0 l=1 rev1=1 crypto rx=0x7fec4800eaa0 tx=0x7fec4800ef60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:30.734 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.731+0000 7fec548d8640 1 -- 172.21.15.32:0/896577190 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fec4800cdb0 con 0x7fec50101cd0 2024-04-25T00:45:30.734 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.731+0000 7fec548d8640 1 -- 172.21.15.32:0/896577190 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fec48005020 con 0x7fec50101cd0 2024-04-25T00:45:30.734 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.731+0000 7fec558da640 1 -- 172.21.15.32:0/896577190 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fec501a6cf0 con 0x7fec50101cd0 2024-04-25T00:45:30.734 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.731+0000 7fec548d8640 1 -- 172.21.15.32:0/896577190 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fec480116b0 con 0x7fec50101cd0 2024-04-25T00:45:30.734 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.731+0000 7fec558da640 1 -- 172.21.15.32:0/896577190 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fec501a7240 con 0x7fec50101cd0 2024-04-25T00:45:30.735 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.732+0000 7fec558da640 1 -- 172.21.15.32:0/896577190 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fec500689b0 con 0x7fec50101cd0 2024-04-25T00:45:30.735 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.733+0000 7fec548d8640 1 -- 172.21.15.32:0/896577190 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7fec48005560 con 0x7fec50101cd0 2024-04-25T00:45:30.735 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.734+0000 7fec548d8640 1 --2- 172.21.15.32:0/896577190 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fec380a2db0 0x7fec380a5270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:30.735 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.734+0000 7fec4effd640 1 --2- 172.21.15.32:0/896577190 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fec380a2db0 0x7fec380a5270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:30.735 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.734+0000 7fec548d8640 1 -- 172.21.15.32:0/896577190 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(107..107 src has 1..107) ==== 9554+0+0 (secure 0 0 0) 0x7fec480cfa30 con 0x7fec50101cd0 2024-04-25T00:45:30.735 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.734+0000 7fec4effd640 1 --2- 172.21.15.32:0/896577190 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fec380a2db0 0x7fec380a5270 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7fec40009230 tx=0x7fec400058d0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:30.738 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.738+0000 7fec548d8640 1 -- 172.21.15.32:0/896577190 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fec48016030 con 0x7fec50101cd0 2024-04-25T00:45:30.923 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.923+0000 7fec558da640 1 -- 172.21.15.32:0/896577190 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7fec50109ca0 con 0x7fec50101cd0 2024-04-25T00:45:30.924 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.924+0000 7fec548d8640 1 -- 172.21.15.32:0/896577190 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v107) ==== 93+0+10 (secure 0 0 0) 0x7fec4809c240 con 0x7fec50101cd0 2024-04-25T00:45:30.924 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 1 2024-04-25T00:45:30.926 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.926+0000 7fec558da640 1 -- 172.21.15.32:0/896577190 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fec380a2db0 msgr2=0x7fec380a5270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:30.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.926+0000 7fec558da640 1 --2- 172.21.15.32:0/896577190 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fec380a2db0 0x7fec380a5270 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7fec40009230 tx=0x7fec400058d0 comp rx=0 tx=0).stop 2024-04-25T00:45:30.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.926+0000 7fec558da640 1 -- 172.21.15.32:0/896577190 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fec50101cd0 msgr2=0x7fec501a6300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:30.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.926+0000 7fec558da640 1 --2- 172.21.15.32:0/896577190 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fec50101cd0 0x7fec501a6300 secure :-1 s=READY pgs=661 cs=0 l=1 rev1=1 crypto rx=0x7fec4800eaa0 tx=0x7fec4800ef60 comp rx=0 tx=0).stop 2024-04-25T00:45:30.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.926+0000 7fec558da640 1 -- 172.21.15.32:0/896577190 shutdown_connections 2024-04-25T00:45:30.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.926+0000 7fec558da640 1 --2- 172.21.15.32:0/896577190 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fec380a2db0 0x7fec380a5270 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:30.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.926+0000 7fec558da640 1 --2- 172.21.15.32:0/896577190 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fec50101cd0 0x7fec501a6300 unknown :-1 s=CLOSED pgs=661 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:30.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.926+0000 7fec558da640 1 --2- 172.21.15.32:0/896577190 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fec50106aa0 0x7fec50101790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:30.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.926+0000 7fec558da640 1 --2- 172.21.15.32:0/896577190 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fec501002e0 0x7fec50101250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:30.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.926+0000 7fec558da640 1 -- 172.21.15.32:0/896577190 >> 172.21.15.32:0/896577190 conn(0x7fec500758d0 msgr2=0x7fec500fe380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:30.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.926+0000 7fec558da640 1 -- 172.21.15.32:0/896577190 shutdown_connections 2024-04-25T00:45:30.927 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:30.926+0000 7fec558da640 1 -- 172.21.15.32:0/896577190 wait complete. 2024-04-25T00:45:30.937 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-04-25T00:45:31.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.042+0000 7f9fe32ae640 1 -- 172.21.15.32:0/2780407084 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fdc1032e0 msgr2=0x7f9fdc10b7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:31.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.042+0000 7f9fe32ae640 1 --2- 172.21.15.32:0/2780407084 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fdc1032e0 0x7f9fdc10b7c0 secure :-1 s=READY pgs=662 cs=0 l=1 rev1=1 crypto rx=0x7f9fcc009900 tx=0x7f9fcc02a510 comp rx=0 tx=0).stop 2024-04-25T00:45:31.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.042+0000 7f9fe32ae640 1 -- 172.21.15.32:0/2780407084 shutdown_connections 2024-04-25T00:45:31.049 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.042+0000 7f9fe32ae640 1 --2- 172.21.15.32:0/2780407084 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9fdc10bd00 0x7f9fdc10e1a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.042+0000 7f9fe32ae640 1 --2- 172.21.15.32:0/2780407084 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fdc1032e0 0x7f9fdc10b7c0 unknown :-1 s=CLOSED pgs=662 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.042+0000 7f9fe32ae640 1 --2- 172.21.15.32:0/2780407084 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9fdc1029c0 0x7f9fdc102da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.042+0000 7f9fe32ae640 1 -- 172.21.15.32:0/2780407084 >> 172.21.15.32:0/2780407084 conn(0x7f9fdc077740 msgr2=0x7f9fdc077b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:31.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.042+0000 7f9fe32ae640 1 -- 172.21.15.32:0/2780407084 shutdown_connections 2024-04-25T00:45:31.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.042+0000 7f9fe32ae640 1 -- 172.21.15.32:0/2780407084 wait complete. 2024-04-25T00:45:31.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.042+0000 7f9fe32ae640 1 Processor -- start 2024-04-25T00:45:31.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.042+0000 7f9fe32ae640 1 -- start start 2024-04-25T00:45:31.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.043+0000 7f9fe32ae640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fdc1029c0 0x7f9fdc198060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:31.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.043+0000 7f9fe32ae640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9fdc1032e0 0x7f9fdc1985a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:31.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.043+0000 7f9fe32ae640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9fdc10bd00 0x7f9fdc19c8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:31.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.043+0000 7f9fe32ae640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f9fdc111b00 con 0x7f9fdc1029c0 2024-04-25T00:45:31.050 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.043+0000 7f9fe32ae640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f9fdc111980 con 0x7f9fdc10bd00 2024-04-25T00:45:31.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.043+0000 7f9fe32ae640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f9fdc111c80 con 0x7f9fdc1032e0 2024-04-25T00:45:31.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.043+0000 7f9fe104a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fdc1029c0 0x7f9fdc198060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:31.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.043+0000 7f9fe104a640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fdc1029c0 0x7f9fdc198060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:56728/0 (socket says 172.21.15.32:56728) 2024-04-25T00:45:31.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.043+0000 7f9fe104a640 1 -- 172.21.15.32:0/440433825 learned_addr learned my addr 172.21.15.32:0/440433825 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:45:31.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.043+0000 7f9fe0849640 1 --2- 172.21.15.32:0/440433825 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9fdc1032e0 0x7f9fdc1985a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:31.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.043+0000 7f9fe184b640 1 --2- 172.21.15.32:0/440433825 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9fdc10bd00 0x7f9fdc19c8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:31.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.043+0000 7f9fe104a640 1 -- 172.21.15.32:0/440433825 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9fdc1032e0 msgr2=0x7f9fdc1985a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:31.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.043+0000 7f9fe104a640 1 --2- 172.21.15.32:0/440433825 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9fdc1032e0 0x7f9fdc1985a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.043+0000 7f9fe104a640 1 -- 172.21.15.32:0/440433825 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9fdc10bd00 msgr2=0x7f9fdc19c8d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:31.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.043+0000 7f9fe104a640 1 --2- 172.21.15.32:0/440433825 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9fdc10bd00 0x7f9fdc19c8d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.043+0000 7f9fe104a640 1 -- 172.21.15.32:0/440433825 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9fdc19d000 con 0x7f9fdc1029c0 2024-04-25T00:45:31.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.043+0000 7f9fe184b640 1 --2- 172.21.15.32:0/440433825 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9fdc10bd00 0x7f9fdc19c8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:45:31.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.043+0000 7f9fe104a640 1 --2- 172.21.15.32:0/440433825 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fdc1029c0 0x7f9fdc198060 secure :-1 s=READY pgs=663 cs=0 l=1 rev1=1 crypto rx=0x7f9fd800e9f0 tx=0x7f9fd800edf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:31.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.043+0000 7f9fd27fc640 1 -- 172.21.15.32:0/440433825 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9fd800cd80 con 0x7f9fdc1029c0 2024-04-25T00:45:31.051 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.043+0000 7f9fe32ae640 1 -- 172.21.15.32:0/440433825 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9fdc19d2f0 con 0x7f9fdc1029c0 2024-04-25T00:45:31.052 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.043+0000 7f9fd27fc640 1 -- 172.21.15.32:0/440433825 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9fd8005500 con 0x7f9fdc1029c0 2024-04-25T00:45:31.052 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.043+0000 7f9fd27fc640 1 -- 172.21.15.32:0/440433825 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9fd8010690 con 0x7f9fdc1029c0 2024-04-25T00:45:31.052 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.043+0000 7f9fe32ae640 1 -- 172.21.15.32:0/440433825 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9fdc1a4b80 con 0x7f9fdc1029c0 2024-04-25T00:45:31.052 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.044+0000 7f9fe32ae640 1 -- 172.21.15.32:0/440433825 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9fdc068560 con 0x7f9fdc1029c0 2024-04-25T00:45:31.052 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.045+0000 7f9fd27fc640 1 -- 172.21.15.32:0/440433825 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f9fd80046c0 con 0x7f9fdc1029c0 2024-04-25T00:45:31.052 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.046+0000 7f9fd27fc640 1 --2- 172.21.15.32:0/440433825 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f9fc80a2d00 0x7f9fc80a51c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:31.052 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.046+0000 7f9fe0849640 1 --2- 172.21.15.32:0/440433825 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f9fc80a2d00 0x7f9fc80a51c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:31.052 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.046+0000 7f9fd27fc640 1 -- 172.21.15.32:0/440433825 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(107..107 src has 1..107) ==== 9554+0+0 (secure 0 0 0) 0x7f9fd80cfd30 con 0x7f9fdc1029c0 2024-04-25T00:45:31.052 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.047+0000 7f9fe0849640 1 --2- 172.21.15.32:0/440433825 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f9fc80a2d00 0x7f9fc80a51c0 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f9fcc009230 tx=0x7f9fcc0058d0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:31.052 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.051+0000 7f9fd27fc640 1 -- 172.21.15.32:0/440433825 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f9fd809c540 con 0x7f9fdc1029c0 2024-04-25T00:45:31.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.234+0000 7f9fe32ae640 1 -- 172.21.15.32:0/440433825 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) -- 0x7f9fdc106ee0 con 0x7f9fdc1029c0 2024-04-25T00:45:31.235 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.235+0000 7f9fd27fc640 1 -- 172.21.15.32:0/440433825 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v107) ==== 104+0+11 (secure 0 0 0) 0x7f9fd809be40 con 0x7f9fdc1029c0 2024-04-25T00:45:31.235 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 32 2024-04-25T00:45:31.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.236+0000 7f9fe32ae640 1 -- 172.21.15.32:0/440433825 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f9fc80a2d00 msgr2=0x7f9fc80a51c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:31.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.236+0000 7f9fe32ae640 1 --2- 172.21.15.32:0/440433825 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f9fc80a2d00 0x7f9fc80a51c0 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f9fcc009230 tx=0x7f9fcc0058d0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.237 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.236+0000 7f9fe32ae640 1 -- 172.21.15.32:0/440433825 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fdc1029c0 msgr2=0x7f9fdc198060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:31.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.236+0000 7f9fe32ae640 1 --2- 172.21.15.32:0/440433825 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fdc1029c0 0x7f9fdc198060 secure :-1 s=READY pgs=663 cs=0 l=1 rev1=1 crypto rx=0x7f9fd800e9f0 tx=0x7f9fd800edf0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.237+0000 7f9fe32ae640 1 -- 172.21.15.32:0/440433825 shutdown_connections 2024-04-25T00:45:31.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.237+0000 7f9fe32ae640 1 --2- 172.21.15.32:0/440433825 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f9fc80a2d00 0x7f9fc80a51c0 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.237+0000 7f9fe32ae640 1 --2- 172.21.15.32:0/440433825 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f9fdc10bd00 0x7f9fdc19c8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.237+0000 7f9fe32ae640 1 --2- 172.21.15.32:0/440433825 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9fdc1032e0 0x7f9fdc1985a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.237+0000 7f9fe32ae640 1 --2- 172.21.15.32:0/440433825 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f9fdc1029c0 0x7f9fdc198060 unknown :-1 s=CLOSED pgs=663 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.237+0000 7f9fe32ae640 1 -- 172.21.15.32:0/440433825 >> 172.21.15.32:0/440433825 conn(0x7f9fdc077740 msgr2=0x7f9fdc0fd740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:31.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.237+0000 7f9fe32ae640 1 -- 172.21.15.32:0/440433825 shutdown_connections 2024-04-25T00:45:31.238 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.237+0000 7f9fe32ae640 1 -- 172.21.15.32:0/440433825 wait complete. 2024-04-25T00:45:31.251 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-04-25T00:45:31.357 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.355+0000 7f1e0a693640 1 -- 172.21.15.32:0/2768425301 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1e04100620 msgr2=0x7f1e04100aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:31.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.355+0000 7f1e0a693640 1 --2- 172.21.15.32:0/2768425301 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1e04100620 0x7f1e04100aa0 secure :-1 s=READY pgs=664 cs=0 l=1 rev1=1 crypto rx=0x7f1dec009930 tx=0x7f1dec02a420 comp rx=0 tx=0).stop 2024-04-25T00:45:31.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.355+0000 7f1e0a693640 1 -- 172.21.15.32:0/2768425301 shutdown_connections 2024-04-25T00:45:31.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.355+0000 7f1e0a693640 1 --2- 172.21.15.32:0/2768425301 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1e04100fe0 0x7f1e0410d4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.355+0000 7f1e0a693640 1 --2- 172.21.15.32:0/2768425301 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1e04100620 0x7f1e04100aa0 unknown :-1 s=CLOSED pgs=664 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.355+0000 7f1e0a693640 1 --2- 172.21.15.32:0/2768425301 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1e04106630 0x7f1e04106a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.355+0000 7f1e0a693640 1 -- 172.21.15.32:0/2768425301 >> 172.21.15.32:0/2768425301 conn(0x7f1e040776a0 msgr2=0x7f1e04077ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:31.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.355+0000 7f1e0a693640 1 -- 172.21.15.32:0/2768425301 shutdown_connections 2024-04-25T00:45:31.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.355+0000 7f1e0a693640 1 -- 172.21.15.32:0/2768425301 wait complete. 2024-04-25T00:45:31.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.355+0000 7f1e0a693640 1 Processor -- start 2024-04-25T00:45:31.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.356+0000 7f1e0a693640 1 -- start start 2024-04-25T00:45:31.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.356+0000 7f1e0a693640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1e04100620 0x7f1e04198050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:31.358 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.356+0000 7f1e0a693640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1e04100fe0 0x7f1e04198590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:31.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.356+0000 7f1e0a693640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1e04106630 0x7f1e0419c8c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:31.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.356+0000 7f1e0a693640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f1e04110a00 con 0x7f1e04106630 2024-04-25T00:45:31.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.356+0000 7f1e0a693640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f1e04110880 con 0x7f1e04100fe0 2024-04-25T00:45:31.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.356+0000 7f1e0a693640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f1e04110b80 con 0x7f1e04100620 2024-04-25T00:45:31.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.356+0000 7f1e08c30640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1e04106630 0x7f1e0419c8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:31.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.356+0000 7f1e037fe640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1e04100fe0 0x7f1e04198590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:31.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.356+0000 7f1e08c30640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1e04106630 0x7f1e0419c8c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:56742/0 (socket says 172.21.15.32:56742) 2024-04-25T00:45:31.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.356+0000 7f1e08c30640 1 -- 172.21.15.32:0/1138571269 learned_addr learned my addr 172.21.15.32:0/1138571269 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:45:31.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.356+0000 7f1e03fff640 1 --2- 172.21.15.32:0/1138571269 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1e04100620 0x7f1e04198050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:31.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.356+0000 7f1e08c30640 1 -- 172.21.15.32:0/1138571269 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1e04100620 msgr2=0x7f1e04198050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:31.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.356+0000 7f1e08c30640 1 --2- 172.21.15.32:0/1138571269 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1e04100620 0x7f1e04198050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.356+0000 7f1e08c30640 1 -- 172.21.15.32:0/1138571269 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1e04100fe0 msgr2=0x7f1e04198590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:31.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.356+0000 7f1e08c30640 1 --2- 172.21.15.32:0/1138571269 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1e04100fe0 0x7f1e04198590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.356+0000 7f1e08c30640 1 -- 172.21.15.32:0/1138571269 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1e0419cff0 con 0x7f1e04106630 2024-04-25T00:45:31.359 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.356+0000 7f1e037fe640 1 --2- 172.21.15.32:0/1138571269 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1e04100fe0 0x7f1e04198590 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:45:31.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.357+0000 7f1e08c30640 1 --2- 172.21.15.32:0/1138571269 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1e04106630 0x7f1e0419c8c0 secure :-1 s=READY pgs=665 cs=0 l=1 rev1=1 crypto rx=0x7f1dfc00eb10 tx=0x7f1dfc00ef10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:31.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.357+0000 7f1e017fa640 1 -- 172.21.15.32:0/1138571269 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1dfc00ce20 con 0x7f1e04106630 2024-04-25T00:45:31.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.357+0000 7f1e0a693640 1 -- 172.21.15.32:0/1138571269 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1e0419d2e0 con 0x7f1e04106630 2024-04-25T00:45:31.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.357+0000 7f1e017fa640 1 -- 172.21.15.32:0/1138571269 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1dfc005020 con 0x7f1e04106630 2024-04-25T00:45:31.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.357+0000 7f1e017fa640 1 -- 172.21.15.32:0/1138571269 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1dfc010690 con 0x7f1e04106630 2024-04-25T00:45:31.360 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.357+0000 7f1e0a693640 1 -- 172.21.15.32:0/1138571269 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1e041a4b70 con 0x7f1e04106630 2024-04-25T00:45:31.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.358+0000 7f1e0a693640 1 -- 172.21.15.32:0/1138571269 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1e04101d60 con 0x7f1e04106630 2024-04-25T00:45:31.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.359+0000 7f1e017fa640 1 -- 172.21.15.32:0/1138571269 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f1dfc005540 con 0x7f1e04106630 2024-04-25T00:45:31.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.359+0000 7f1e017fa640 1 --2- 172.21.15.32:0/1138571269 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f1df00a2d00 0x7f1df00a51c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:31.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.360+0000 7f1e017fa640 1 -- 172.21.15.32:0/1138571269 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(107..107 src has 1..107) ==== 9554+0+0 (secure 0 0 0) 0x7f1dfc0cfe20 con 0x7f1e04106630 2024-04-25T00:45:31.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.360+0000 7f1e03fff640 1 --2- 172.21.15.32:0/1138571269 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f1df00a2d00 0x7f1df00a51c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:31.361 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.360+0000 7f1e03fff640 1 --2- 172.21.15.32:0/1138571269 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f1df00a2d00 0x7f1df00a51c0 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f1df400ab60 tx=0x7f1df400a2d0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:31.364 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.364+0000 7f1e017fa640 1 -- 172.21.15.32:0/1138571269 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f1dfc09c630 con 0x7f1e04106630 2024-04-25T00:45:31.549 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.548+0000 7f1e0a693640 1 -- 172.21.15.32:0/1138571269 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) -- 0x7f1e041a4da0 con 0x7f1e04106630 2024-04-25T00:45:31.549 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.548+0000 7f1e017fa640 1 -- 172.21.15.32:0/1138571269 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v107) ==== 100+0+11 (secure 0 0 0) 0x7f1dfc09bf30 con 0x7f1e04106630 2024-04-25T00:45:31.549 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 64 2024-04-25T00:45:31.551 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.550+0000 7f1e0a693640 1 -- 172.21.15.32:0/1138571269 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f1df00a2d00 msgr2=0x7f1df00a51c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:31.551 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.550+0000 7f1e0a693640 1 --2- 172.21.15.32:0/1138571269 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f1df00a2d00 0x7f1df00a51c0 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f1df400ab60 tx=0x7f1df400a2d0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.551 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.550+0000 7f1e0a693640 1 -- 172.21.15.32:0/1138571269 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1e04106630 msgr2=0x7f1e0419c8c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:31.551 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.550+0000 7f1e0a693640 1 --2- 172.21.15.32:0/1138571269 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1e04106630 0x7f1e0419c8c0 secure :-1 s=READY pgs=665 cs=0 l=1 rev1=1 crypto rx=0x7f1dfc00eb10 tx=0x7f1dfc00ef10 comp rx=0 tx=0).stop 2024-04-25T00:45:31.551 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.550+0000 7f1e0a693640 1 -- 172.21.15.32:0/1138571269 shutdown_connections 2024-04-25T00:45:31.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.550+0000 7f1e0a693640 1 --2- 172.21.15.32:0/1138571269 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f1df00a2d00 0x7f1df00a51c0 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.551+0000 7f1e0a693640 1 --2- 172.21.15.32:0/1138571269 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1e04106630 0x7f1e0419c8c0 unknown :-1 s=CLOSED pgs=665 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.551+0000 7f1e0a693640 1 --2- 172.21.15.32:0/1138571269 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1e04100fe0 0x7f1e04198590 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.551+0000 7f1e0a693640 1 --2- 172.21.15.32:0/1138571269 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1e04100620 0x7f1e04198050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.551+0000 7f1e0a693640 1 -- 172.21.15.32:0/1138571269 >> 172.21.15.32:0/1138571269 conn(0x7f1e040776a0 msgr2=0x7f1e040ff830 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:31.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.551+0000 7f1e0a693640 1 -- 172.21.15.32:0/1138571269 shutdown_connections 2024-04-25T00:45:31.552 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.551+0000 7f1e0a693640 1 -- 172.21.15.32:0/1138571269 wait complete. 2024-04-25T00:45:31.561 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data_ec pg_num 2024-04-25T00:45:31.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.666+0000 7fc259b29640 1 -- 172.21.15.32:0/1201497928 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc2541077a0 msgr2=0x7fc254107b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:31.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.666+0000 7fc259b29640 1 --2- 172.21.15.32:0/1201497928 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc2541077a0 0x7fc254107b80 secure :-1 s=READY pgs=666 cs=0 l=1 rev1=1 crypto rx=0x7fc244009cd0 tx=0x7fc24402a8e0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.666+0000 7fc259b29640 1 -- 172.21.15.32:0/1201497928 shutdown_connections 2024-04-25T00:45:31.668 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.666+0000 7fc259b29640 1 --2- 172.21.15.32:0/1201497928 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc254102150 0x7fc25410e660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.666+0000 7fc259b29640 1 --2- 172.21.15.32:0/1201497928 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc254101790 0x7fc254101c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.666+0000 7fc259b29640 1 --2- 172.21.15.32:0/1201497928 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc2541077a0 0x7fc254107b80 unknown :-1 s=CLOSED pgs=666 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.666+0000 7fc259b29640 1 -- 172.21.15.32:0/1201497928 >> 172.21.15.32:0/1201497928 conn(0x7fc2540fd2a0 msgr2=0x7fc2540ff6c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:31.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.666+0000 7fc259b29640 1 -- 172.21.15.32:0/1201497928 shutdown_connections 2024-04-25T00:45:31.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.666+0000 7fc259b29640 1 -- 172.21.15.32:0/1201497928 wait complete. 2024-04-25T00:45:31.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.666+0000 7fc259b29640 1 Processor -- start 2024-04-25T00:45:31.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.666+0000 7fc259b29640 1 -- start start 2024-04-25T00:45:31.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.667+0000 7fc259b29640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc254101790 0x7fc254198110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:31.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.667+0000 7fc259b29640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc254102150 0x7fc254198650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:31.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.667+0000 7fc259b29640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc2541077a0 0x7fc25419c9d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:31.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.667+0000 7fc259b29640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fc254111bb0 con 0x7fc254102150 2024-04-25T00:45:31.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.667+0000 7fc259b29640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fc254111a30 con 0x7fc254101790 2024-04-25T00:45:31.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.667+0000 7fc259b29640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fc254111d30 con 0x7fc2541077a0 2024-04-25T00:45:31.669 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.667+0000 7fc252ffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc254102150 0x7fc254198650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:31.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.667+0000 7fc2537fe640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc254101790 0x7fc254198110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:31.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.667+0000 7fc252ffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc254102150 0x7fc254198650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:56758/0 (socket says 172.21.15.32:56758) 2024-04-25T00:45:31.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.667+0000 7fc252ffd640 1 -- 172.21.15.32:0/2582686106 learned_addr learned my addr 172.21.15.32:0/2582686106 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:45:31.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.667+0000 7fc253fff640 1 --2- 172.21.15.32:0/2582686106 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc2541077a0 0x7fc25419c9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:31.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.667+0000 7fc252ffd640 1 -- 172.21.15.32:0/2582686106 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc2541077a0 msgr2=0x7fc25419c9d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:31.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.667+0000 7fc252ffd640 1 --2- 172.21.15.32:0/2582686106 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc2541077a0 0x7fc25419c9d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.667+0000 7fc252ffd640 1 -- 172.21.15.32:0/2582686106 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc254101790 msgr2=0x7fc254198110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:31.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.667+0000 7fc252ffd640 1 --2- 172.21.15.32:0/2582686106 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc254101790 0x7fc254198110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.667+0000 7fc252ffd640 1 -- 172.21.15.32:0/2582686106 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc25419d100 con 0x7fc254102150 2024-04-25T00:45:31.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.667+0000 7fc253fff640 1 --2- 172.21.15.32:0/2582686106 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc2541077a0 0x7fc25419c9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-25T00:45:31.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.667+0000 7fc2537fe640 1 --2- 172.21.15.32:0/2582686106 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc254101790 0x7fc254198110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:45:31.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.667+0000 7fc252ffd640 1 --2- 172.21.15.32:0/2582686106 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc254102150 0x7fc254198650 secure :-1 s=READY pgs=667 cs=0 l=1 rev1=1 crypto rx=0x7fc24800d770 tx=0x7fc24800dc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:31.670 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.667+0000 7fc250ff9640 1 -- 172.21.15.32:0/2582686106 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc248017070 con 0x7fc254102150 2024-04-25T00:45:31.671 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.668+0000 7fc259b29640 1 -- 172.21.15.32:0/2582686106 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc25419d3f0 con 0x7fc254102150 2024-04-25T00:45:31.671 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.668+0000 7fc250ff9640 1 -- 172.21.15.32:0/2582686106 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc248005500 con 0x7fc254102150 2024-04-25T00:45:31.671 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.668+0000 7fc250ff9640 1 -- 172.21.15.32:0/2582686106 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc248004e00 con 0x7fc254102150 2024-04-25T00:45:31.671 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.668+0000 7fc259b29640 1 -- 172.21.15.32:0/2582686106 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc2541a4c80 con 0x7fc254102150 2024-04-25T00:45:31.672 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.669+0000 7fc259b29640 1 -- 172.21.15.32:0/2582686106 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc254068560 con 0x7fc254102150 2024-04-25T00:45:31.672 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.670+0000 7fc250ff9640 1 -- 172.21.15.32:0/2582686106 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7fc248004fa0 con 0x7fc254102150 2024-04-25T00:45:31.672 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.671+0000 7fc250ff9640 1 --2- 172.21.15.32:0/2582686106 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fc23c0a2d00 0x7fc23c0a51c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:31.672 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.671+0000 7fc2537fe640 1 --2- 172.21.15.32:0/2582686106 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fc23c0a2d00 0x7fc23c0a51c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:31.672 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.671+0000 7fc250ff9640 1 -- 172.21.15.32:0/2582686106 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(107..107 src has 1..107) ==== 9554+0+0 (secure 0 0 0) 0x7fc2480cf340 con 0x7fc254102150 2024-04-25T00:45:31.672 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.671+0000 7fc2537fe640 1 --2- 172.21.15.32:0/2582686106 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fc23c0a2d00 0x7fc23c0a51c0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7fc244006010 tx=0x7fc244005ef0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:31.676 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.675+0000 7fc250ff9640 1 -- 172.21.15.32:0/2582686106 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fc24809baf0 con 0x7fc254102150 2024-04-25T00:45:31.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:31 smithi121 ceph-mon[32231]: pgmap v382: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:45:31.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:31 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1728972212' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:45:31.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:31 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/896577190' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:45:31.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:31 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/440433825' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:45:31.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.860+0000 7fc259b29640 1 -- 172.21.15.32:0/2582686106 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"} v 0) -- 0x7fc254109d30 con 0x7fc254102150 2024-04-25T00:45:31.862 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.861+0000 7fc250ff9640 1 -- 172.21.15.32:0/2582686106 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]=0 v107) ==== 103+0+11 (secure 0 0 0) 0x7fc24809b3f0 con 0x7fc254102150 2024-04-25T00:45:31.863 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 32 2024-04-25T00:45:31.863 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.862+0000 7fc259b29640 1 -- 172.21.15.32:0/2582686106 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fc23c0a2d00 msgr2=0x7fc23c0a51c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:31.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.862+0000 7fc259b29640 1 --2- 172.21.15.32:0/2582686106 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fc23c0a2d00 0x7fc23c0a51c0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7fc244006010 tx=0x7fc244005ef0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.862+0000 7fc259b29640 1 -- 172.21.15.32:0/2582686106 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc254102150 msgr2=0x7fc254198650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:31.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.862+0000 7fc259b29640 1 --2- 172.21.15.32:0/2582686106 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc254102150 0x7fc254198650 secure :-1 s=READY pgs=667 cs=0 l=1 rev1=1 crypto rx=0x7fc24800d770 tx=0x7fc24800dc30 comp rx=0 tx=0).stop 2024-04-25T00:45:31.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.862+0000 7fc259b29640 1 -- 172.21.15.32:0/2582686106 shutdown_connections 2024-04-25T00:45:31.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.862+0000 7fc259b29640 1 --2- 172.21.15.32:0/2582686106 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fc23c0a2d00 0x7fc23c0a51c0 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.862+0000 7fc259b29640 1 --2- 172.21.15.32:0/2582686106 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc2541077a0 0x7fc25419c9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.862+0000 7fc259b29640 1 --2- 172.21.15.32:0/2582686106 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fc254102150 0x7fc254198650 unknown :-1 s=CLOSED pgs=667 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.862+0000 7fc259b29640 1 --2- 172.21.15.32:0/2582686106 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fc254101790 0x7fc254198110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.862+0000 7fc259b29640 1 -- 172.21.15.32:0/2582686106 >> 172.21.15.32:0/2582686106 conn(0x7fc2540fd2a0 msgr2=0x7fc2540fdc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:31.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.863+0000 7fc259b29640 1 -- 172.21.15.32:0/2582686106 shutdown_connections 2024-04-25T00:45:31.864 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.863+0000 7fc259b29640 1 -- 172.21.15.32:0/2582686106 wait complete. 2024-04-25T00:45:31.873 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:31 smithi032 ceph-mon[29357]: pgmap v382: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:45:31.873 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:31 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1728972212' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:45:31.873 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:31 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/896577190' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:45:31.873 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:31 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/440433825' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:45:31.874 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-04-25T00:45:31.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:31 smithi130 ceph-mon[32525]: pgmap v382: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s 2024-04-25T00:45:31.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:31 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1728972212' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:45:31.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:31 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/896577190' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T00:45:31.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:31 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/440433825' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-25T00:45:31.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.976+0000 7fe170e4d640 1 -- 172.21.15.32:0/1501815527 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe16c069dc0 msgr2=0x7fe16c06a240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:31.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.976+0000 7fe170e4d640 1 --2- 172.21.15.32:0/1501815527 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe16c069dc0 0x7fe16c06a240 secure :-1 s=READY pgs=668 cs=0 l=1 rev1=1 crypto rx=0x7fe160009930 tx=0x7fe16002a420 comp rx=0 tx=0).stop 2024-04-25T00:45:31.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.976+0000 7fe170e4d640 1 -- 172.21.15.32:0/1501815527 shutdown_connections 2024-04-25T00:45:31.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.976+0000 7fe170e4d640 1 --2- 172.21.15.32:0/1501815527 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe16c06a780 0x7fe16c10e5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.976+0000 7fe170e4d640 1 --2- 172.21.15.32:0/1501815527 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe16c069dc0 0x7fe16c06a240 unknown :-1 s=CLOSED pgs=668 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.976+0000 7fe170e4d640 1 --2- 172.21.15.32:0/1501815527 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe16c107730 0x7fe16c107b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.976+0000 7fe170e4d640 1 -- 172.21.15.32:0/1501815527 >> 172.21.15.32:0/1501815527 conn(0x7fe16c0776a0 msgr2=0x7fe16c077ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:31.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.976+0000 7fe170e4d640 1 -- 172.21.15.32:0/1501815527 shutdown_connections 2024-04-25T00:45:31.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.976+0000 7fe170e4d640 1 -- 172.21.15.32:0/1501815527 wait complete. 2024-04-25T00:45:31.979 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.977+0000 7fe170e4d640 1 Processor -- start 2024-04-25T00:45:31.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.977+0000 7fe170e4d640 1 -- start start 2024-04-25T00:45:31.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.977+0000 7fe170e4d640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe16c069dc0 0x7fe16c198100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:31.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.977+0000 7fe170e4d640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe16c06a780 0x7fe16c198640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:31.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.977+0000 7fe170e4d640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe16c107730 0x7fe16c19c970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:31.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.977+0000 7fe170e4d640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fe16c111b40 con 0x7fe16c069dc0 2024-04-25T00:45:31.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.977+0000 7fe170e4d640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fe16c1119c0 con 0x7fe16c06a780 2024-04-25T00:45:31.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.977+0000 7fe170e4d640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fe16c111cc0 con 0x7fe16c107730 2024-04-25T00:45:31.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.977+0000 7fe16a59c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe16c069dc0 0x7fe16c198100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:31.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.977+0000 7fe16a59c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe16c069dc0 0x7fe16c198100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:50948/0 (socket says 172.21.15.32:50948) 2024-04-25T00:45:31.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.977+0000 7fe16a59c640 1 -- 172.21.15.32:0/3532885447 learned_addr learned my addr 172.21.15.32:0/3532885447 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:45:31.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.978+0000 7fe169d9b640 1 --2- 172.21.15.32:0/3532885447 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe16c06a780 0x7fe16c198640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:31.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.978+0000 7fe16a59c640 1 -- 172.21.15.32:0/3532885447 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe16c107730 msgr2=0x7fe16c19c970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:31.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.978+0000 7fe16a59c640 1 --2- 172.21.15.32:0/3532885447 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe16c107730 0x7fe16c19c970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.980 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.978+0000 7fe16a59c640 1 -- 172.21.15.32:0/3532885447 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe16c06a780 msgr2=0x7fe16c198640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:31.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.978+0000 7fe16a59c640 1 --2- 172.21.15.32:0/3532885447 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe16c06a780 0x7fe16c198640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:31.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.978+0000 7fe16a59c640 1 -- 172.21.15.32:0/3532885447 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe16c19d0a0 con 0x7fe16c069dc0 2024-04-25T00:45:31.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.978+0000 7fe16a59c640 1 --2- 172.21.15.32:0/3532885447 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe16c069dc0 0x7fe16c198100 secure :-1 s=READY pgs=669 cs=0 l=1 rev1=1 crypto rx=0x7fe15c00c890 tx=0x7fe15c00cd50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:31.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.978+0000 7fe15b7fe640 1 -- 172.21.15.32:0/3532885447 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe15c004220 con 0x7fe16c069dc0 2024-04-25T00:45:31.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.978+0000 7fe170e4d640 1 -- 172.21.15.32:0/3532885447 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe16c19d390 con 0x7fe16c069dc0 2024-04-25T00:45:31.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.978+0000 7fe15b7fe640 1 -- 172.21.15.32:0/3532885447 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe15c0043c0 con 0x7fe16c069dc0 2024-04-25T00:45:31.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.978+0000 7fe15b7fe640 1 -- 172.21.15.32:0/3532885447 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe15c004a00 con 0x7fe16c069dc0 2024-04-25T00:45:31.981 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.978+0000 7fe170e4d640 1 -- 172.21.15.32:0/3532885447 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe16c1a4c20 con 0x7fe16c069dc0 2024-04-25T00:45:31.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.979+0000 7fe170e4d640 1 -- 172.21.15.32:0/3532885447 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe16c0689b0 con 0x7fe16c069dc0 2024-04-25T00:45:31.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.980+0000 7fe15b7fe640 1 -- 172.21.15.32:0/3532885447 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7fe15c0079b0 con 0x7fe16c069dc0 2024-04-25T00:45:31.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.981+0000 7fe15b7fe640 1 --2- 172.21.15.32:0/3532885447 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe1540a2d00 0x7fe1540a51c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:31.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.981+0000 7fe169d9b640 1 --2- 172.21.15.32:0/3532885447 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe1540a2d00 0x7fe1540a51c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:31.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.981+0000 7fe15b7fe640 1 -- 172.21.15.32:0/3532885447 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(107..107 src has 1..107) ==== 9554+0+0 (secure 0 0 0) 0x7fe15c0cf9e0 con 0x7fe16c069dc0 2024-04-25T00:45:31.982 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.982+0000 7fe169d9b640 1 --2- 172.21.15.32:0/3532885447 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe1540a2d00 0x7fe1540a51c0 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7fe160000c00 tx=0x7fe160031490 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:31.985 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:31.985+0000 7fe15b7fe640 1 -- 172.21.15.32:0/3532885447 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fe15c014030 con 0x7fe16c069dc0 2024-04-25T00:45:32.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.181+0000 7fe170e4d640 1 -- 172.21.15.32:0/3532885447 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) -- 0x7fe16c02d170 con 0x7fe16c069dc0 2024-04-25T00:45:32.182 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.181+0000 7fe15b7fe640 1 -- 172.21.15.32:0/3532885447 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 16 entries v107) ==== 81+0+942 (secure 0 0 0) 0x7fe15c09c1f0 con 0x7fe16c069dc0 2024-04-25T00:45:32.182 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:6801/585810931 2024-04-26T00:33:17.919894+0000 2024-04-25T00:45:32.182 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:0/1096527590 2024-04-26T00:33:17.919894+0000 2024-04-25T00:45:32.182 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:0/1767429535 2024-04-26T00:33:17.919894+0000 2024-04-25T00:45:32.182 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:0/3915353181 2024-04-26T00:33:17.919894+0000 2024-04-25T00:45:32.182 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:0/2999636731 2024-04-26T00:23:42.776439+0000 2024-04-25T00:45:32.183 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:6801/4160192081 2024-04-26T00:23:42.776439+0000 2024-04-25T00:45:32.183 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:6800/4160192081 2024-04-26T00:23:42.776439+0000 2024-04-25T00:45:32.183 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:0/53550767 2024-04-26T00:23:42.776439+0000 2024-04-25T00:45:32.183 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:0/1066471448 2024-04-26T00:23:14.613921+0000 2024-04-25T00:45:32.183 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:6800/585810931 2024-04-26T00:33:17.919894+0000 2024-04-25T00:45:32.183 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:0/3987019264 2024-04-26T00:23:14.613921+0000 2024-04-25T00:45:32.183 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:0/3041942084 2024-04-26T00:23:14.613921+0000 2024-04-25T00:45:32.183 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:0/1694990206 2024-04-26T00:33:17.919894+0000 2024-04-25T00:45:32.183 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:0/3004184616 2024-04-26T00:23:42.776439+0000 2024-04-25T00:45:32.183 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:6800/1637256323 2024-04-26T00:23:14.613921+0000 2024-04-25T00:45:32.183 INFO:teuthology.orchestra.run.smithi032.stdout:172.21.15.32:6801/1637256323 2024-04-26T00:23:14.613921+0000 2024-04-25T00:45:32.184 INFO:teuthology.orchestra.run.smithi032.stderr:listed 16 entries 2024-04-25T00:45:32.184 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.183+0000 7fe170e4d640 1 -- 172.21.15.32:0/3532885447 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe1540a2d00 msgr2=0x7fe1540a51c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:32.184 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.183+0000 7fe170e4d640 1 --2- 172.21.15.32:0/3532885447 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe1540a2d00 0x7fe1540a51c0 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7fe160000c00 tx=0x7fe160031490 comp rx=0 tx=0).stop 2024-04-25T00:45:32.184 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.183+0000 7fe170e4d640 1 -- 172.21.15.32:0/3532885447 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe16c069dc0 msgr2=0x7fe16c198100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:32.185 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.183+0000 7fe170e4d640 1 --2- 172.21.15.32:0/3532885447 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe16c069dc0 0x7fe16c198100 secure :-1 s=READY pgs=669 cs=0 l=1 rev1=1 crypto rx=0x7fe15c00c890 tx=0x7fe15c00cd50 comp rx=0 tx=0).stop 2024-04-25T00:45:32.185 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.183+0000 7fe170e4d640 1 -- 172.21.15.32:0/3532885447 shutdown_connections 2024-04-25T00:45:32.185 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.183+0000 7fe170e4d640 1 --2- 172.21.15.32:0/3532885447 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe1540a2d00 0x7fe1540a51c0 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:32.185 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.183+0000 7fe170e4d640 1 --2- 172.21.15.32:0/3532885447 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe16c107730 0x7fe16c19c970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:32.185 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.183+0000 7fe170e4d640 1 --2- 172.21.15.32:0/3532885447 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe16c06a780 0x7fe16c198640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:32.185 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.183+0000 7fe170e4d640 1 --2- 172.21.15.32:0/3532885447 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe16c069dc0 0x7fe16c198100 unknown :-1 s=CLOSED pgs=669 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:32.185 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.183+0000 7fe170e4d640 1 -- 172.21.15.32:0/3532885447 >> 172.21.15.32:0/3532885447 conn(0x7fe16c0776a0 msgr2=0x7fe16c0fe690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:32.185 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.183+0000 7fe170e4d640 1 -- 172.21.15.32:0/3532885447 shutdown_connections 2024-04-25T00:45:32.185 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.183+0000 7fe170e4d640 1 -- 172.21.15.32:0/3532885447 wait complete. 2024-04-25T00:45:32.194 DEBUG:tasks.cephfs.kernel_mount:Unmounting client client.0... 2024-04-25T00:45:32.194 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:45:32.195 DEBUG:teuthology.orchestra.run.smithi032:> sudo umount /home/ubuntu/cephtest/mnt.0 2024-04-25T00:45:32.245 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi032.front.sepia.ceph.com 2024-04-25T00:45:32.245 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:45:32.245 DEBUG:teuthology.orchestra.run.smithi032:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.0) 2024-04-25T00:45:32.290 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:45:32.290 DEBUG:teuthology.orchestra.run.smithi032:> ip netns list 2024-04-25T00:45:32.312 INFO:teuthology.orchestra.run.smithi032.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-04-25T00:45:32.312 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:45:32.312 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip netns delete ceph-ns--home-ubuntu-cephtest-mnt.0 2024-04-25T00:45:32.378 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:45:32.378 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip link delete ceph-brx 2024-04-25T00:45:32.464 INFO:tasks.kclient:Unmounting kernel clients... 2024-04-25T00:45:32.465 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-04-25T00:45:32.465 DEBUG:teuthology.orchestra.run.smithi032:> dd if=/proc/self/mounts of=/dev/stdout 2024-04-25T00:45:32.481 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056882,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498932k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=1118 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysusers.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249464k,nr_inodes=812366,mode=700,uid=1000,gid=1316,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/sda1 /var/lib/containers/storage/overlay ext4 rw,seclabel,relatime 0 0 overlay /var/lib/containers/storage/overlay/34155947345d5e1279ddfaff075403f9c08260a5fc5ab9972fd1f762a12420da/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",relatime,lowerdir=/var/lib/containers/storage/overlay/l/E5KY34WLNDXI3CLZK2R2H7HGE3:/var/lib/containers/storage/overlay/l/5P2R4SMJXERBRYFZQRK7QTKFDX,upperdir=/var/lib/containers/storage/overlay/34155947345d5e1279ddfaff075403f9c08260a5fc5ab9972fd1f762a12420da/diff,workdir=/var/lib/containers/storage/overlay/34155947345d5e1279ddfaff075403f9c08260a5fc5ab9972fd1f762a12420da/work,metacopy=on,volatile 0 0 overlay /var/lib/containers/storage/overlay/5ec22dd33e87923e6628917767aab474f46515b93275f46511f7969afe245636/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",nodev,relatime,lowerdir=/var/lib/containers/storage/overlay/l/E5KY34WLNDXI3CLZK2R2H7HGE3:/var/lib/containers/storage/overlay/l/5P2R4SMJXERBRYFZQRK7QTKFDX,upperdir=/var/lib/containers/storage/overlay/5ec22dd33e87923e6628917767aab474f46515b93275f46511f7969afe245636/diff,workdir=/var/lib/containers/storage/overlay/5ec22dd33e87923e6628917767aab474f46515b93275f46511f7969afe245636/work,metacopy=on,volatile 0 0 tmpfs /run/user/0 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249464k,nr_inodes=812366,mode=700,inode64 0 0 binfmt_misc /proc/sys/fs/binfmt_misc binfmt_misc rw,nosuid,nodev,noexec,relatime 0 0 overlay /var/lib/containers/storage/overlay/16272f03717e283e4c165a9667c2d715b6be7134b2abb1f4bb581d16d317c39c/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",relatime,lowerdir=/var/lib/containers/storage/overlay/l/E5KY34WLNDXI3CLZK2R2H7HGE3:/var/lib/containers/storage/overlay/l/5P2R4SMJXERBRYFZQRK7QTKFDX,upperdir=/var/lib/containers/storage/overlay/16272f03717e283e4c165a9667c2d715b6be7134b2abb1f4bb581d16d317c39c/diff,workdir=/var/lib/containers/storage/overlay/16272f03717e283e4c165a9667c2d715b6be7134b2abb1f4bb581d16d317c39c/work,metacopy=on,volatile 0 0 overlay /var/lib/containers/storage/overlay/b334735334bcc68b2451d311c5255685794a3f6606fab96a8e1bcbd4ae59064d/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",relatime,lowerdir=/var/lib/containers/storage/overlay/l/E5KY34WLNDXI3CLZK2R2H7HGE3:/var/lib/containers/storage/overlay/l/5P2R4SMJXERBRYFZQRK7QTKFDX,upperdir=/var/lib/containers/storage/overlay/b334735334bcc68b2451d311c5255685794a3f6606fab96a8e1bcbd4ae59064d/diff,workdir=/var/lib/containers/storage/overlay/b334735334bcc68b2451d311c5255685794a3f6606fab96a8e1bcbd4ae59064d/work,metacopy=on,volatile 0 0 overlay /var/lib/containers/storage/overlay/d0f8d7db666f4cd21acee5182c8f8fb8f6acfbb902e8bf267047ba30de52aa79/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",relatime,lowerdir=/var/lib/containers/storage/overlay/l/E5KY34WLNDXI3CLZK2R2H7HGE3:/var/lib/containers/storage/overlay/l/5P2R4SMJXERBRYFZQRK7QTKFDX,upperdir=/var/lib/containers/storage/overlay/d0f8d7db666f4cd21acee5182c8f8fb8f6acfbb902e8bf267047ba30de52aa79/diff,workdir=/var/lib/containers/storage/overlay/d0f8d7db666f4cd21acee5182c8f8fb8f6acfbb902e8bf267047ba30de52aa79/work,metacopy=on,volatile 0 0 overlay /var/lib/containers/storage/overlay/092d3e3c11700cbab46f6e752060078cc9e7b6f24911398ce163290571048a0d/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",relatime,lowerdir=/var/lib/containers/storage/overlay/l/E5KY34WLNDXI3CLZK2R2H7HGE3:/var/lib/containers/storage/overlay/l/5P2R4SMJXERBRYFZQRK7QTKFDX,upperdir=/var/lib/containers/storage/overlay/092d3e3c11700cbab46f6e752060078cc9e7b6f24911398ce163290571048a0d/diff,workdir=/var/lib/containers/storage/overlay/092d3e3c11700cbab46f6e752060078cc9e7b6f24911398ce163290571048a0d/work,metacopy=on,volatile 0 0 overlay /var/lib/containers/storage/overlay/564269b61a46d927f7753189d846a278374e70b4d55e1db4fec3a1a6dee9032b/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",nodev,relatime,lowerdir=/var/lib/containers/storage/overlay/l/E5KY34WLNDXI3CLZK2R2H7HGE3:/var/lib/containers/storage/overlay/l/5P2R4SMJXERBRYFZQRK7QTKFDX,upperdir=/var/lib/containers/storage/overlay/564269b61a46d927f7753189d846a278374e70b4d55e1db4fec3a1a6dee9032b/diff,workdir=/var/lib/containers/storage/overlay/564269b61a46d927f7753189d846a278374e70b4d55e1db4fec3a1a6dee9032b/work,metacopy=on,volatile 0 0 overlay /var/lib/containers/storage/overlay/3e4c6be55aa7f47d463d2b46969db01be7907349b6bed80e2307ea969e868b71/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",nodev,relatime,lowerdir=/var/lib/containers/storage/overlay/l/E5KY34WLNDXI3CLZK2R2H7HGE3:/var/lib/containers/storage/overlay/l/5P2R4SMJXERBRYFZQRK7QTKFDX,upperdir=/var/lib/containers/storage/overlay/3e4c6be55aa7f47d463d2b46969db01be7907349b6bed80e2307ea969e868b71/diff,workdir=/var/lib/containers/storage/overlay/3e4c6be55aa7f47d463d2b46969db01be7907349b6bed80e2307ea969e868b71/work,metacopy=on,volatile 0 0 overlay /var/lib/containers/storage/overlay/1478e67e3c70fb5971457aac4f57e040e6364153eaa3cd74ed348a42546213f7/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",nodev,relatime,lowerdir=/var/lib/containers/storage/overlay/l/E5KY34WLNDXI3CLZK2R2H7HGE3:/var/lib/containers/storage/overlay/l/5P2R4SMJXERBRYFZQRK7QTKFDX,upperdir=/var/lib/containers/storage/overlay/1478e67e3c70fb5971457aac4f57e040e6364153eaa3cd74ed348a42546213f7/diff,workdir=/var/lib/containers/storage/overlay/1478e67e3c70fb5971457aac4f57e040e6364153eaa3cd74ed348a42546213f7/work,metacopy=on,volatile 0 0 overlay /var/lib/containers/storage/overlay/1b758a1eadfa81ab2a2105f4bf50a80b68784c080304d78e93e1c871d4e450b2/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",nodev,relatime,lowerdir=/var/lib/containers/storage/overlay/l/E5KY34WLNDXI3CLZK2R2H7HGE3:/var/lib/containers/storage/overlay/l/5P2R4SMJXERBRYFZQRK7QTKFDX,upperdir=/var/lib/containers/storage/overlay/1b758a1eadfa81ab2a2105f4bf50a80b68784c080304d78e93e1c871d4e450b2/diff,workdir=/var/lib/containers/storage/overlay/1b758a1eadfa81ab2a2105f4bf50a80b68784c080304d78e93e1c871d4e450b2/work,metacopy=on,volatile 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498932k,nr_inodes=819200,mode=755,inode64 0 0 2024-04-25T00:45:32.481 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:45:32.481 DEBUG:teuthology.orchestra.run.smithi032:> ip netns list 2024-04-25T00:45:32.537 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-25T00:45:32.537 DEBUG:teuthology.orchestra.run.smithi032:> sudo ip link delete ceph-brx 2024-04-25T00:45:32.649 INFO:teuthology.orchestra.run.smithi032.stderr:Cannot find device "ceph-brx" 2024-04-25T00:45:32.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-25T00:45:32.651 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-25T00:45:32.663 INFO:tasks.cephadm:Teardown begin 2024-04-25T00:45:32.663 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-04-25T00:45:32.663 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:45:32.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:32 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1138571269' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:45:32.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:32 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2582686106' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:45:32.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:32 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3532885447' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-25T00:45:32.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.766+0000 7f1ad54ba640 1 -- 172.21.15.32:0/3384160515 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1ad01017c0 msgr2=0x7f1ad0101c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:32.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.766+0000 7f1ad54ba640 1 --2- 172.21.15.32:0/3384160515 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1ad01017c0 0x7f1ad0101c20 secure :-1 s=READY pgs=670 cs=0 l=1 rev1=1 crypto rx=0x7f1ab8009900 tx=0x7f1ab802a480 comp rx=0 tx=0).stop 2024-04-25T00:45:32.769 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.766+0000 7f1ad54ba640 1 -- 172.21.15.32:0/3384160515 shutdown_connections 2024-04-25T00:45:32.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.766+0000 7f1ad54ba640 1 --2- 172.21.15.32:0/3384160515 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1ad0102160 0x7f1ad010e5a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:32.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.766+0000 7f1ad54ba640 1 --2- 172.21.15.32:0/3384160515 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1ad01017c0 0x7f1ad0101c20 unknown :-1 s=CLOSED pgs=670 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:32.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.766+0000 7f1ad54ba640 1 --2- 172.21.15.32:0/3384160515 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1ad0107750 0x7f1ad0107b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:32.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.767+0000 7f1ad54ba640 1 -- 172.21.15.32:0/3384160515 >> 172.21.15.32:0/3384160515 conn(0x7f1ad00788d0 msgr2=0x7f1ad0078ce0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:32.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.767+0000 7f1ad54ba640 1 -- 172.21.15.32:0/3384160515 shutdown_connections 2024-04-25T00:45:32.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.767+0000 7f1ad54ba640 1 -- 172.21.15.32:0/3384160515 wait complete. 2024-04-25T00:45:32.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.767+0000 7f1ad54ba640 1 Processor -- start 2024-04-25T00:45:32.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.767+0000 7f1ad54ba640 1 -- start start 2024-04-25T00:45:32.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.767+0000 7f1ad54ba640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1ad01017c0 0x7f1ad0198010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:32.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.767+0000 7f1ad54ba640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1ad0102160 0x7f1ad0198550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:32.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.767+0000 7f1ad54ba640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1ad0107750 0x7f1ad019c8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:32.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.767+0000 7f1ad54ba640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f1ad0111b10 con 0x7f1ad01017c0 2024-04-25T00:45:32.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.767+0000 7f1ad54ba640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f1ad0111990 con 0x7f1ad0107750 2024-04-25T00:45:32.770 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.767+0000 7f1ad54ba640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f1ad0111c90 con 0x7f1ad0102160 2024-04-25T00:45:32.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.767+0000 7f1aceffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1ad01017c0 0x7f1ad0198010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:32.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.768+0000 7f1aceffd640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1ad01017c0 0x7f1ad0198010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:50976/0 (socket says 172.21.15.32:50976) 2024-04-25T00:45:32.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.768+0000 7f1aceffd640 1 -- 172.21.15.32:0/2210767016 learned_addr learned my addr 172.21.15.32:0/2210767016 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:45:32.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.768+0000 7f1acf7fe640 1 --2- 172.21.15.32:0/2210767016 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1ad0107750 0x7f1ad019c8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:32.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.768+0000 7f1ac7fff640 1 --2- 172.21.15.32:0/2210767016 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1ad0102160 0x7f1ad0198550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:32.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.768+0000 7f1aceffd640 1 -- 172.21.15.32:0/2210767016 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1ad0102160 msgr2=0x7f1ad0198550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:32.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.768+0000 7f1aceffd640 1 --2- 172.21.15.32:0/2210767016 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1ad0102160 0x7f1ad0198550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:32.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.768+0000 7f1aceffd640 1 -- 172.21.15.32:0/2210767016 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1ad0107750 msgr2=0x7f1ad019c8d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:32.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.768+0000 7f1aceffd640 1 --2- 172.21.15.32:0/2210767016 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1ad0107750 0x7f1ad019c8d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:32.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.768+0000 7f1aceffd640 1 -- 172.21.15.32:0/2210767016 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1ad019d000 con 0x7f1ad01017c0 2024-04-25T00:45:32.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.768+0000 7f1aceffd640 1 --2- 172.21.15.32:0/2210767016 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1ad01017c0 0x7f1ad0198010 secure :-1 s=READY pgs=671 cs=0 l=1 rev1=1 crypto rx=0x7f1ac000c920 tx=0x7f1ac000cde0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:32.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.768+0000 7f1accff9640 1 -- 172.21.15.32:0/2210767016 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1ac0007c70 con 0x7f1ad01017c0 2024-04-25T00:45:32.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.768+0000 7f1accff9640 1 -- 172.21.15.32:0/2210767016 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1ac0007e10 con 0x7f1ad01017c0 2024-04-25T00:45:32.771 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.768+0000 7f1ad54ba640 1 -- 172.21.15.32:0/2210767016 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1ad019d2f0 con 0x7f1ad01017c0 2024-04-25T00:45:32.772 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.768+0000 7f1accff9640 1 -- 172.21.15.32:0/2210767016 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1ac0018890 con 0x7f1ad01017c0 2024-04-25T00:45:32.772 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.768+0000 7f1ad54ba640 1 -- 172.21.15.32:0/2210767016 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1ad01a4b80 con 0x7f1ad01017c0 2024-04-25T00:45:32.773 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.769+0000 7f1ad54ba640 1 -- 172.21.15.32:0/2210767016 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1ad00689b0 con 0x7f1ad01017c0 2024-04-25T00:45:32.773 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.770+0000 7f1accff9640 1 -- 172.21.15.32:0/2210767016 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f1ac0018a70 con 0x7f1ad01017c0 2024-04-25T00:45:32.773 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.771+0000 7f1accff9640 1 --2- 172.21.15.32:0/2210767016 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f1abc0a2d00 0x7f1abc0a51c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:32.773 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.771+0000 7f1ac7fff640 1 --2- 172.21.15.32:0/2210767016 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f1abc0a2d00 0x7f1abc0a51c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:32.773 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.771+0000 7f1accff9640 1 -- 172.21.15.32:0/2210767016 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(107..107 src has 1..107) ==== 9554+0+0 (secure 0 0 0) 0x7f1ac00cf8e0 con 0x7f1ad01017c0 2024-04-25T00:45:32.773 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.772+0000 7f1ac7fff640 1 --2- 172.21.15.32:0/2210767016 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f1abc0a2d00 0x7f1abc0a51c0 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f1ab8009230 tx=0x7f1ab80058d0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:32.776 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:32.776+0000 7f1accff9640 1 -- 172.21.15.32:0/2210767016 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f1ac0014030 con 0x7f1ad01017c0 2024-04-25T00:45:32.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:32 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1138571269' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:45:32.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:32 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2582686106' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:45:32.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:32 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3532885447' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-25T00:45:32.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:32 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1138571269' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-25T00:45:32.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:32 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2582686106' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2024-04-25T00:45:32.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:32 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3532885447' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-25T00:45:33.000 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.000+0000 7f1ad54ba640 1 -- 172.21.15.32:0/2210767016 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f1ad0109c50 con 0x7f1ad01017c0 2024-04-25T00:45:33.001 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.000+0000 7f1accff9640 1 -- 172.21.15.32:0/2210767016 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 32 v32) ==== 94+0+11879 (secure 0 0 0) 0x7f1ac009c090 con 0x7f1ad01017c0 2024-04-25T00:45:33.001 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:45:33.001 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":32,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14643,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6839/187492674","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6838","nonce":187492674},{"type":"v1","addr":"172.21.15.32:6839","nonce":187492674}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14655,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6841/1948785545","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6840","nonce":1948785545},{"type":"v1","addr":"172.21.15.32:6841","nonce":1948785545}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":32,"flags":306,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":true},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-25T00:30:47.866875+0000","modified":"2024-04-25T00:44:43.367798+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24506,"mds_1":24481,"mds_2":24476},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24443":{"gid":24443,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24445":{"gid":24445,"name":"f","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24457":{"gid":24457,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24476":{"gid":24476,"name":"h","rank":2,"incarnation":22,"state":"up:active","state_seq":12,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[0],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24481":{"gid":24481,"name":"i","rank":1,"incarnation":20,"state":"up:active","state_seq":108,"addr":"172.21.15.130:6839/4267125809","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":4267125809},{"type":"v1","addr":"172.21.15.130:6839","nonce":4267125809}]},"join_fscid":-1,"export_targets":[0],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24506":{"gid":24506,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.121:6839/258444853","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6838","nonce":258444853},{"type":"v1","addr":"172.21.15.121:6839","nonce":258444853}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24506,"qdb_cluster":[24481,24506,24476]},"id":1}]} 2024-04-25T00:45:33.001 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 32 2024-04-25T00:45:33.003 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.002+0000 7f1ad54ba640 1 -- 172.21.15.32:0/2210767016 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f1abc0a2d00 msgr2=0x7f1abc0a51c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:33.003 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.002+0000 7f1ad54ba640 1 --2- 172.21.15.32:0/2210767016 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f1abc0a2d00 0x7f1abc0a51c0 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f1ab8009230 tx=0x7f1ab80058d0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.003 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.002+0000 7f1ad54ba640 1 -- 172.21.15.32:0/2210767016 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1ad01017c0 msgr2=0x7f1ad0198010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:33.003 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.002+0000 7f1ad54ba640 1 --2- 172.21.15.32:0/2210767016 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1ad01017c0 0x7f1ad0198010 secure :-1 s=READY pgs=671 cs=0 l=1 rev1=1 crypto rx=0x7f1ac000c920 tx=0x7f1ac000cde0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.003 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.002+0000 7f1ad54ba640 1 -- 172.21.15.32:0/2210767016 shutdown_connections 2024-04-25T00:45:33.003 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.002+0000 7f1ad54ba640 1 --2- 172.21.15.32:0/2210767016 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f1abc0a2d00 0x7f1abc0a51c0 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.002+0000 7f1ad54ba640 1 --2- 172.21.15.32:0/2210767016 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1ad0107750 0x7f1ad019c8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.002+0000 7f1ad54ba640 1 --2- 172.21.15.32:0/2210767016 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1ad0102160 0x7f1ad0198550 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.002+0000 7f1ad54ba640 1 --2- 172.21.15.32:0/2210767016 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1ad01017c0 0x7f1ad0198010 unknown :-1 s=CLOSED pgs=671 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.002+0000 7f1ad54ba640 1 -- 172.21.15.32:0/2210767016 >> 172.21.15.32:0/2210767016 conn(0x7f1ad00788d0 msgr2=0x7f1ad01003e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:33.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.003+0000 7f1ad54ba640 1 -- 172.21.15.32:0/2210767016 shutdown_connections 2024-04-25T00:45:33.004 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.003+0000 7f1ad54ba640 1 -- 172.21.15.32:0/2210767016 wait complete. 2024-04-25T00:45:33.014 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-25T00:45:33.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.117+0000 7f16a9a38640 1 -- 172.21.15.32:0/75080907 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f16a4100690 msgr2=0x7f16a4100b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:33.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.117+0000 7f16a9a38640 1 --2- 172.21.15.32:0/75080907 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f16a4100690 0x7f16a4100b10 secure :-1 s=READY pgs=672 cs=0 l=1 rev1=1 crypto rx=0x7f1698009930 tx=0x7f169802a420 comp rx=0 tx=0).stop 2024-04-25T00:45:33.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.117+0000 7f16a9a38640 1 -- 172.21.15.32:0/75080907 shutdown_connections 2024-04-25T00:45:33.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.117+0000 7f16a9a38640 1 --2- 172.21.15.32:0/75080907 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16a4101050 0x7f16a410d510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.117+0000 7f16a9a38640 1 --2- 172.21.15.32:0/75080907 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f16a4100690 0x7f16a4100b10 unknown :-1 s=CLOSED pgs=672 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.117+0000 7f16a9a38640 1 --2- 172.21.15.32:0/75080907 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f16a4106650 0x7f16a4106a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.117+0000 7f16a9a38640 1 -- 172.21.15.32:0/75080907 >> 172.21.15.32:0/75080907 conn(0x7f16a40777a0 msgr2=0x7f16a4077bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:33.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.117+0000 7f16a9a38640 1 -- 172.21.15.32:0/75080907 shutdown_connections 2024-04-25T00:45:33.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.117+0000 7f16a9a38640 1 -- 172.21.15.32:0/75080907 wait complete. 2024-04-25T00:45:33.120 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.117+0000 7f16a9a38640 1 Processor -- start 2024-04-25T00:45:33.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.118+0000 7f16a9a38640 1 -- start start 2024-04-25T00:45:33.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.118+0000 7f16a9a38640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16a4100690 0x7f16a41980f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:33.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.118+0000 7f16a9a38640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f16a4101050 0x7f16a4198630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:33.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.118+0000 7f16a9a38640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f16a4106650 0x7f16a419c960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:33.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.118+0000 7f16a9a38640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f16a4110a60 con 0x7f16a4106650 2024-04-25T00:45:33.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.118+0000 7f16a9a38640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f16a41108e0 con 0x7f16a4101050 2024-04-25T00:45:33.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.118+0000 7f16a9a38640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f16a4110be0 con 0x7f16a4100690 2024-04-25T00:45:33.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.118+0000 7f16a37fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f16a4106650 0x7f16a419c960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:33.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.118+0000 7f16a37fe640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f16a4106650 0x7f16a419c960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:50998/0 (socket says 172.21.15.32:50998) 2024-04-25T00:45:33.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.118+0000 7f16a37fe640 1 -- 172.21.15.32:0/2510032596 learned_addr learned my addr 172.21.15.32:0/2510032596 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:45:33.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.118+0000 7f16a2ffd640 1 --2- 172.21.15.32:0/2510032596 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16a4100690 0x7f16a41980f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:33.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.118+0000 7f16a27fc640 1 --2- 172.21.15.32:0/2510032596 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f16a4101050 0x7f16a4198630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:33.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.119+0000 7f16a2ffd640 1 -- 172.21.15.32:0/2510032596 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f16a4101050 msgr2=0x7f16a4198630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:33.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.119+0000 7f16a2ffd640 1 --2- 172.21.15.32:0/2510032596 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f16a4101050 0x7f16a4198630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.121 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.119+0000 7f16a2ffd640 1 -- 172.21.15.32:0/2510032596 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f16a4106650 msgr2=0x7f16a419c960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:33.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.119+0000 7f16a2ffd640 1 --2- 172.21.15.32:0/2510032596 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f16a4106650 0x7f16a419c960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.119+0000 7f16a2ffd640 1 -- 172.21.15.32:0/2510032596 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f16a419d090 con 0x7f16a4100690 2024-04-25T00:45:33.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.119+0000 7f16a37fe640 1 --2- 172.21.15.32:0/2510032596 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f16a4106650 0x7f16a419c960 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-25T00:45:33.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.119+0000 7f16a2ffd640 1 --2- 172.21.15.32:0/2510032596 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16a4100690 0x7f16a41980f0 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f169400e9f0 tx=0x7f169400eeb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:33.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.119+0000 7f16a8a36640 1 -- 172.21.15.32:0/2510032596 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f169400ccc0 con 0x7f16a4100690 2024-04-25T00:45:33.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.119+0000 7f16a9a38640 1 -- 172.21.15.32:0/2510032596 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f16a419d380 con 0x7f16a4100690 2024-04-25T00:45:33.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.119+0000 7f16a8a36640 1 -- 172.21.15.32:0/2510032596 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1694010910 con 0x7f16a4100690 2024-04-25T00:45:33.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.119+0000 7f16a8a36640 1 -- 172.21.15.32:0/2510032596 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1694018960 con 0x7f16a4100690 2024-04-25T00:45:33.122 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.119+0000 7f16a9a38640 1 -- 172.21.15.32:0/2510032596 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f16a41a4c10 con 0x7f16a4100690 2024-04-25T00:45:33.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.120+0000 7f16a9a38640 1 -- 172.21.15.32:0/2510032596 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f16a40689b0 con 0x7f16a4100690 2024-04-25T00:45:33.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.122+0000 7f16a8a36640 1 -- 172.21.15.32:0/2510032596 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f1694010ac0 con 0x7f16a4100690 2024-04-25T00:45:33.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.122+0000 7f16a8a36640 1 --2- 172.21.15.32:0/2510032596 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f168c0a2d00 0x7f168c0a51c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:33.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.122+0000 7f16a27fc640 1 --2- 172.21.15.32:0/2510032596 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f168c0a2d00 0x7f168c0a51c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:33.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.122+0000 7f16a8a36640 1 -- 172.21.15.32:0/2510032596 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(107..107 src has 1..107) ==== 9554+0+0 (secure 0 0 0) 0x7f16940d00c0 con 0x7f16a4100690 2024-04-25T00:45:33.123 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.123+0000 7f16a27fc640 1 --2- 172.21.15.32:0/2510032596 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f168c0a2d00 0x7f168c0a51c0 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f1698009900 tx=0x7f16980058d0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:33.126 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.126+0000 7f16a8a36640 1 -- 172.21.15.32:0/2510032596 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f169409c220 con 0x7f16a4100690 2024-04-25T00:45:33.350 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.350+0000 7f16a9a38640 1 -- 172.21.15.32:0/2510032596 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f16a41a4de0 con 0x7f16a4100690 2024-04-25T00:45:33.351 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.351+0000 7f16a8a36640 1 -- 172.21.15.32:0/2510032596 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 32 v32) ==== 94+0+11879 (secure 0 0 0) 0x7f16940a0520 con 0x7f16a4100690 2024-04-25T00:45:33.351 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:45:33.351 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":32,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14559,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6835/3511697670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6834","nonce":3511697670},{"type":"v1","addr":"172.21.15.32:6835","nonce":3511697670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14571,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6837/2212842316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6836","nonce":2212842316},{"type":"v1","addr":"172.21.15.32:6837","nonce":2212842316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14643,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6839/187492674","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6838","nonce":187492674},{"type":"v1","addr":"172.21.15.32:6839","nonce":187492674}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14655,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.32:6841/1948785545","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6840","nonce":1948785545},{"type":"v1","addr":"172.21.15.32:6841","nonce":1948785545}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24412,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/3000660373","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3000660373},{"type":"v1","addr":"172.21.15.130:6833","nonce":3000660373}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.121:6833/436688740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":436688740},{"type":"v1","addr":"172.21.15.121:6833","nonce":436688740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":32,"flags":306,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":true},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-25T00:30:47.866875+0000","modified":"2024-04-25T00:44:43.367798+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24506,"mds_1":24481,"mds_2":24476},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24443":{"gid":24443,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.121:6835/271690252","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6834","nonce":271690252},{"type":"v1","addr":"172.21.15.121:6835","nonce":271690252}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24445":{"gid":24445,"name":"f","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6835/2892666533","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2892666533},{"type":"v1","addr":"172.21.15.130:6835","nonce":2892666533}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24457":{"gid":24457,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6837/1920876316","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":1920876316},{"type":"v1","addr":"172.21.15.130:6837","nonce":1920876316}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24476":{"gid":24476,"name":"h","rank":2,"incarnation":22,"state":"up:active","state_seq":12,"addr":"172.21.15.121:6837/3456022860","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6836","nonce":3456022860},{"type":"v1","addr":"172.21.15.121:6837","nonce":3456022860}]},"join_fscid":-1,"export_targets":[0],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24481":{"gid":24481,"name":"i","rank":1,"incarnation":20,"state":"up:active","state_seq":108,"addr":"172.21.15.130:6839/4267125809","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":4267125809},{"type":"v1","addr":"172.21.15.130:6839","nonce":4267125809}]},"join_fscid":-1,"export_targets":[0],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24506":{"gid":24506,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.121:6839/258444853","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6838","nonce":258444853},{"type":"v1","addr":"172.21.15.121:6839","nonce":258444853}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24506,"qdb_cluster":[24481,24506,24476]},"id":1}]} 2024-04-25T00:45:33.352 INFO:teuthology.orchestra.run.smithi032.stderr:dumped fsmap epoch 32 2024-04-25T00:45:33.353 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.353+0000 7f16a9a38640 1 -- 172.21.15.32:0/2510032596 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f168c0a2d00 msgr2=0x7f168c0a51c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:33.353 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.353+0000 7f16a9a38640 1 --2- 172.21.15.32:0/2510032596 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f168c0a2d00 0x7f168c0a51c0 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f1698009900 tx=0x7f16980058d0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.353 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.353+0000 7f16a9a38640 1 -- 172.21.15.32:0/2510032596 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16a4100690 msgr2=0x7f16a41980f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:33.353 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.353+0000 7f16a9a38640 1 --2- 172.21.15.32:0/2510032596 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16a4100690 0x7f16a41980f0 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f169400e9f0 tx=0x7f169400eeb0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.353+0000 7f16a9a38640 1 -- 172.21.15.32:0/2510032596 shutdown_connections 2024-04-25T00:45:33.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.353+0000 7f16a9a38640 1 --2- 172.21.15.32:0/2510032596 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f168c0a2d00 0x7f168c0a51c0 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.353+0000 7f16a9a38640 1 --2- 172.21.15.32:0/2510032596 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f16a4106650 0x7f16a419c960 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.353+0000 7f16a9a38640 1 --2- 172.21.15.32:0/2510032596 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f16a4101050 0x7f16a4198630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.353+0000 7f16a9a38640 1 --2- 172.21.15.32:0/2510032596 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16a4100690 0x7f16a41980f0 unknown :-1 s=CLOSED pgs=138 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.353+0000 7f16a9a38640 1 -- 172.21.15.32:0/2510032596 >> 172.21.15.32:0/2510032596 conn(0x7f16a40777a0 msgr2=0x7f16a40ff970 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:33.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.353+0000 7f16a9a38640 1 -- 172.21.15.32:0/2510032596 shutdown_connections 2024-04-25T00:45:33.354 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.353+0000 7f16a9a38640 1 -- 172.21.15.32:0/2510032596 wait complete. 2024-04-25T00:45:33.364 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-25T00:45:33.470 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.467+0000 7fe0dffc8640 1 -- 172.21.15.32:0/3887544213 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe0d8101050 msgr2=0x7fe0d810d510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:33.470 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.467+0000 7fe0dffc8640 1 --2- 172.21.15.32:0/3887544213 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe0d8101050 0x7fe0d810d510 secure :-1 s=READY pgs=673 cs=0 l=1 rev1=1 crypto rx=0x7fe0d0009cd0 tx=0x7fe0d002a8e0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.470 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.467+0000 7fe0dffc8640 1 -- 172.21.15.32:0/3887544213 shutdown_connections 2024-04-25T00:45:33.470 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.467+0000 7fe0dffc8640 1 --2- 172.21.15.32:0/3887544213 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe0d8101050 0x7fe0d810d510 unknown :-1 s=CLOSED pgs=673 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.470 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.467+0000 7fe0dffc8640 1 --2- 172.21.15.32:0/3887544213 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe0d8100690 0x7fe0d8100b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.470 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.467+0000 7fe0dffc8640 1 --2- 172.21.15.32:0/3887544213 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe0d8106650 0x7fe0d8106a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.470 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.467+0000 7fe0dffc8640 1 -- 172.21.15.32:0/3887544213 >> 172.21.15.32:0/3887544213 conn(0x7fe0d80777a0 msgr2=0x7fe0d8077bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:33.470 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.467+0000 7fe0dffc8640 1 -- 172.21.15.32:0/3887544213 shutdown_connections 2024-04-25T00:45:33.470 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.467+0000 7fe0dffc8640 1 -- 172.21.15.32:0/3887544213 wait complete. 2024-04-25T00:45:33.470 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.467+0000 7fe0dffc8640 1 Processor -- start 2024-04-25T00:45:33.471 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.467+0000 7fe0dffc8640 1 -- start start 2024-04-25T00:45:33.471 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.468+0000 7fe0dffc8640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe0d8100690 0x7fe0d8198010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:33.471 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.468+0000 7fe0dffc8640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe0d8101050 0x7fe0d8198550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:33.471 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.468+0000 7fe0dffc8640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe0d8106650 0x7fe0d819c8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:33.471 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.468+0000 7fe0dffc8640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7fe0d8110ab0 con 0x7fe0d8100690 2024-04-25T00:45:33.471 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.468+0000 7fe0dffc8640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7fe0d8110930 con 0x7fe0d8106650 2024-04-25T00:45:33.471 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.468+0000 7fe0dffc8640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7fe0d8110c30 con 0x7fe0d8101050 2024-04-25T00:45:33.471 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.468+0000 7fe0ddd64640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe0d8100690 0x7fe0d8198010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:33.471 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.468+0000 7fe0ddd64640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe0d8100690 0x7fe0d8198010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:51018/0 (socket says 172.21.15.32:51018) 2024-04-25T00:45:33.471 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.468+0000 7fe0ddd64640 1 -- 172.21.15.32:0/3363945738 learned_addr learned my addr 172.21.15.32:0/3363945738 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:45:33.471 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.468+0000 7fe0de565640 1 --2- 172.21.15.32:0/3363945738 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe0d8106650 0x7fe0d819c8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:33.471 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.468+0000 7fe0dd563640 1 --2- 172.21.15.32:0/3363945738 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe0d8101050 0x7fe0d8198550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:33.471 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.468+0000 7fe0ddd64640 1 -- 172.21.15.32:0/3363945738 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe0d8101050 msgr2=0x7fe0d8198550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:33.471 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.468+0000 7fe0ddd64640 1 --2- 172.21.15.32:0/3363945738 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe0d8101050 0x7fe0d8198550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.472 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.468+0000 7fe0ddd64640 1 -- 172.21.15.32:0/3363945738 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe0d8106650 msgr2=0x7fe0d819c8d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:33.472 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.468+0000 7fe0ddd64640 1 --2- 172.21.15.32:0/3363945738 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe0d8106650 0x7fe0d819c8d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.472 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.468+0000 7fe0ddd64640 1 -- 172.21.15.32:0/3363945738 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe0d819d000 con 0x7fe0d8100690 2024-04-25T00:45:33.472 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.468+0000 7fe0ddd64640 1 --2- 172.21.15.32:0/3363945738 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe0d8100690 0x7fe0d8198010 secure :-1 s=READY pgs=674 cs=0 l=1 rev1=1 crypto rx=0x7fe0d400c970 tx=0x7fe0d400ce30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:33.472 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.468+0000 7fe0ceffd640 1 -- 172.21.15.32:0/3363945738 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe0d4007bb0 con 0x7fe0d8100690 2024-04-25T00:45:33.472 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.468+0000 7fe0ceffd640 1 -- 172.21.15.32:0/3363945738 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe0d4007d50 con 0x7fe0d8100690 2024-04-25T00:45:33.472 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.468+0000 7fe0dffc8640 1 -- 172.21.15.32:0/3363945738 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe0d819d2f0 con 0x7fe0d8100690 2024-04-25T00:45:33.472 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.468+0000 7fe0ceffd640 1 -- 172.21.15.32:0/3363945738 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe0d400f450 con 0x7fe0d8100690 2024-04-25T00:45:33.472 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.468+0000 7fe0dffc8640 1 -- 172.21.15.32:0/3363945738 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe0d81a4b80 con 0x7fe0d8100690 2024-04-25T00:45:33.472 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.469+0000 7fe0dffc8640 1 -- 172.21.15.32:0/3363945738 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe0d80689b0 con 0x7fe0d8100690 2024-04-25T00:45:33.473 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.470+0000 7fe0ceffd640 1 -- 172.21.15.32:0/3363945738 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7fe0d4005020 con 0x7fe0d8100690 2024-04-25T00:45:33.473 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.471+0000 7fe0ceffd640 1 --2- 172.21.15.32:0/3363945738 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe0c40a2d00 0x7fe0c40a51c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:33.473 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.471+0000 7fe0dd563640 1 --2- 172.21.15.32:0/3363945738 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe0c40a2d00 0x7fe0c40a51c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:33.473 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.471+0000 7fe0ceffd640 1 -- 172.21.15.32:0/3363945738 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(107..107 src has 1..107) ==== 9554+0+0 (secure 0 0 0) 0x7fe0d40cf6a0 con 0x7fe0d8100690 2024-04-25T00:45:33.473 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.472+0000 7fe0dd563640 1 --2- 172.21.15.32:0/3363945738 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe0c40a2d00 0x7fe0c40a51c0 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7fe0d8077df0 tx=0x7fe0c800a4c0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:33.476 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.475+0000 7fe0ceffd640 1 -- 172.21.15.32:0/3363945738 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7fe0d4014030 con 0x7fe0d8100690 2024-04-25T00:45:33.662 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.662+0000 7fe0dffc8640 1 -- 172.21.15.32:0/3363945738 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fe0d8199360 con 0x7fe0d8100690 2024-04-25T00:45:33.664 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.663+0000 7fe0ceffd640 1 -- 172.21.15.32:0/3363945738 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v107) ==== 74+0+25405 (secure 0 0 0) 0x7fe0d409beb0 con 0x7fe0d8100690 2024-04-25T00:45:33.664 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-25T00:45:33.664 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":107,"fsid":"cb997156-0299-11ef-bc93-c7b262605968","created":"2024-04-25T00:22:46.105398+0000","modified":"2024-04-25T00:45:17.009986+0000","last_up_change":"2024-04-25T00:29:57.747517+0000","last_in_change":"2024-04-25T00:29:37.009846+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T00:26:07.306839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-25T00:30:46.239556+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-25T00:30:47.257182+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"106","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":19,"snap_epoch":106,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2024-04-25T00:30:51.193975+0000","flags":8197,"flags_names":"hashpspool,ec_overwrites,selfmanaged_snaps","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"106","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":19,"snap_epoch":106,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"074681d6-1be7-49a1-9518-9968ad62dc08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6803","nonce":3984695193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6805","nonce":3984695193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6809","nonce":3984695193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":3984695193},{"type":"v1","addr":"172.21.15.32:6807","nonce":3984695193}]},"public_addr":"172.21.15.32:6803/3984695193","cluster_addr":"172.21.15.32:6805/3984695193","heartbeat_back_addr":"172.21.15.32:6809/3984695193","heartbeat_front_addr":"172.21.15.32:6807/3984695193","state":["exists","up"]},{"osd":1,"uuid":"e7ea9910-062b-4801-b1a5-a90a56d73b66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6801","nonce":431442351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6803","nonce":431442351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6807","nonce":431442351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":431442351},{"type":"v1","addr":"172.21.15.121:6805","nonce":431442351}]},"public_addr":"172.21.15.121:6801/431442351","cluster_addr":"172.21.15.121:6803/431442351","heartbeat_back_addr":"172.21.15.121:6807/431442351","heartbeat_front_addr":"172.21.15.121:6805/431442351","state":["exists","up"]},{"osd":2,"uuid":"8b49a8b4-65a2-4b96-8ab0-350f338cd6f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6801","nonce":1393844110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6803","nonce":1393844110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6807","nonce":1393844110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1393844110},{"type":"v1","addr":"172.21.15.130:6805","nonce":1393844110}]},"public_addr":"172.21.15.130:6801/1393844110","cluster_addr":"172.21.15.130:6803/1393844110","heartbeat_back_addr":"172.21.15.130:6807/1393844110","heartbeat_front_addr":"172.21.15.130:6805/1393844110","state":["exists","up"]},{"osd":3,"uuid":"3956963a-c4a2-41ee-a138-3c5acef75dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6811","nonce":1510492708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6813","nonce":1510492708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6817","nonce":1510492708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1510492708},{"type":"v1","addr":"172.21.15.32:6815","nonce":1510492708}]},"public_addr":"172.21.15.32:6811/1510492708","cluster_addr":"172.21.15.32:6813/1510492708","heartbeat_back_addr":"172.21.15.32:6817/1510492708","heartbeat_front_addr":"172.21.15.32:6815/1510492708","state":["exists","up"]},{"osd":4,"uuid":"5dede06a-2cc8-494b-a0ec-13551677b878","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6809","nonce":1286757826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6811","nonce":1286757826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6815","nonce":1286757826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1286757826},{"type":"v1","addr":"172.21.15.121:6813","nonce":1286757826}]},"public_addr":"172.21.15.121:6809/1286757826","cluster_addr":"172.21.15.121:6811/1286757826","heartbeat_back_addr":"172.21.15.121:6815/1286757826","heartbeat_front_addr":"172.21.15.121:6813/1286757826","state":["exists","up"]},{"osd":5,"uuid":"3360c304-a7ae-4efc-a931-3b6261bbf24e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6809","nonce":909656116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6811","nonce":909656116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6815","nonce":909656116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":909656116},{"type":"v1","addr":"172.21.15.130:6813","nonce":909656116}]},"public_addr":"172.21.15.130:6809/909656116","cluster_addr":"172.21.15.130:6811/909656116","heartbeat_back_addr":"172.21.15.130:6815/909656116","heartbeat_front_addr":"172.21.15.130:6813/909656116","state":["exists","up"]},{"osd":6,"uuid":"f60af4f3-4dd1-4021-a778-d0d97e9a674e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6819","nonce":3198957027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6821","nonce":3198957027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6825","nonce":3198957027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3198957027},{"type":"v1","addr":"172.21.15.32:6823","nonce":3198957027}]},"public_addr":"172.21.15.32:6819/3198957027","cluster_addr":"172.21.15.32:6821/3198957027","heartbeat_back_addr":"172.21.15.32:6825/3198957027","heartbeat_front_addr":"172.21.15.32:6823/3198957027","state":["exists","up"]},{"osd":7,"uuid":"7181e550-d24b-40ee-8d52-9b650a041739","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6817","nonce":3677614004}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6819","nonce":3677614004}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6823","nonce":3677614004}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":3677614004},{"type":"v1","addr":"172.21.15.121:6821","nonce":3677614004}]},"public_addr":"172.21.15.121:6817/3677614004","cluster_addr":"172.21.15.121:6819/3677614004","heartbeat_back_addr":"172.21.15.121:6823/3677614004","heartbeat_front_addr":"172.21.15.121:6821/3677614004","state":["exists","up"]},{"osd":8,"uuid":"700b2360-e592-406b-a97e-cf9387feae70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6817","nonce":285467211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6819","nonce":285467211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6823","nonce":285467211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":285467211},{"type":"v1","addr":"172.21.15.130:6821","nonce":285467211}]},"public_addr":"172.21.15.130:6817/285467211","cluster_addr":"172.21.15.130:6819/285467211","heartbeat_back_addr":"172.21.15.130:6823/285467211","heartbeat_front_addr":"172.21.15.130:6821/285467211","state":["exists","up"]},{"osd":9,"uuid":"4457f5a8-6991-48c5-a382-dbca97cb206e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6827","nonce":4290214050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6829","nonce":4290214050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6833","nonce":4290214050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":4290214050},{"type":"v1","addr":"172.21.15.32:6831","nonce":4290214050}]},"public_addr":"172.21.15.32:6827/4290214050","cluster_addr":"172.21.15.32:6829/4290214050","heartbeat_back_addr":"172.21.15.32:6833/4290214050","heartbeat_front_addr":"172.21.15.32:6831/4290214050","state":["exists","up"]},{"osd":10,"uuid":"16161ce2-7971-4403-95d4-1c6b2977ce2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6825","nonce":3217408278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6827","nonce":3217408278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6831","nonce":3217408278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3217408278},{"type":"v1","addr":"172.21.15.121:6829","nonce":3217408278}]},"public_addr":"172.21.15.121:6825/3217408278","cluster_addr":"172.21.15.121:6827/3217408278","heartbeat_back_addr":"172.21.15.121:6831/3217408278","heartbeat_front_addr":"172.21.15.121:6829/3217408278","state":["exists","up"]},{"osd":11,"uuid":"fdf0b14b-2f62-4c0b-aec5-a8304d03042b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6825","nonce":2386788822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6827","nonce":2386788822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6831","nonce":2386788822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":2386788822},{"type":"v1","addr":"172.21.15.130:6829","nonce":2386788822}]},"public_addr":"172.21.15.130:6825/2386788822","cluster_addr":"172.21.15.130:6827/2386788822","heartbeat_back_addr":"172.21.15.130:6831/2386788822","heartbeat_front_addr":"172.21.15.130:6829/2386788822","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:17.669304+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:25:41.459623+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:04.766375+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:30.351421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:26:54.841788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:19.616221+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:27:46.384527+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:12.101555+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:28:37.342159+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:04.237546+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:29.371926+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T00:29:55.726225+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.b","mappings":[{"from":8,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":1}]},{"pgid":"2.f","mappings":[{"from":8,"to":11}]},{"pgid":"2.10","mappings":[{"from":2,"to":4}]},{"pgid":"3.6","mappings":[{"from":3,"to":6}]},{"pgid":"3.17","mappings":[{"from":4,"to":7}]},{"pgid":"3.1b","mappings":[{"from":5,"to":9}]},{"pgid":"3.20","mappings":[{"from":5,"to":9}]},{"pgid":"3.23","mappings":[{"from":4,"to":8}]},{"pgid":"3.2c","mappings":[{"from":4,"to":9}]},{"pgid":"4.c","mappings":[{"from":9,"to":10}]},{"pgid":"4.12","mappings":[{"from":9,"to":10}]},{"pgid":"4.1c","mappings":[{"from":9,"to":8}]},{"pgid":"4.1d","mappings":[{"from":5,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:6801/585810931":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/1096527590":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/1767429535":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/3915353181":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/2999636731":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6801/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/4160192081":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/53550767":"2024-04-26T00:23:42.776439+0000","172.21.15.32:0/1066471448":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6800/585810931":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/3987019264":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/3041942084":"2024-04-26T00:23:14.613921+0000","172.21.15.32:0/1694990206":"2024-04-26T00:33:17.919894+0000","172.21.15.32:0/3004184616":"2024-04-26T00:23:42.776439+0000","172.21.15.32:6800/1637256323":"2024-04-26T00:23:14.613921+0000","172.21.15.32:6801/1637256323":"2024-04-26T00:23:14.613921+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-num-failure-domains":"0","crush-osds-per-failure-domain":"0","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":3,"snaps":[{"begin":7,"length":1},{"begin":19,"length":1}]},{"pool":4,"snaps":[{"begin":7,"length":1},{"begin":19,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T00:45:33.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.666+0000 7fe0dffc8640 1 -- 172.21.15.32:0/3363945738 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe0c40a2d00 msgr2=0x7fe0c40a51c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:33.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.666+0000 7fe0dffc8640 1 --2- 172.21.15.32:0/3363945738 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe0c40a2d00 0x7fe0c40a51c0 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7fe0d8077df0 tx=0x7fe0c800a4c0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.666 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.666+0000 7fe0dffc8640 1 -- 172.21.15.32:0/3363945738 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe0d8100690 msgr2=0x7fe0d8198010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:33.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.666+0000 7fe0dffc8640 1 --2- 172.21.15.32:0/3363945738 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe0d8100690 0x7fe0d8198010 secure :-1 s=READY pgs=674 cs=0 l=1 rev1=1 crypto rx=0x7fe0d400c970 tx=0x7fe0d400ce30 comp rx=0 tx=0).stop 2024-04-25T00:45:33.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.666+0000 7fe0dffc8640 1 -- 172.21.15.32:0/3363945738 shutdown_connections 2024-04-25T00:45:33.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.666+0000 7fe0dffc8640 1 --2- 172.21.15.32:0/3363945738 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7fe0c40a2d00 0x7fe0c40a51c0 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.666+0000 7fe0dffc8640 1 --2- 172.21.15.32:0/3363945738 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7fe0d8106650 0x7fe0d819c8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.666+0000 7fe0dffc8640 1 --2- 172.21.15.32:0/3363945738 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe0d8101050 0x7fe0d8198550 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.666+0000 7fe0dffc8640 1 --2- 172.21.15.32:0/3363945738 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7fe0d8100690 0x7fe0d8198010 unknown :-1 s=CLOSED pgs=674 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.666+0000 7fe0dffc8640 1 -- 172.21.15.32:0/3363945738 >> 172.21.15.32:0/3363945738 conn(0x7fe0d80777a0 msgr2=0x7fe0d806a020 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:33.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.666+0000 7fe0dffc8640 1 -- 172.21.15.32:0/3363945738 shutdown_connections 2024-04-25T00:45:33.667 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.666+0000 7fe0dffc8640 1 -- 172.21.15.32:0/3363945738 wait complete. 2024-04-25T00:45:33.676 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:33 smithi032 ceph-mon[29357]: pgmap v383: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:45:33.676 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:33 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2210767016' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:45:33.676 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:33 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2510032596' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:45:33.677 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs fail cephfs 2024-04-25T00:45:33.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.780+0000 7f708f7f9640 1 -- 172.21.15.32:0/1728394973 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7088101710 msgr2=0x7f7088101b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:33.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.780+0000 7f708f7f9640 1 --2- 172.21.15.32:0/1728394973 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7088101710 0x7f7088101b90 secure :-1 s=READY pgs=675 cs=0 l=1 rev1=1 crypto rx=0x7f7078009930 tx=0x7f707802a420 comp rx=0 tx=0).stop 2024-04-25T00:45:33.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.780+0000 7f708f7f9640 1 -- 172.21.15.32:0/1728394973 shutdown_connections 2024-04-25T00:45:33.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.780+0000 7f708f7f9640 1 --2- 172.21.15.32:0/1728394973 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f70881020d0 0x7f708810e5e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.780+0000 7f708f7f9640 1 --2- 172.21.15.32:0/1728394973 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7088101710 0x7f7088101b90 unknown :-1 s=CLOSED pgs=675 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.783 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.780+0000 7f708f7f9640 1 --2- 172.21.15.32:0/1728394973 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7088107720 0x7f7088107b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.784 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.780+0000 7f708f7f9640 1 -- 172.21.15.32:0/1728394973 >> 172.21.15.32:0/1728394973 conn(0x7f7088078b90 msgr2=0x7f7088078fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:33.784 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.781+0000 7f708f7f9640 1 -- 172.21.15.32:0/1728394973 shutdown_connections 2024-04-25T00:45:33.784 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.781+0000 7f708f7f9640 1 -- 172.21.15.32:0/1728394973 wait complete. 2024-04-25T00:45:33.784 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.781+0000 7f708f7f9640 1 Processor -- start 2024-04-25T00:45:33.784 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.781+0000 7f708f7f9640 1 -- start start 2024-04-25T00:45:33.784 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.781+0000 7f708f7f9640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7088101710 0x7f7088073a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:33.784 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.781+0000 7f708f7f9640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f70881020d0 0x7f7088073f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:33.784 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.781+0000 7f708f7f9640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7088107720 0x7f70880744c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:33.784 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.781+0000 7f708f7f9640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f7088111b30 con 0x7f7088101710 2024-04-25T00:45:33.784 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.781+0000 7f708f7f9640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f70881119b0 con 0x7f7088107720 2024-04-25T00:45:33.784 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.781+0000 7f708f7f9640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f7088111cb0 con 0x7f70881020d0 2024-04-25T00:45:33.785 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.781+0000 7f708d595640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7088101710 0x7f7088073a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:33.785 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.781+0000 7f708d595640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7088101710 0x7f7088073a40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:51026/0 (socket says 172.21.15.32:51026) 2024-04-25T00:45:33.785 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.781+0000 7f708d595640 1 -- 172.21.15.32:0/1499663936 learned_addr learned my addr 172.21.15.32:0/1499663936 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:45:33.785 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.782+0000 7f708cd94640 1 --2- 172.21.15.32:0/1499663936 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f70881020d0 0x7f7088073f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:33.785 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.782+0000 7f708d595640 1 -- 172.21.15.32:0/1499663936 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f70881020d0 msgr2=0x7f7088073f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:33.785 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.782+0000 7f708d595640 1 --2- 172.21.15.32:0/1499663936 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f70881020d0 0x7f7088073f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.785 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.782+0000 7f708d595640 1 -- 172.21.15.32:0/1499663936 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7088107720 msgr2=0x7f70880744c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:33.785 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.782+0000 7f708dd96640 1 --2- 172.21.15.32:0/1499663936 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7088107720 0x7f70880744c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:33.785 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.782+0000 7f708d595640 1 --2- 172.21.15.32:0/1499663936 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7088107720 0x7f70880744c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:33.785 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.782+0000 7f708d595640 1 -- 172.21.15.32:0/1499663936 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f70881a6590 con 0x7f7088101710 2024-04-25T00:45:33.785 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.782+0000 7f708d595640 1 --2- 172.21.15.32:0/1499663936 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7088101710 0x7f7088073a40 secure :-1 s=READY pgs=676 cs=0 l=1 rev1=1 crypto rx=0x7f708400d950 tx=0x7f708400de10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:33.785 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.782+0000 7f707e7fc640 1 -- 172.21.15.32:0/1499663936 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7084017070 con 0x7f7088101710 2024-04-25T00:45:33.785 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.782+0000 7f707e7fc640 1 -- 172.21.15.32:0/1499663936 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7084009c30 con 0x7f7088101710 2024-04-25T00:45:33.786 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.782+0000 7f708f7f9640 1 -- 172.21.15.32:0/1499663936 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f70881a6880 con 0x7f7088101710 2024-04-25T00:45:33.786 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.782+0000 7f707e7fc640 1 -- 172.21.15.32:0/1499663936 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f708400f6b0 con 0x7f7088101710 2024-04-25T00:45:33.786 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.782+0000 7f708f7f9640 1 -- 172.21.15.32:0/1499663936 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f70881a6dd0 con 0x7f7088101710 2024-04-25T00:45:33.786 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.783+0000 7f708f7f9640 1 -- 172.21.15.32:0/1499663936 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f70880689b0 con 0x7f7088101710 2024-04-25T00:45:33.787 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.784+0000 7f707e7fc640 1 -- 172.21.15.32:0/1499663936 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f70840109d0 con 0x7f7088101710 2024-04-25T00:45:33.787 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.785+0000 7f707e7fc640 1 --2- 172.21.15.32:0/1499663936 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f70740a2d00 0x7f70740a51c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:33.787 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.785+0000 7f708cd94640 1 --2- 172.21.15.32:0/1499663936 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f70740a2d00 0x7f70740a51c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:33.787 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.785+0000 7f707e7fc640 1 -- 172.21.15.32:0/1499663936 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(107..107 src has 1..107) ==== 9554+0+0 (secure 0 0 0) 0x7f70840cf500 con 0x7f7088101710 2024-04-25T00:45:33.787 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.785+0000 7f708cd94640 1 --2- 172.21.15.32:0/1499663936 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f70740a2d00 0x7f70740a51c0 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f70780096c0 tx=0x7f7078000bc0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:33.789 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:33.789+0000 7f707e7fc640 1 -- 172.21.15.32:0/1499663936 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f7084009de0 con 0x7f7088101710 2024-04-25T00:45:33.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:33 smithi121 ceph-mon[32231]: pgmap v383: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:45:33.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:33 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2210767016' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:45:33.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:33 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2510032596' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:45:33.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:33 smithi130 ceph-mon[32525]: pgmap v383: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-25T00:45:33.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:33 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2210767016' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:45:33.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:33 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2510032596' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-25T00:45:34.013 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.012+0000 7f708f7f9640 1 -- 172.21.15.32:0/1499663936 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs fail", "fs_name": "cephfs"} v 0) -- 0x7f7088109c80 con 0x7f7088101710 2024-04-25T00:45:34.018 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.015+0000 7f707e7fc640 1 -- 172.21.15.32:0/1499663936 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs fail", "fs_name": "cephfs"}]=0 cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. v33) ==== 161+0+0 (secure 0 0 0) 0x7f708409bcb0 con 0x7f7088101710 2024-04-25T00:45:34.018 INFO:teuthology.orchestra.run.smithi032.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-04-25T00:45:34.018 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.017+0000 7f708f7f9640 1 -- 172.21.15.32:0/1499663936 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f70740a2d00 msgr2=0x7f70740a51c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:34.019 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.017+0000 7f708f7f9640 1 --2- 172.21.15.32:0/1499663936 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f70740a2d00 0x7f70740a51c0 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f70780096c0 tx=0x7f7078000bc0 comp rx=0 tx=0).stop 2024-04-25T00:45:34.019 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.017+0000 7f708f7f9640 1 -- 172.21.15.32:0/1499663936 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7088101710 msgr2=0x7f7088073a40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:34.019 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.017+0000 7f708f7f9640 1 --2- 172.21.15.32:0/1499663936 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7088101710 0x7f7088073a40 secure :-1 s=READY pgs=676 cs=0 l=1 rev1=1 crypto rx=0x7f708400d950 tx=0x7f708400de10 comp rx=0 tx=0).stop 2024-04-25T00:45:34.019 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.018+0000 7f708f7f9640 1 -- 172.21.15.32:0/1499663936 shutdown_connections 2024-04-25T00:45:34.019 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.018+0000 7f708f7f9640 1 --2- 172.21.15.32:0/1499663936 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f70740a2d00 0x7f70740a51c0 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:34.019 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.018+0000 7f708f7f9640 1 --2- 172.21.15.32:0/1499663936 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f7088107720 0x7f70880744c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:34.019 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.018+0000 7f708f7f9640 1 --2- 172.21.15.32:0/1499663936 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f70881020d0 0x7f7088073f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:34.019 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.018+0000 7f708f7f9640 1 --2- 172.21.15.32:0/1499663936 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f7088101710 0x7f7088073a40 unknown :-1 s=CLOSED pgs=676 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:34.019 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.018+0000 7f708f7f9640 1 -- 172.21.15.32:0/1499663936 >> 172.21.15.32:0/1499663936 conn(0x7f7088078b90 msgr2=0x7f70880ff160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:34.019 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.018+0000 7f708f7f9640 1 -- 172.21.15.32:0/1499663936 shutdown_connections 2024-04-25T00:45:34.019 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.018+0000 7f708f7f9640 1 -- 172.21.15.32:0/1499663936 wait complete. 2024-04-25T00:45:34.030 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs rm cephfs --yes-i-really-mean-it 2024-04-25T00:45:34.139 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.136+0000 7efecb1a9640 1 -- 172.21.15.32:0/1828129951 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efec4100fd0 msgr2=0x7efec410d4e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:34.139 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.136+0000 7efecb1a9640 1 --2- 172.21.15.32:0/1828129951 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efec4100fd0 0x7efec410d4e0 secure :-1 s=READY pgs=677 cs=0 l=1 rev1=1 crypto rx=0x7efebc009cd0 tx=0x7efebc02a8e0 comp rx=0 tx=0).stop 2024-04-25T00:45:34.139 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.136+0000 7efecb1a9640 1 -- 172.21.15.32:0/1828129951 shutdown_connections 2024-04-25T00:45:34.139 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.136+0000 7efecb1a9640 1 --2- 172.21.15.32:0/1828129951 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efec4100fd0 0x7efec410d4e0 unknown :-1 s=CLOSED pgs=677 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:34.139 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.136+0000 7efecb1a9640 1 --2- 172.21.15.32:0/1828129951 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efec4100610 0x7efec4100a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:34.139 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.136+0000 7efecb1a9640 1 --2- 172.21.15.32:0/1828129951 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efec41066b0 0x7efec4106a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:34.139 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.136+0000 7efecb1a9640 1 -- 172.21.15.32:0/1828129951 >> 172.21.15.32:0/1828129951 conn(0x7efec4077740 msgr2=0x7efec4077b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:34.139 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.136+0000 7efecb1a9640 1 -- 172.21.15.32:0/1828129951 shutdown_connections 2024-04-25T00:45:34.139 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.136+0000 7efecb1a9640 1 -- 172.21.15.32:0/1828129951 wait complete. 2024-04-25T00:45:34.139 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.137+0000 7efecb1a9640 1 Processor -- start 2024-04-25T00:45:34.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.137+0000 7efecb1a9640 1 -- start start 2024-04-25T00:45:34.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.137+0000 7efecb1a9640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efec4100610 0x7efec4197f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:34.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.137+0000 7efecb1a9640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efec4100fd0 0x7efec41984c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:34.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.137+0000 7efecb1a9640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efec41066b0 0x7efec419c7f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:34.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.137+0000 7efecb1a9640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7efec4110a40 con 0x7efec41066b0 2024-04-25T00:45:34.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.137+0000 7efecb1a9640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7efec41108c0 con 0x7efec4100610 2024-04-25T00:45:34.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.137+0000 7efecb1a9640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7efec4110bc0 con 0x7efec4100fd0 2024-04-25T00:45:34.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.137+0000 7efec9746640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efec41066b0 0x7efec419c7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:34.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.137+0000 7efec9746640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efec41066b0 0x7efec419c7f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:51032/0 (socket says 172.21.15.32:51032) 2024-04-25T00:45:34.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.137+0000 7efec9746640 1 -- 172.21.15.32:0/2423509742 learned_addr learned my addr 172.21.15.32:0/2423509742 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:45:34.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.137+0000 7efec8f45640 1 --2- 172.21.15.32:0/2423509742 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efec4100610 0x7efec4197f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:34.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.137+0000 7efec3fff640 1 --2- 172.21.15.32:0/2423509742 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efec4100fd0 0x7efec41984c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:34.140 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.138+0000 7efec9746640 1 -- 172.21.15.32:0/2423509742 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efec4100fd0 msgr2=0x7efec41984c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:34.141 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.138+0000 7efec9746640 1 --2- 172.21.15.32:0/2423509742 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efec4100fd0 0x7efec41984c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:34.141 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.138+0000 7efec9746640 1 -- 172.21.15.32:0/2423509742 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efec4100610 msgr2=0x7efec4197f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:34.141 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.138+0000 7efec9746640 1 --2- 172.21.15.32:0/2423509742 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efec4100610 0x7efec4197f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:34.141 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.138+0000 7efec9746640 1 -- 172.21.15.32:0/2423509742 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7efec419cf20 con 0x7efec41066b0 2024-04-25T00:45:34.141 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.138+0000 7efec9746640 1 --2- 172.21.15.32:0/2423509742 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efec41066b0 0x7efec419c7f0 secure :-1 s=READY pgs=678 cs=0 l=1 rev1=1 crypto rx=0x7efebc033040 tx=0x7efebc003970 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:34.141 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.138+0000 7efec1ffb640 1 -- 172.21.15.32:0/2423509742 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efebc004040 con 0x7efec41066b0 2024-04-25T00:45:34.141 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.138+0000 7efecb1a9640 1 -- 172.21.15.32:0/2423509742 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7efec419d1b0 con 0x7efec41066b0 2024-04-25T00:45:34.141 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.138+0000 7efec1ffb640 1 -- 172.21.15.32:0/2423509742 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7efebc0046c0 con 0x7efec41066b0 2024-04-25T00:45:34.141 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.138+0000 7efec1ffb640 1 -- 172.21.15.32:0/2423509742 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efebc046480 con 0x7efec41066b0 2024-04-25T00:45:34.141 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.138+0000 7efecb1a9640 1 -- 172.21.15.32:0/2423509742 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7efec41a4aa0 con 0x7efec41066b0 2024-04-25T00:45:34.142 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.139+0000 7efecb1a9640 1 -- 172.21.15.32:0/2423509742 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7efec4068560 con 0x7efec41066b0 2024-04-25T00:45:34.142 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.140+0000 7efec1ffb640 1 -- 172.21.15.32:0/2423509742 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7efebc0305a0 con 0x7efec41066b0 2024-04-25T00:45:34.143 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.141+0000 7efec1ffb640 1 --2- 172.21.15.32:0/2423509742 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7efeb00a2d00 0x7efeb00a51c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:34.143 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.141+0000 7efec1ffb640 1 -- 172.21.15.32:0/2423509742 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(108..108 src has 1..108) ==== 9724+0+0 (secure 0 0 0) 0x7efebc02e030 con 0x7efec41066b0 2024-04-25T00:45:34.143 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.141+0000 7efec8f45640 1 --2- 172.21.15.32:0/2423509742 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7efeb00a2d00 0x7efeb00a51c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:34.143 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.142+0000 7efec8f45640 1 --2- 172.21.15.32:0/2423509742 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7efeb00a2d00 0x7efeb00a51c0 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7efec4101750 tx=0x7efeb400b390 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:34.149 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.149+0000 7efec1ffb640 1 -- 172.21.15.32:0/2423509742 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7efebc04b060 con 0x7efec41066b0 2024-04-25T00:45:34.317 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:34 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-a[29353]: 2024-04-25T00:45:34.013+0000 7fe956d96700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-25T00:45:34.337 INFO:journalctl@ceph.mds.b.smithi121.stdout:Apr 25 00:45:34 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-b[62582]: ignoring --setuser ceph since I am not root 2024-04-25T00:45:34.338 INFO:journalctl@ceph.mds.b.smithi121.stdout:Apr 25 00:45:34 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-b[62582]: ignoring --setgroup ceph since I am not root 2024-04-25T00:45:34.338 INFO:journalctl@ceph.mds.b.smithi121.stdout:Apr 25 00:45:34 smithi121 ceph-mds[62586]: -- 172.21.15.121:0/804151822 <== mon.2 v2:172.21.15.121:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c5feec3a40 con 0x55c5fee96800 2024-04-25T00:45:34.338 INFO:journalctl@ceph.mds.b.smithi121.stdout:Apr 25 00:45:34 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-b[62582]: starting mds.b at 2024-04-25T00:45:34.338 INFO:journalctl@ceph.mds.e.smithi121.stdout:Apr 25 00:45:34 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-e[61960]: ignoring --setuser ceph since I am not root 2024-04-25T00:45:34.338 INFO:journalctl@ceph.mds.e.smithi121.stdout:Apr 25 00:45:34 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-e[61960]: ignoring --setgroup ceph since I am not root 2024-04-25T00:45:34.338 INFO:journalctl@ceph.mds.e.smithi121.stdout:Apr 25 00:45:34 smithi121 ceph-mds[61964]: -- 172.21.15.121:0/1345403059 <== mon.2 v2:172.21.15.121:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c7cbce1a40 con 0x55c7cbcb4800 2024-04-25T00:45:34.339 INFO:journalctl@ceph.mds.e.smithi121.stdout:Apr 25 00:45:34 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-e[61960]: starting mds.e at 2024-04-25T00:45:34.339 INFO:journalctl@ceph.mds.h.smithi121.stdout:Apr 25 00:45:34 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-h[62273]: ignoring --setuser ceph since I am not root 2024-04-25T00:45:34.339 INFO:journalctl@ceph.mds.h.smithi121.stdout:Apr 25 00:45:34 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-h[62273]: ignoring --setgroup ceph since I am not root 2024-04-25T00:45:34.339 INFO:journalctl@ceph.mds.h.smithi121.stdout:Apr 25 00:45:34 smithi121 ceph-mds[62277]: -- 172.21.15.121:0/2676159577 <== mon.2 v2:172.21.15.121:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f40656ba40 con 0x55f40653f000 2024-04-25T00:45:34.339 INFO:journalctl@ceph.mds.h.smithi121.stdout:Apr 25 00:45:34 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-h[62273]: starting mds.h at 2024-04-25T00:45:34.378 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:34.378+0000 7efecb1a9640 1 -- 172.21.15.32:0/2423509742 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} v 0) -- 0x7efec41a4cd0 con 0x7efec41066b0 2024-04-25T00:45:34.391 INFO:journalctl@ceph.mds.i.smithi130.stdout:Apr 25 00:45:34 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-i[63369]: ignoring --setuser ceph since I am not root 2024-04-25T00:45:34.392 INFO:journalctl@ceph.mds.i.smithi130.stdout:Apr 25 00:45:34 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-i[63369]: ignoring --setgroup ceph since I am not root 2024-04-25T00:45:34.392 INFO:journalctl@ceph.mds.i.smithi130.stdout:Apr 25 00:45:34 smithi130 ceph-mds[63373]: -- 172.21.15.130:0/1579525549 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5603a28c3a40 con 0x5603a2896800 2024-04-25T00:45:34.392 INFO:journalctl@ceph.mds.i.smithi130.stdout:Apr 25 00:45:34 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-i[63369]: starting mds.i at 2024-04-25T00:45:34.392 INFO:journalctl@ceph.mds.c.smithi130.stdout:Apr 25 00:45:34 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-c[63059]: ignoring --setuser ceph since I am not root 2024-04-25T00:45:34.392 INFO:journalctl@ceph.mds.c.smithi130.stdout:Apr 25 00:45:34 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-c[63059]: ignoring --setgroup ceph since I am not root 2024-04-25T00:45:34.392 INFO:journalctl@ceph.mds.c.smithi130.stdout:Apr 25 00:45:34 smithi130 ceph-mds[63063]: -- 172.21.15.130:0/680991170 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562c065dda40 con 0x562c065b1000 2024-04-25T00:45:34.392 INFO:journalctl@ceph.mds.c.smithi130.stdout:Apr 25 00:45:34 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-c[63059]: starting mds.c at 2024-04-25T00:45:34.393 INFO:journalctl@ceph.mds.f.smithi130.stdout:Apr 25 00:45:34 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-f[62750]: ignoring --setuser ceph since I am not root 2024-04-25T00:45:34.393 INFO:journalctl@ceph.mds.f.smithi130.stdout:Apr 25 00:45:34 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-f[62750]: ignoring --setgroup ceph since I am not root 2024-04-25T00:45:34.393 INFO:journalctl@ceph.mds.f.smithi130.stdout:Apr 25 00:45:34 smithi130 ceph-mds[62754]: -- 172.21.15.130:0/2763624584 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559c57db3a40 con 0x559c57d86800 2024-04-25T00:45:34.393 INFO:journalctl@ceph.mds.f.smithi130.stdout:Apr 25 00:45:34 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-f[62750]: starting mds.f at 2024-04-25T00:45:34.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:34 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3363945738' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:45:34.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:34 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1499663936' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2024-04-25T00:45:34.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:34 smithi032 ceph-mon[29357]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-25T00:45:34.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:34 smithi032 ceph-mon[29357]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-25T00:45:34.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:34 smithi032 ceph-mon[29357]: Health check cleared: MDS_DAMAGE (was: 1 MDSs report damaged metadata) 2024-04-25T00:45:34.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:34 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1499663936' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-04-25T00:45:34.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:34 smithi032 ceph-mon[29357]: osdmap e108: 12 total, 12 up, 12 in 2024-04-25T00:45:34.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:34 smithi032 ceph-mon[29357]: fsmap cephfs:0/3 6 up:standby, 3 failed 2024-04-25T00:45:34.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:34 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2423509742' entity='client.admin' cmd=[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]: dispatch 2024-04-25T00:45:34.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:34 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3363945738' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:45:34.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:34 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1499663936' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2024-04-25T00:45:34.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:34 smithi121 ceph-mon[32231]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-25T00:45:34.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:34 smithi121 ceph-mon[32231]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-25T00:45:34.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:34 smithi121 ceph-mon[32231]: Health check cleared: MDS_DAMAGE (was: 1 MDSs report damaged metadata) 2024-04-25T00:45:34.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:34 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1499663936' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-04-25T00:45:34.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:34 smithi121 ceph-mon[32231]: osdmap e108: 12 total, 12 up, 12 in 2024-04-25T00:45:34.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:34 smithi121 ceph-mon[32231]: fsmap cephfs:0/3 6 up:standby, 3 failed 2024-04-25T00:45:34.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:34 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2423509742' entity='client.admin' cmd=[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]: dispatch 2024-04-25T00:45:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:34 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3363945738' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T00:45:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:34 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1499663936' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2024-04-25T00:45:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:34 smithi130 ceph-mon[32525]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-25T00:45:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:34 smithi130 ceph-mon[32525]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-25T00:45:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:34 smithi130 ceph-mon[32525]: Health check cleared: MDS_DAMAGE (was: 1 MDSs report damaged metadata) 2024-04-25T00:45:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:34 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1499663936' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-04-25T00:45:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:34 smithi130 ceph-mon[32525]: osdmap e108: 12 total, 12 up, 12 in 2024-04-25T00:45:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:34 smithi130 ceph-mon[32525]: fsmap cephfs:0/3 6 up:standby, 3 failed 2024-04-25T00:45:34.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:34 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2423509742' entity='client.admin' cmd=[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]: dispatch 2024-04-25T00:45:35.022 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.021+0000 7efec1ffb640 1 -- 172.21.15.32:0/2423509742 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]=0 v34) ==== 104+0+0 (secure 0 0 0) 0x7efebc0bd960 con 0x7efec41066b0 2024-04-25T00:45:35.024 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.024+0000 7efecb1a9640 1 -- 172.21.15.32:0/2423509742 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7efeb00a2d00 msgr2=0x7efeb00a51c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:35.024 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.024+0000 7efecb1a9640 1 --2- 172.21.15.32:0/2423509742 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7efeb00a2d00 0x7efeb00a51c0 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7efec4101750 tx=0x7efeb400b390 comp rx=0 tx=0).stop 2024-04-25T00:45:35.024 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.024+0000 7efecb1a9640 1 -- 172.21.15.32:0/2423509742 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efec41066b0 msgr2=0x7efec419c7f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:35.025 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.024+0000 7efecb1a9640 1 --2- 172.21.15.32:0/2423509742 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efec41066b0 0x7efec419c7f0 secure :-1 s=READY pgs=678 cs=0 l=1 rev1=1 crypto rx=0x7efebc033040 tx=0x7efebc003970 comp rx=0 tx=0).stop 2024-04-25T00:45:35.025 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.024+0000 7efecb1a9640 1 -- 172.21.15.32:0/2423509742 shutdown_connections 2024-04-25T00:45:35.025 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.024+0000 7efecb1a9640 1 --2- 172.21.15.32:0/2423509742 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7efeb00a2d00 0x7efeb00a51c0 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:35.025 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.024+0000 7efecb1a9640 1 --2- 172.21.15.32:0/2423509742 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7efec41066b0 0x7efec419c7f0 unknown :-1 s=CLOSED pgs=678 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:35.025 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.024+0000 7efecb1a9640 1 --2- 172.21.15.32:0/2423509742 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efec4100fd0 0x7efec41984c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:35.025 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.024+0000 7efecb1a9640 1 --2- 172.21.15.32:0/2423509742 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7efec4100610 0x7efec4197f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:35.025 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.024+0000 7efecb1a9640 1 -- 172.21.15.32:0/2423509742 >> 172.21.15.32:0/2423509742 conn(0x7efec4077740 msgr2=0x7efec40ff800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:35.025 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.024+0000 7efecb1a9640 1 -- 172.21.15.32:0/2423509742 shutdown_connections 2024-04-25T00:45:35.025 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.024+0000 7efecb1a9640 1 -- 172.21.15.32:0/2423509742 wait complete. 2024-04-25T00:45:35.037 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-04-25T00:45:35.146 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.143+0000 7f925f9a2640 1 -- 172.21.15.32:0/1260052533 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f92581032e0 msgr2=0x7f925810b7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:35.146 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.143+0000 7f925f9a2640 1 --2- 172.21.15.32:0/1260052533 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f92581032e0 0x7f925810b7c0 secure :-1 s=READY pgs=679 cs=0 l=1 rev1=1 crypto rx=0x7f9248009900 tx=0x7f924802a510 comp rx=0 tx=0).stop 2024-04-25T00:45:35.146 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.143+0000 7f925f9a2640 1 -- 172.21.15.32:0/1260052533 shutdown_connections 2024-04-25T00:45:35.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.143+0000 7f925f9a2640 1 --2- 172.21.15.32:0/1260052533 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f925810bd00 0x7f925810e1a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:35.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.143+0000 7f925f9a2640 1 --2- 172.21.15.32:0/1260052533 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f92581032e0 0x7f925810b7c0 unknown :-1 s=CLOSED pgs=679 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:35.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.143+0000 7f925f9a2640 1 --2- 172.21.15.32:0/1260052533 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f92581029c0 0x7f9258102da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:35.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.143+0000 7f925f9a2640 1 -- 172.21.15.32:0/1260052533 >> 172.21.15.32:0/1260052533 conn(0x7f9258077740 msgr2=0x7f9258077b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:35.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.143+0000 7f925f9a2640 1 -- 172.21.15.32:0/1260052533 shutdown_connections 2024-04-25T00:45:35.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.143+0000 7f925f9a2640 1 -- 172.21.15.32:0/1260052533 wait complete. 2024-04-25T00:45:35.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.144+0000 7f925f9a2640 1 Processor -- start 2024-04-25T00:45:35.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.144+0000 7f925f9a2640 1 -- start start 2024-04-25T00:45:35.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.144+0000 7f925f9a2640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f92581029c0 0x7f9258198060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:35.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.144+0000 7f925f9a2640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f92581032e0 0x7f92581985a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:35.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.144+0000 7f925f9a2640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f925810bd00 0x7f925819c920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:35.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.144+0000 7f925f9a2640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f9258111b40 con 0x7f92581029c0 2024-04-25T00:45:35.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.144+0000 7f925f9a2640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f92581119c0 con 0x7f925810bd00 2024-04-25T00:45:35.147 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.144+0000 7f925f9a2640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f9258111cc0 con 0x7f92581032e0 2024-04-25T00:45:35.148 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.144+0000 7f925d73e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f92581029c0 0x7f9258198060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:35.148 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.144+0000 7f925d73e640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f92581029c0 0x7f9258198060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:51050/0 (socket says 172.21.15.32:51050) 2024-04-25T00:45:35.148 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.144+0000 7f925d73e640 1 -- 172.21.15.32:0/3874087931 learned_addr learned my addr 172.21.15.32:0/3874087931 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:45:35.148 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.144+0000 7f925cf3d640 1 --2- 172.21.15.32:0/3874087931 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f92581032e0 0x7f92581985a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:35.148 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.145+0000 7f925df3f640 1 --2- 172.21.15.32:0/3874087931 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f925810bd00 0x7f925819c920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:35.148 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.145+0000 7f925d73e640 1 -- 172.21.15.32:0/3874087931 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f92581032e0 msgr2=0x7f92581985a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:35.148 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.145+0000 7f925d73e640 1 --2- 172.21.15.32:0/3874087931 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f92581032e0 0x7f92581985a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:35.148 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.145+0000 7f925d73e640 1 -- 172.21.15.32:0/3874087931 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f925810bd00 msgr2=0x7f925819c920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:35.148 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.145+0000 7f925d73e640 1 --2- 172.21.15.32:0/3874087931 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f925810bd00 0x7f925819c920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:35.148 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.145+0000 7f925d73e640 1 -- 172.21.15.32:0/3874087931 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f925819d050 con 0x7f92581029c0 2024-04-25T00:45:35.148 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.145+0000 7f925cf3d640 1 --2- 172.21.15.32:0/3874087931 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f92581032e0 0x7f92581985a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-25T00:45:35.149 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.145+0000 7f925d73e640 1 --2- 172.21.15.32:0/3874087931 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f92581029c0 0x7f9258198060 secure :-1 s=READY pgs=680 cs=0 l=1 rev1=1 crypto rx=0x7f925400b4a0 tx=0x7f925400b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:35.149 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.145+0000 7f924e7fc640 1 -- 172.21.15.32:0/3874087931 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9254014070 con 0x7f92581029c0 2024-04-25T00:45:35.149 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.145+0000 7f925f9a2640 1 -- 172.21.15.32:0/3874087931 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f925819d340 con 0x7f92581029c0 2024-04-25T00:45:35.149 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.145+0000 7f924e7fc640 1 -- 172.21.15.32:0/3874087931 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9254010af0 con 0x7f92581029c0 2024-04-25T00:45:35.149 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.145+0000 7f924e7fc640 1 -- 172.21.15.32:0/3874087931 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9254017d10 con 0x7f92581029c0 2024-04-25T00:45:35.149 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.145+0000 7f925f9a2640 1 -- 172.21.15.32:0/3874087931 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f92581a4bd0 con 0x7f92581029c0 2024-04-25T00:45:35.149 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.146+0000 7f925f9a2640 1 -- 172.21.15.32:0/3874087931 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9258068560 con 0x7f92581029c0 2024-04-25T00:45:35.150 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.147+0000 7f924e7fc640 1 -- 172.21.15.32:0/3874087931 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f9254017070 con 0x7f92581029c0 2024-04-25T00:45:35.150 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.148+0000 7f924e7fc640 1 --2- 172.21.15.32:0/3874087931 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f92440a2d00 0x7f92440a51c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:35.150 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.148+0000 7f925cf3d640 1 --2- 172.21.15.32:0/3874087931 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f92440a2d00 0x7f92440a51c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:35.150 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.148+0000 7f924e7fc640 1 -- 172.21.15.32:0/3874087931 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(108..108 src has 1..108) ==== 9724+0+0 (secure 0 0 0) 0x7f92540cec10 con 0x7f92581029c0 2024-04-25T00:45:35.150 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.148+0000 7f925cf3d640 1 --2- 172.21.15.32:0/3874087931 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f92440a2d00 0x7f92440a51c0 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f9258100040 tx=0x7f9248005a20 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:35.152 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.152+0000 7f924e7fc640 1 -- 172.21.15.32:0/3874087931 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f925409b490 con 0x7f92581029c0 2024-04-25T00:45:35.339 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:35.338+0000 7f925f9a2640 1 -- 172.21.15.32:0/3874087931 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} v 0) -- 0x7f9258106e80 con 0x7f92581029c0 2024-04-25T00:45:35.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:35 smithi032 ceph-mon[29357]: pgmap v385: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 307 B/s wr, 6 op/s 2024-04-25T00:45:35.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:35 smithi032 ceph-mon[29357]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-25T00:45:35.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:35 smithi032 ceph-mon[29357]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-25T00:45:35.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:35 smithi032 ceph-mon[29357]: Cluster is now healthy 2024-04-25T00:45:35.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:35 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/2423509742' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-04-25T00:45:35.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:35 smithi032 ceph-mon[29357]: mds.? [v2:172.21.15.130:6834/1424600823,v1:172.21.15.130:6835/1424600823] up:boot 2024-04-25T00:45:35.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:35 smithi032 ceph-mon[29357]: mds.? [v2:172.21.15.130:6836/1426395850,v1:172.21.15.130:6838/1426395850] up:boot 2024-04-25T00:45:35.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:35 smithi032 ceph-mon[29357]: mds.? [v2:172.21.15.130:6837/2219106339,v1:172.21.15.130:6839/2219106339] up:boot 2024-04-25T00:45:35.749 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:35 smithi032 ceph-mon[29357]: mds.? [v2:172.21.15.121:6834/2838597744,v1:172.21.15.121:6835/2838597744] up:boot 2024-04-25T00:45:35.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:35 smithi032 ceph-mon[29357]: mds.? [v2:172.21.15.121:6836/48105958,v1:172.21.15.121:6838/48105958] up:boot 2024-04-25T00:45:35.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:35 smithi032 ceph-mon[29357]: mds.? [v2:172.21.15.121:6837/4012263137,v1:172.21.15.121:6839/4012263137] up:boot 2024-04-25T00:45:35.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:35 smithi032 ceph-mon[29357]: fsmap 12 up:standby 2024-04-25T00:45:35.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:35 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-04-25T00:45:35.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:35 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-04-25T00:45:35.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:35 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-04-25T00:45:35.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:35 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-04-25T00:45:35.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:35 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-04-25T00:45:35.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:35 smithi032 ceph-mon[29357]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-04-25T00:45:35.750 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:35 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3874087931' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-25T00:45:35.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:35 smithi121 ceph-mon[32231]: pgmap v385: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 307 B/s wr, 6 op/s 2024-04-25T00:45:35.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:35 smithi121 ceph-mon[32231]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-25T00:45:35.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:35 smithi121 ceph-mon[32231]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-25T00:45:35.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:35 smithi121 ceph-mon[32231]: Cluster is now healthy 2024-04-25T00:45:35.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:35 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/2423509742' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-04-25T00:45:35.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:35 smithi121 ceph-mon[32231]: mds.? [v2:172.21.15.130:6834/1424600823,v1:172.21.15.130:6835/1424600823] up:boot 2024-04-25T00:45:35.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:35 smithi121 ceph-mon[32231]: mds.? [v2:172.21.15.130:6836/1426395850,v1:172.21.15.130:6838/1426395850] up:boot 2024-04-25T00:45:35.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:35 smithi121 ceph-mon[32231]: mds.? [v2:172.21.15.130:6837/2219106339,v1:172.21.15.130:6839/2219106339] up:boot 2024-04-25T00:45:35.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:35 smithi121 ceph-mon[32231]: mds.? [v2:172.21.15.121:6834/2838597744,v1:172.21.15.121:6835/2838597744] up:boot 2024-04-25T00:45:35.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:35 smithi121 ceph-mon[32231]: mds.? [v2:172.21.15.121:6836/48105958,v1:172.21.15.121:6838/48105958] up:boot 2024-04-25T00:45:35.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:35 smithi121 ceph-mon[32231]: mds.? [v2:172.21.15.121:6837/4012263137,v1:172.21.15.121:6839/4012263137] up:boot 2024-04-25T00:45:35.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:35 smithi121 ceph-mon[32231]: fsmap 12 up:standby 2024-04-25T00:45:35.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:35 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-04-25T00:45:35.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:35 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-04-25T00:45:35.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:35 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-04-25T00:45:35.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:35 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-04-25T00:45:35.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:35 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-04-25T00:45:35.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:35 smithi121 ceph-mon[32231]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-04-25T00:45:35.839 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:35 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3874087931' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-25T00:45:35.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:35 smithi130 ceph-mon[32525]: pgmap v385: 129 pgs: 129 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 307 B/s wr, 6 op/s 2024-04-25T00:45:35.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:35 smithi130 ceph-mon[32525]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-25T00:45:35.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:35 smithi130 ceph-mon[32525]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-25T00:45:35.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:35 smithi130 ceph-mon[32525]: Cluster is now healthy 2024-04-25T00:45:35.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:35 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/2423509742' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-04-25T00:45:35.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:35 smithi130 ceph-mon[32525]: mds.? [v2:172.21.15.130:6834/1424600823,v1:172.21.15.130:6835/1424600823] up:boot 2024-04-25T00:45:35.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:35 smithi130 ceph-mon[32525]: mds.? [v2:172.21.15.130:6836/1426395850,v1:172.21.15.130:6838/1426395850] up:boot 2024-04-25T00:45:35.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:35 smithi130 ceph-mon[32525]: mds.? [v2:172.21.15.130:6837/2219106339,v1:172.21.15.130:6839/2219106339] up:boot 2024-04-25T00:45:35.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:35 smithi130 ceph-mon[32525]: mds.? [v2:172.21.15.121:6834/2838597744,v1:172.21.15.121:6835/2838597744] up:boot 2024-04-25T00:45:35.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:35 smithi130 ceph-mon[32525]: mds.? [v2:172.21.15.121:6836/48105958,v1:172.21.15.121:6838/48105958] up:boot 2024-04-25T00:45:35.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:35 smithi130 ceph-mon[32525]: mds.? [v2:172.21.15.121:6837/4012263137,v1:172.21.15.121:6839/4012263137] up:boot 2024-04-25T00:45:35.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:35 smithi130 ceph-mon[32525]: fsmap 12 up:standby 2024-04-25T00:45:35.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:35 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-04-25T00:45:35.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:35 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-04-25T00:45:35.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:35 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-04-25T00:45:35.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:35 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-04-25T00:45:35.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:35 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-04-25T00:45:35.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:35 smithi130 ceph-mon[32525]: from='mgr.15483 172.21.15.32:0/3382440423' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-04-25T00:45:35.895 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:35 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3874087931' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-25T00:45:36.024 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.023+0000 7f924e7fc640 1 -- 172.21.15.32:0/3874087931 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_metadata' removed v109) ==== 181+0+0 (secure 0 0 0) 0x7f925409ad90 con 0x7f92581029c0 2024-04-25T00:45:36.024 INFO:teuthology.orchestra.run.smithi032.stderr:pool 'cephfs_metadata' removed 2024-04-25T00:45:36.028 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.027+0000 7f925f9a2640 1 -- 172.21.15.32:0/3874087931 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f92440a2d00 msgr2=0x7f92440a51c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:36.028 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.027+0000 7f925f9a2640 1 --2- 172.21.15.32:0/3874087931 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f92440a2d00 0x7f92440a51c0 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f9258100040 tx=0x7f9248005a20 comp rx=0 tx=0).stop 2024-04-25T00:45:36.028 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.027+0000 7f925f9a2640 1 -- 172.21.15.32:0/3874087931 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f92581029c0 msgr2=0x7f9258198060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:36.028 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.027+0000 7f925f9a2640 1 --2- 172.21.15.32:0/3874087931 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f92581029c0 0x7f9258198060 secure :-1 s=READY pgs=680 cs=0 l=1 rev1=1 crypto rx=0x7f925400b4a0 tx=0x7f925400b960 comp rx=0 tx=0).stop 2024-04-25T00:45:36.028 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.027+0000 7f925f9a2640 1 -- 172.21.15.32:0/3874087931 shutdown_connections 2024-04-25T00:45:36.028 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.027+0000 7f925f9a2640 1 --2- 172.21.15.32:0/3874087931 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f92440a2d00 0x7f92440a51c0 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:36.028 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.027+0000 7f925f9a2640 1 --2- 172.21.15.32:0/3874087931 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f925810bd00 0x7f925819c920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:36.028 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.027+0000 7f925f9a2640 1 --2- 172.21.15.32:0/3874087931 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f92581032e0 0x7f92581985a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:36.028 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.027+0000 7f925f9a2640 1 --2- 172.21.15.32:0/3874087931 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f92581029c0 0x7f9258198060 unknown :-1 s=CLOSED pgs=680 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:36.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.027+0000 7f925f9a2640 1 -- 172.21.15.32:0/3874087931 >> 172.21.15.32:0/3874087931 conn(0x7f9258077740 msgr2=0x7f92580fd700 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:36.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.028+0000 7f925f9a2640 1 -- 172.21.15.32:0/3874087931 shutdown_connections 2024-04-25T00:45:36.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.028+0000 7f925f9a2640 1 -- 172.21.15.32:0/3874087931 wait complete. 2024-04-25T00:45:36.044 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-04-25T00:45:36.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.158+0000 7f1b50770640 1 -- 172.21.15.32:0/1101486554 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1b481020d0 msgr2=0x7f1b4810e5e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:36.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.158+0000 7f1b50770640 1 --2- 172.21.15.32:0/1101486554 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1b481020d0 0x7f1b4810e5e0 secure :-1 s=READY pgs=681 cs=0 l=1 rev1=1 crypto rx=0x7f1b40009cd0 tx=0x7f1b4002a8e0 comp rx=0 tx=0).stop 2024-04-25T00:45:36.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.158+0000 7f1b50770640 1 -- 172.21.15.32:0/1101486554 shutdown_connections 2024-04-25T00:45:36.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.158+0000 7f1b50770640 1 --2- 172.21.15.32:0/1101486554 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1b481020d0 0x7f1b4810e5e0 unknown :-1 s=CLOSED pgs=681 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:36.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.158+0000 7f1b50770640 1 --2- 172.21.15.32:0/1101486554 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1b48101710 0x7f1b48101b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:36.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.158+0000 7f1b50770640 1 --2- 172.21.15.32:0/1101486554 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1b48107720 0x7f1b48107b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:36.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.158+0000 7f1b50770640 1 -- 172.21.15.32:0/1101486554 >> 172.21.15.32:0/1101486554 conn(0x7f1b480fd180 msgr2=0x7f1b480ff5a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:36.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.158+0000 7f1b50770640 1 -- 172.21.15.32:0/1101486554 shutdown_connections 2024-04-25T00:45:36.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.158+0000 7f1b50770640 1 -- 172.21.15.32:0/1101486554 wait complete. 2024-04-25T00:45:36.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.159+0000 7f1b50770640 1 Processor -- start 2024-04-25T00:45:36.162 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.159+0000 7f1b50770640 1 -- start start 2024-04-25T00:45:36.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.159+0000 7f1b50770640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1b48101710 0x7f1b48197ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:36.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.159+0000 7f1b50770640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1b481020d0 0x7f1b48198530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:36.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.159+0000 7f1b50770640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1b48107720 0x7f1b4819c8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:36.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.159+0000 7f1b50770640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f1b48111af0 con 0x7f1b48101710 2024-04-25T00:45:36.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.159+0000 7f1b50770640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f1b48111970 con 0x7f1b481020d0 2024-04-25T00:45:36.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.159+0000 7f1b50770640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f1b48111c70 con 0x7f1b48107720 2024-04-25T00:45:36.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.159+0000 7f1b4e50c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1b48101710 0x7f1b48197ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:36.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.159+0000 7f1b4e50c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1b48101710 0x7f1b48197ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:51074/0 (socket says 172.21.15.32:51074) 2024-04-25T00:45:36.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.159+0000 7f1b4e50c640 1 -- 172.21.15.32:0/1611987678 learned_addr learned my addr 172.21.15.32:0/1611987678 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:45:36.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.159+0000 7f1b4dd0b640 1 --2- 172.21.15.32:0/1611987678 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1b481020d0 0x7f1b48198530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:36.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.159+0000 7f1b4ed0d640 1 --2- 172.21.15.32:0/1611987678 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1b48107720 0x7f1b4819c8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:36.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.159+0000 7f1b4e50c640 1 -- 172.21.15.32:0/1611987678 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1b48107720 msgr2=0x7f1b4819c8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:36.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.159+0000 7f1b4e50c640 1 --2- 172.21.15.32:0/1611987678 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1b48107720 0x7f1b4819c8b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:36.163 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.159+0000 7f1b4e50c640 1 -- 172.21.15.32:0/1611987678 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1b481020d0 msgr2=0x7f1b48198530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:36.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.159+0000 7f1b4e50c640 1 --2- 172.21.15.32:0/1611987678 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1b481020d0 0x7f1b48198530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:36.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.160+0000 7f1b4e50c640 1 -- 172.21.15.32:0/1611987678 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1b4819cfe0 con 0x7f1b48101710 2024-04-25T00:45:36.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.160+0000 7f1b4e50c640 1 --2- 172.21.15.32:0/1611987678 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1b48101710 0x7f1b48197ff0 secure :-1 s=READY pgs=682 cs=0 l=1 rev1=1 crypto rx=0x7f1b4400d950 tx=0x7f1b4400de10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:36.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.160+0000 7f1b3f7fe640 1 -- 172.21.15.32:0/1611987678 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1b44017070 con 0x7f1b48101710 2024-04-25T00:45:36.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.160+0000 7f1b3f7fe640 1 -- 172.21.15.32:0/1611987678 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1b44009c30 con 0x7f1b48101710 2024-04-25T00:45:36.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.160+0000 7f1b50770640 1 -- 172.21.15.32:0/1611987678 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1b4819d2d0 con 0x7f1b48101710 2024-04-25T00:45:36.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.160+0000 7f1b3f7fe640 1 -- 172.21.15.32:0/1611987678 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1b4400f6b0 con 0x7f1b48101710 2024-04-25T00:45:36.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.160+0000 7f1b50770640 1 -- 172.21.15.32:0/1611987678 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1b481a4b60 con 0x7f1b48101710 2024-04-25T00:45:36.164 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.161+0000 7f1b50770640 1 -- 172.21.15.32:0/1611987678 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1b480689b0 con 0x7f1b48101710 2024-04-25T00:45:36.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.163+0000 7f1b3f7fe640 1 -- 172.21.15.32:0/1611987678 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f1b440109d0 con 0x7f1b48101710 2024-04-25T00:45:36.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.163+0000 7f1b3f7fe640 1 --2- 172.21.15.32:0/1611987678 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f1b340a2d00 0x7f1b340a51c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:36.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.163+0000 7f1b4dd0b640 1 --2- 172.21.15.32:0/1611987678 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f1b340a2d00 0x7f1b340a51c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:36.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.163+0000 7f1b3f7fe640 1 -- 172.21.15.32:0/1611987678 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(109..109 src has 1..109) ==== 9170+0+0 (secure 0 0 0) 0x7f1b440cf270 con 0x7f1b48101710 2024-04-25T00:45:36.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.164+0000 7f1b4dd0b640 1 --2- 172.21.15.32:0/1611987678 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f1b340a2d00 0x7f1b340a51c0 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f1b480ffbc0 tx=0x7f1b3800a2d0 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:36.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.167+0000 7f1b3f7fe640 1 -- 172.21.15.32:0/1611987678 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f1b44009de0 con 0x7f1b48101710 2024-04-25T00:45:36.377 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:36.376+0000 7f1b50770640 1 -- 172.21.15.32:0/1611987678 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} v 0) -- 0x7f1b48199340 con 0x7f1b48101710 2024-04-25T00:45:37.029 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.029+0000 7f1b3f7fe640 1 -- 172.21.15.32:0/1611987678 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_data' removed v110) ==== 169+0+0 (secure 0 0 0) 0x7f1b4409bc00 con 0x7f1b48101710 2024-04-25T00:45:37.029 INFO:teuthology.orchestra.run.smithi032.stderr:pool 'cephfs_data' removed 2024-04-25T00:45:37.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.033+0000 7f1b50770640 1 -- 172.21.15.32:0/1611987678 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f1b340a2d00 msgr2=0x7f1b340a51c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:37.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.033+0000 7f1b50770640 1 --2- 172.21.15.32:0/1611987678 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f1b340a2d00 0x7f1b340a51c0 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f1b480ffbc0 tx=0x7f1b3800a2d0 comp rx=0 tx=0).stop 2024-04-25T00:45:37.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.033+0000 7f1b50770640 1 -- 172.21.15.32:0/1611987678 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1b48101710 msgr2=0x7f1b48197ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:37.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.033+0000 7f1b50770640 1 --2- 172.21.15.32:0/1611987678 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1b48101710 0x7f1b48197ff0 secure :-1 s=READY pgs=682 cs=0 l=1 rev1=1 crypto rx=0x7f1b4400d950 tx=0x7f1b4400de10 comp rx=0 tx=0).stop 2024-04-25T00:45:37.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.033+0000 7f1b50770640 1 -- 172.21.15.32:0/1611987678 shutdown_connections 2024-04-25T00:45:37.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.033+0000 7f1b50770640 1 --2- 172.21.15.32:0/1611987678 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f1b340a2d00 0x7f1b340a51c0 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:37.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.033+0000 7f1b50770640 1 --2- 172.21.15.32:0/1611987678 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1b48107720 0x7f1b4819c8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:37.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.033+0000 7f1b50770640 1 --2- 172.21.15.32:0/1611987678 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f1b481020d0 0x7f1b48198530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:37.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.033+0000 7f1b50770640 1 --2- 172.21.15.32:0/1611987678 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f1b48101710 0x7f1b48197ff0 unknown :-1 s=CLOSED pgs=682 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:37.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.033+0000 7f1b50770640 1 -- 172.21.15.32:0/1611987678 >> 172.21.15.32:0/1611987678 conn(0x7f1b480fd180 msgr2=0x7f1b480fd780 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:37.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.033+0000 7f1b50770640 1 -- 172.21.15.32:0/1611987678 shutdown_connections 2024-04-25T00:45:37.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.033+0000 7f1b50770640 1 -- 172.21.15.32:0/1611987678 wait complete. 2024-04-25T00:45:37.050 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_data_ec cephfs_data_ec --yes-i-really-really-mean-it 2024-04-25T00:45:37.165 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.163+0000 7f2088ef7640 1 -- 172.21.15.32:0/3446998684 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2084107730 msgr2=0x7f2084107b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:37.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.163+0000 7f2088ef7640 1 --2- 172.21.15.32:0/3446998684 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2084107730 0x7f2084107b10 secure :-1 s=READY pgs=683 cs=0 l=1 rev1=1 crypto rx=0x7f2074009930 tx=0x7f207402a420 comp rx=0 tx=0).stop 2024-04-25T00:45:37.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.163+0000 7f2088ef7640 1 -- 172.21.15.32:0/3446998684 shutdown_connections 2024-04-25T00:45:37.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.163+0000 7f2088ef7640 1 --2- 172.21.15.32:0/3446998684 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f208406a780 0x7f208410e5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:37.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.163+0000 7f2088ef7640 1 --2- 172.21.15.32:0/3446998684 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f2084069dc0 0x7f208406a240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:37.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.163+0000 7f2088ef7640 1 --2- 172.21.15.32:0/3446998684 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2084107730 0x7f2084107b10 unknown :-1 s=CLOSED pgs=683 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:37.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.163+0000 7f2088ef7640 1 -- 172.21.15.32:0/3446998684 >> 172.21.15.32:0/3446998684 conn(0x7f20840776a0 msgr2=0x7f2084077ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:37.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.163+0000 7f2088ef7640 1 -- 172.21.15.32:0/3446998684 shutdown_connections 2024-04-25T00:45:37.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.163+0000 7f2088ef7640 1 -- 172.21.15.32:0/3446998684 wait complete. 2024-04-25T00:45:37.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.163+0000 7f2088ef7640 1 Processor -- start 2024-04-25T00:45:37.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.164+0000 7f2088ef7640 1 -- start start 2024-04-25T00:45:37.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.164+0000 7f2088ef7640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2084069dc0 0x7f2084198070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:37.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.164+0000 7f2088ef7640 1 --2- >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f208406a780 0x7f20841985b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:37.166 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.164+0000 7f2088ef7640 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2084107730 0x7f208419c930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:37.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.164+0000 7f2088ef7640 1 -- --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_getmap magic: 0 -- 0x7f2084111bb0 con 0x7f2084069dc0 2024-04-25T00:45:37.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.164+0000 7f2088ef7640 1 -- --> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] -- mon_getmap magic: 0 -- 0x7f2084111a30 con 0x7f208406a780 2024-04-25T00:45:37.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.164+0000 7f2088ef7640 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 -- 0x7f2084111d30 con 0x7f2084107730 2024-04-25T00:45:37.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.164+0000 7f208259c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2084069dc0 0x7f2084198070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:37.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.164+0000 7f208259c640 1 --2- >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2084069dc0 0x7f2084198070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.32:3300/0 says I am v2:172.21.15.32:51086/0 (socket says 172.21.15.32:51086) 2024-04-25T00:45:37.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.164+0000 7f208259c640 1 -- 172.21.15.32:0/348776393 learned_addr learned my addr 172.21.15.32:0/348776393 (peer_addr_for_me v2:172.21.15.32:0/0) 2024-04-25T00:45:37.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.164+0000 7f2079d9b640 1 --2- 172.21.15.32:0/348776393 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f208406a780 0x7f20841985b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:37.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.164+0000 7f2082d9d640 1 --2- 172.21.15.32:0/348776393 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2084107730 0x7f208419c930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:37.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.164+0000 7f208259c640 1 -- 172.21.15.32:0/348776393 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2084107730 msgr2=0x7f208419c930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:37.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.164+0000 7f208259c640 1 --2- 172.21.15.32:0/348776393 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2084107730 0x7f208419c930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:37.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.164+0000 7f208259c640 1 -- 172.21.15.32:0/348776393 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f208406a780 msgr2=0x7f20841985b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:37.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.164+0000 7f208259c640 1 --2- 172.21.15.32:0/348776393 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f208406a780 0x7f20841985b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:37.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.164+0000 7f208259c640 1 -- 172.21.15.32:0/348776393 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f208419d060 con 0x7f2084069dc0 2024-04-25T00:45:37.167 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.165+0000 7f208259c640 1 --2- 172.21.15.32:0/348776393 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2084069dc0 0x7f2084198070 secure :-1 s=READY pgs=684 cs=0 l=1 rev1=1 crypto rx=0x7f207402a940 tx=0x7f2074003b90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:37.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.165+0000 7f207bfff640 1 -- 172.21.15.32:0/348776393 <== mon.0 v2:172.21.15.32:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f207403f070 con 0x7f2084069dc0 2024-04-25T00:45:37.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.165+0000 7f2088ef7640 1 -- 172.21.15.32:0/348776393 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f208419d2f0 con 0x7f2084069dc0 2024-04-25T00:45:37.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.165+0000 7f207bfff640 1 -- 172.21.15.32:0/348776393 <== mon.0 v2:172.21.15.32:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f2074031c90 con 0x7f2084069dc0 2024-04-25T00:45:37.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.165+0000 7f207bfff640 1 -- 172.21.15.32:0/348776393 <== mon.0 v2:172.21.15.32:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2074035030 con 0x7f2084069dc0 2024-04-25T00:45:37.168 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.165+0000 7f2088ef7640 1 -- 172.21.15.32:0/348776393 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f20841a4be0 con 0x7f2084069dc0 2024-04-25T00:45:37.169 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.166+0000 7f2088ef7640 1 -- 172.21.15.32:0/348776393 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f20840689b0 con 0x7f2084069dc0 2024-04-25T00:45:37.169 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.167+0000 7f207bfff640 1 -- 172.21.15.32:0/348776393 <== mon.0 v2:172.21.15.32:3300/0 4 ==== mgrmap(e 26) ==== 137964+0+0 (secure 0 0 0) 0x7f2074042070 con 0x7f2084069dc0 2024-04-25T00:45:37.169 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.168+0000 7f207bfff640 1 --2- 172.21.15.32:0/348776393 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f20700a2d00 0x7f20700a51c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-25T00:45:37.169 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.168+0000 7f207bfff640 1 -- 172.21.15.32:0/348776393 <== mon.0 v2:172.21.15.32:3300/0 5 ==== osd_map(110..110 src has 1..110) ==== 8582+0+0 (secure 0 0 0) 0x7f20740f0120 con 0x7f2084069dc0 2024-04-25T00:45:37.169 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.168+0000 7f2079d9b640 1 --2- 172.21.15.32:0/348776393 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f20700a2d00 0x7f20700a51c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-25T00:45:37.169 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.168+0000 7f2079d9b640 1 --2- 172.21.15.32:0/348776393 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f20700a2d00 0x7f20700a51c0 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f2084077ca0 tx=0x7f206c009110 comp rx=0 tx=0).ready entity=mgr.15483 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-25T00:45:37.172 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.171+0000 7f207bfff640 1 -- 172.21.15.32:0/348776393 <== mon.0 v2:172.21.15.32:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186147 (secure 0 0 0) 0x7f20740bcda0 con 0x7f2084069dc0 2024-04-25T00:45:37.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:37 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/3874087931' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-04-25T00:45:37.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:37 smithi121 ceph-mon[32231]: osdmap e109: 12 total, 12 up, 12 in 2024-04-25T00:45:37.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:37 smithi121 ceph-mon[32231]: pgmap v387: 97 pgs: 97 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:45:37.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:37 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1611987678' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-25T00:45:37.357 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:37.356+0000 7f2088ef7640 1 -- 172.21.15.32:0/348776393 --> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_data_ec", "pool2": "cephfs_data_ec", "yes_i_really_really_mean_it": true} v 0) -- 0x7f2084109cb0 con 0x7f2084069dc0 2024-04-25T00:45:37.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:37 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/3874087931' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-04-25T00:45:37.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:37 smithi130 ceph-mon[32525]: osdmap e109: 12 total, 12 up, 12 in 2024-04-25T00:45:37.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:37 smithi130 ceph-mon[32525]: pgmap v387: 97 pgs: 97 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:45:37.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:37 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1611987678' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-25T00:45:37.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:37 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/3874087931' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-04-25T00:45:37.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:37 smithi032 ceph-mon[29357]: osdmap e109: 12 total, 12 up, 12 in 2024-04-25T00:45:37.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:37 smithi032 ceph-mon[29357]: pgmap v387: 97 pgs: 97 active+clean; 12 GiB data, 34 GiB used, 1.0 TiB / 1.0 TiB avail 2024-04-25T00:45:37.498 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:37 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1611987678' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-25T00:45:38.031 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:38.030+0000 7f207bfff640 1 -- 172.21.15.32:0/348776393 <== mon.0 v2:172.21.15.32:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_data_ec", "pool2": "cephfs_data_ec", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_data_ec' removed v111) ==== 178+0+0 (secure 0 0 0) 0x7f20740bc6a0 con 0x7f2084069dc0 2024-04-25T00:45:38.031 INFO:teuthology.orchestra.run.smithi032.stderr:pool 'cephfs_data_ec' removed 2024-04-25T00:45:38.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:38.033+0000 7f2088ef7640 1 -- 172.21.15.32:0/348776393 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f20700a2d00 msgr2=0x7f20700a51c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:38.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:38.034+0000 7f2088ef7640 1 --2- 172.21.15.32:0/348776393 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f20700a2d00 0x7f20700a51c0 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f2084077ca0 tx=0x7f206c009110 comp rx=0 tx=0).stop 2024-04-25T00:45:38.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:38.034+0000 7f2088ef7640 1 -- 172.21.15.32:0/348776393 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2084069dc0 msgr2=0x7f2084198070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-25T00:45:38.034 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:38.034+0000 7f2088ef7640 1 --2- 172.21.15.32:0/348776393 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2084069dc0 0x7f2084198070 secure :-1 s=READY pgs=684 cs=0 l=1 rev1=1 crypto rx=0x7f207402a940 tx=0x7f2074003b90 comp rx=0 tx=0).stop 2024-04-25T00:45:38.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:38.034+0000 7f2088ef7640 1 -- 172.21.15.32:0/348776393 shutdown_connections 2024-04-25T00:45:38.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:38.034+0000 7f2088ef7640 1 --2- 172.21.15.32:0/348776393 >> [v2:172.21.15.32:6800/3307874911,v1:172.21.15.32:6801/3307874911] conn(0x7f20700a2d00 0x7f20700a51c0 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:38.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:38.034+0000 7f2088ef7640 1 --2- 172.21.15.32:0/348776393 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2084107730 0x7f208419c930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:38.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:38.034+0000 7f2088ef7640 1 --2- 172.21.15.32:0/348776393 >> [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] conn(0x7f208406a780 0x7f20841985b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:38.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:38.034+0000 7f2088ef7640 1 --2- 172.21.15.32:0/348776393 >> [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] conn(0x7f2084069dc0 0x7f2084198070 unknown :-1 s=CLOSED pgs=684 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-25T00:45:38.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:38.034+0000 7f2088ef7640 1 -- 172.21.15.32:0/348776393 >> 172.21.15.32:0/348776393 conn(0x7f20840776a0 msgr2=0x7f20840fe650 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-25T00:45:38.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:38.034+0000 7f2088ef7640 1 -- 172.21.15.32:0/348776393 shutdown_connections 2024-04-25T00:45:38.035 INFO:teuthology.orchestra.run.smithi032.stderr:2024-04-25T00:45:38.034+0000 7f2088ef7640 1 -- 172.21.15.32:0/348776393 wait complete. 2024-04-25T00:45:38.055 DEBUG:teuthology.orchestra.run.smithi032:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-25T00:45:38.088 DEBUG:teuthology.orchestra.run.smithi121:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-25T00:45:38.126 DEBUG:teuthology.orchestra.run.smithi130:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-25T00:45:38.178 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-25T00:45:38.178 DEBUG:teuthology.orchestra.run.smithi032:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-25T00:45:38.210 DEBUG:teuthology.orchestra.run.smithi121:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-25T00:45:38.224 DEBUG:teuthology.orchestra.run.smithi130:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-25T00:45:38.241 INFO:tasks.cephadm:Stopping all daemons... 2024-04-25T00:45:38.241 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-04-25T00:45:38.241 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@mon.a 2024-04-25T00:45:38.297 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:38 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/1611987678' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]': finished 2024-04-25T00:45:38.297 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:38 smithi032 ceph-mon[29357]: osdmap e110: 12 total, 12 up, 12 in 2024-04-25T00:45:38.297 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:38 smithi032 ceph-mon[29357]: from='client.? 172.21.15.32:0/348776393' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data_ec", "pool2": "cephfs_data_ec", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-25T00:45:38.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:38 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/1611987678' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]': finished 2024-04-25T00:45:38.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:38 smithi121 ceph-mon[32231]: osdmap e110: 12 total, 12 up, 12 in 2024-04-25T00:45:38.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:38 smithi121 ceph-mon[32231]: from='client.? 172.21.15.32:0/348776393' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data_ec", "pool2": "cephfs_data_ec", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-25T00:45:38.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:38 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/1611987678' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]': finished 2024-04-25T00:45:38.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:38 smithi130 ceph-mon[32525]: osdmap e110: 12 total, 12 up, 12 in 2024-04-25T00:45:38.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:38 smithi130 ceph-mon[32525]: from='client.? 172.21.15.32:0/348776393' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data_ec", "pool2": "cephfs_data_ec", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-25T00:45:38.748 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:38 smithi032 systemd[1]: Stopping Ceph mon.a for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:45:39.243 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:38 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-a[29353]: 2024-04-25T00:45:38.784+0000 7fe95c5a1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:45:39.243 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:38 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-a[29353]: 2024-04-25T00:45:38.784+0000 7fe95c5a1700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-04-25T00:45:39.243 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:38 smithi032 podman[93369]: 2024-04-25 00:45:38.937993004 +0000 UTC m=+0.308904303 container died 6ab91bb240b189eb93adc0fa53b419bd511bfa26b3a62228a47ce326d35bc156 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-a, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=squid-d2c45d9, ceph=True) 2024-04-25T00:45:39.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:39 smithi032 podman[93369]: 2024-04-25 00:45:39.348371649 +0000 UTC m=+0.719282949 container cleanup 6ab91bb240b189eb93adc0fa53b419bd511bfa26b3a62228a47ce326d35bc156 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, RELEASE=squid-d2c45d9, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-25T00:45:39.499 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:39 smithi032 bash[93369]: ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-a 2024-04-25T00:45:39.933 INFO:journalctl@ceph.mon.a.smithi032.stdout:Apr 25 00:45:39 smithi032 podman[93383]: 2024-04-25 00:45:39.650465559 +0000 UTC m=+0.708641511 container remove 6ab91bb240b189eb93adc0fa53b419bd511bfa26b3a62228a47ce326d35bc156 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-a, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-d2c45d9, org.label-schema.license=GPLv2) 2024-04-25T00:45:39.956 DEBUG:teuthology.orchestra.run.smithi032:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mon.a.service' 2024-04-25T00:45:40.020 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:45:40.020 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-04-25T00:45:40.020 INFO:tasks.cephadm.mon.c:Stopping mon.b... 2024-04-25T00:45:40.020 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@mon.b 2024-04-25T00:45:40.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:40 smithi121 systemd[1]: Stopping Ceph mon.b for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:45:40.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:40 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-b[32226]: 2024-04-25T00:45:40.436+0000 7fca3ce4d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:45:40.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:40 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-b[32226]: 2024-04-25T00:45:40.436+0000 7fca3ce4d700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-04-25T00:45:40.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:40 smithi121 podman[68193]: 2024-04-25 00:45:40.661547291 +0000 UTC m=+0.356205123 container died 61e6e5458627aac8f5215ee6a2564de11e7786b6511ad2d3b86a05969eddaf4e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, GIT_CLEAN=True, RELEASE=squid-d2c45d9, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T00:45:41.222 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:41 smithi121 podman[68193]: 2024-04-25 00:45:41.004030419 +0000 UTC m=+0.698688250 container cleanup 61e6e5458627aac8f5215ee6a2564de11e7786b6511ad2d3b86a05969eddaf4e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-b, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=squid-d2c45d9) 2024-04-25T00:45:41.222 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:41 smithi121 bash[68193]: ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-b 2024-04-25T00:45:41.505 INFO:journalctl@ceph.mon.b.smithi121.stdout:Apr 25 00:45:41 smithi121 podman[68206]: 2024-04-25 00:45:41.222397998 +0000 UTC m=+0.559153780 container remove 61e6e5458627aac8f5215ee6a2564de11e7786b6511ad2d3b86a05969eddaf4e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-b, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, RELEASE=squid-d2c45d9, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-25T00:45:41.528 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mon.b.service' 2024-04-25T00:45:41.569 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:45:41.570 INFO:tasks.cephadm.mon.c:Stopped mon.b 2024-04-25T00:45:41.570 INFO:tasks.cephadm.mon.c:Stopping mon.c... 2024-04-25T00:45:41.570 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@mon.c 2024-04-25T00:45:41.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:41 smithi130 systemd[1]: Stopping Ceph mon.c for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:45:42.378 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:41 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-c[32521]: 2024-04-25T00:45:41.943+0000 7fb3c724b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:45:42.378 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:41 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-c[32521]: 2024-04-25T00:45:41.943+0000 7fb3c724b700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-04-25T00:45:42.378 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:42 smithi130 podman[69281]: 2024-04-25 00:45:42.087496688 +0000 UTC m=+0.245448831 container died ea82c64cab542c9ec472a26f193defca49c74812c8dcdf60cfa63494e9ee1344 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-c, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, RELEASE=squid-d2c45d9, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-25T00:45:42.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:42 smithi130 podman[69281]: 2024-04-25 00:45:42.470168123 +0000 UTC m=+0.628120259 container cleanup ea82c64cab542c9ec472a26f193defca49c74812c8dcdf60cfa63494e9ee1344 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=squid-d2c45d9, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-25T00:45:42.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 25 00:45:42 smithi130 bash[69281]: ceph-cb997156-0299-11ef-bc93-c7b262605968-mon-c 2024-04-25T00:45:43.247 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mon.c.service' 2024-04-25T00:45:43.294 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:45:43.295 INFO:tasks.cephadm.mon.c:Stopped mon.c 2024-04-25T00:45:43.295 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-04-25T00:45:43.295 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@mgr.x 2024-04-25T00:45:43.688 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:45:43 smithi032 systemd[1]: Stopping Ceph mgr.x for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:45:43.998 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:45:43 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:45:43.687+0000 7fc5565d3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.x -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:45:43.998 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:45:43 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x[29591]: 2024-04-25T00:45:43.687+0000 7fc5565d3700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-04-25T00:45:43.998 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:45:43 smithi032 podman[93482]: 2024-04-25 00:45:43.729090075 +0000 UTC m=+0.232249831 container died a5f04bc0c425d5ef5a16cbf24cb99d27ff513e1f65cfb00edc375567a1319949 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-d2c45d9, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-04-25T00:45:44.498 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:45:44 smithi032 podman[93482]: 2024-04-25 00:45:44.230687298 +0000 UTC m=+0.733847061 container cleanup a5f04bc0c425d5ef5a16cbf24cb99d27ff513e1f65cfb00edc375567a1319949 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:d2c45d977142e024a59b68a8f1dce8f0b9840551, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-d2c45d9, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-25T00:45:44.498 INFO:journalctl@ceph.mgr.x.smithi032.stdout:Apr 25 00:45:44 smithi032 bash[93482]: ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-x 2024-04-25T00:45:44.660 DEBUG:teuthology.orchestra.run.smithi032:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mgr.x.service' 2024-04-25T00:45:44.698 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:45:44.698 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-04-25T00:45:44.698 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-04-25T00:45:44.698 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@mgr.y 2024-04-25T00:45:45.020 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:45:44 smithi121 systemd[1]: Stopping Ceph mgr.y for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:45:45.319 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:45:45 smithi121 podman[68313]: 2024-04-25 00:45:45.020819409 +0000 UTC m=+0.142202221 container died 4f08dc59dd340c2271b29f9ee4f681cf5c6248a86e347425fda208772f692dfd (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=squid-d2c45d9, ceph=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-25T00:45:45.588 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:45:45 smithi121 podman[68313]: 2024-04-25 00:45:45.417410507 +0000 UTC m=+0.538793319 container cleanup 4f08dc59dd340c2271b29f9ee4f681cf5c6248a86e347425fda208772f692dfd (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=squid-d2c45d9, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424) 2024-04-25T00:45:45.588 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Apr 25 00:45:45 smithi121 bash[68313]: ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-y 2024-04-25T00:45:45.875 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mgr.y.service' 2024-04-25T00:45:45.908 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:45:45.908 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-04-25T00:45:45.909 INFO:tasks.cephadm.mgr.z:Stopping mgr.z... 2024-04-25T00:45:45.909 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@mgr.z 2024-04-25T00:45:46.255 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:45:45 smithi130 systemd[1]: Stopping Ceph mgr.z for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:45:46.641 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:45:46 smithi130 podman[69405]: 2024-04-25 00:45:46.255276657 +0000 UTC m=+0.153538326 container died 681d0365cccfb1e43d1036dfbf361fb3d3ebb03cd85c97f4431c73a89cc9e8e7 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-d2c45d9, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True) 2024-04-25T00:45:47.120 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:45:46 smithi130 podman[69405]: 2024-04-25 00:45:46.876893412 +0000 UTC m=+0.775155097 container cleanup 681d0365cccfb1e43d1036dfbf361fb3d3ebb03cd85c97f4431c73a89cc9e8e7 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=squid-d2c45d9, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-25T00:45:47.120 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:45:46 smithi130 bash[69405]: ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z 2024-04-25T00:45:47.376 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:45:47 smithi130 podman[69419]: 2024-04-25 00:45:47.120255597 +0000 UTC m=+0.860384748 container remove 681d0365cccfb1e43d1036dfbf361fb3d3ebb03cd85c97f4431c73a89cc9e8e7 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mgr-z, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, RELEASE=squid-d2c45d9, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True) 2024-04-25T00:45:47.377 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Apr 25 00:45:47 smithi130 systemd[1]: ceph-cb997156-0299-11ef-bc93-c7b262605968@mgr.z.service: Main process exited, code=exited, status=143/n/a 2024-04-25T00:45:47.381 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mgr.z.service' 2024-04-25T00:45:47.414 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:45:47.415 INFO:tasks.cephadm.mgr.z:Stopped mgr.z 2024-04-25T00:45:47.415 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-25T00:45:47.415 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.0 2024-04-25T00:45:47.748 INFO:journalctl@ceph.osd.0.smithi032.stdout:Apr 25 00:45:47 smithi032 systemd[1]: Stopping Ceph osd.0 for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:45:48.248 INFO:journalctl@ceph.osd.0.smithi032.stdout:Apr 25 00:45:47 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-0[44152]: 2024-04-25T00:45:47.786+0000 7f6b620b6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:45:48.248 INFO:journalctl@ceph.osd.0.smithi032.stdout:Apr 25 00:45:47 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-0[44152]: 2024-04-25T00:45:47.786+0000 7f6b620b6700 -1 osd.0 111 *** Got signal Terminated *** 2024-04-25T00:45:48.248 INFO:journalctl@ceph.osd.0.smithi032.stdout:Apr 25 00:45:47 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-0[44152]: 2024-04-25T00:45:47.786+0000 7f6b620b6700 -1 osd.0 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T00:45:53.748 INFO:journalctl@ceph.osd.0.smithi032.stdout:Apr 25 00:45:53 smithi032 podman[93603]: 2024-04-25 00:45:53.514110533 +0000 UTC m=+5.915997450 container died 00e3d91f366a239cca61e0eb632d267de01ea9be21579d83927c242cfeaa9c7c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-d2c45d9, org.label-schema.vendor=CentOS) 2024-04-25T00:45:54.206 INFO:journalctl@ceph.osd.0.smithi032.stdout:Apr 25 00:45:53 smithi032 podman[93603]: 2024-04-25 00:45:53.928683686 +0000 UTC m=+6.330570559 container cleanup 00e3d91f366a239cca61e0eb632d267de01ea9be21579d83927c242cfeaa9c7c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, RELEASE=squid-d2c45d9, ceph=True, io.buildah.version=1.33.5) 2024-04-25T00:45:54.206 INFO:journalctl@ceph.osd.0.smithi032.stdout:Apr 25 00:45:53 smithi032 bash[93603]: ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-0 2024-04-25T00:45:54.498 INFO:journalctl@ceph.osd.0.smithi032.stdout:Apr 25 00:45:54 smithi032 podman[93622]: 2024-04-25 00:45:54.205505539 +0000 UTC m=+0.678810005 container remove 00e3d91f366a239cca61e0eb632d267de01ea9be21579d83927c242cfeaa9c7c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-d2c45d9, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-25T00:45:54.889 INFO:journalctl@ceph.osd.0.smithi032.stdout:Apr 25 00:45:54 smithi032 podman[93716]: 2024-04-25 00:45:54.46472235 +0000 UTC m=+0.018598178 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901 2024-04-25T00:45:54.889 INFO:journalctl@ceph.osd.0.smithi032.stdout:Apr 25 00:45:54 smithi032 podman[93716]: 2024-04-25 00:45:54.606816956 +0000 UTC m=+0.160692783 container create efa2ce926ddadec487bebd3bdacc55fbb32d359f7f1084e73d71852b2adf8462 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-0-deactivate, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, RELEASE=squid-d2c45d9, CEPH_POINT_RELEASE=) 2024-04-25T00:45:54.889 INFO:journalctl@ceph.osd.0.smithi032.stdout:Apr 25 00:45:54 smithi032 podman[93716]: 2024-04-25 00:45:54.888927356 +0000 UTC m=+0.442803183 container init efa2ce926ddadec487bebd3bdacc55fbb32d359f7f1084e73d71852b2adf8462 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=squid-d2c45d9, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_CLEAN=True, ceph=True) 2024-04-25T00:45:55.248 INFO:journalctl@ceph.osd.0.smithi032.stdout:Apr 25 00:45:54 smithi032 podman[93716]: 2024-04-25 00:45:54.892644089 +0000 UTC m=+0.446519917 container start efa2ce926ddadec487bebd3bdacc55fbb32d359f7f1084e73d71852b2adf8462 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=squid-d2c45d9, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2) 2024-04-25T00:45:55.248 INFO:journalctl@ceph.osd.0.smithi032.stdout:Apr 25 00:45:55 smithi032 podman[93716]: 2024-04-25 00:45:55.018994671 +0000 UTC m=+0.572870498 container attach efa2ce926ddadec487bebd3bdacc55fbb32d359f7f1084e73d71852b2adf8462 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-0-deactivate, RELEASE=squid-d2c45d9, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T00:45:55.713 INFO:journalctl@ceph.osd.0.smithi032.stdout:Apr 25 00:45:55 smithi032 podman[93716]: 2024-04-25 00:45:55.408798405 +0000 UTC m=+0.962674243 container died efa2ce926ddadec487bebd3bdacc55fbb32d359f7f1084e73d71852b2adf8462 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=squid-d2c45d9, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-04-25T00:45:56.048 DEBUG:teuthology.orchestra.run.smithi032:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.0.service' 2024-04-25T00:45:56.089 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:45:56.089 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-25T00:45:56.089 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-25T00:45:56.089 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.1 2024-04-25T00:45:56.389 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:45:56 smithi121 systemd[1]: Stopping Ceph osd.1 for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:45:56.837 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:45:56 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-1[36801]: 2024-04-25T00:45:56.388+0000 7f9787bd0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:45:56.837 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:45:56 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-1[36801]: 2024-04-25T00:45:56.388+0000 7f9787bd0700 -1 osd.1 111 *** Got signal Terminated *** 2024-04-25T00:45:56.838 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:45:56 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-1[36801]: 2024-04-25T00:45:56.388+0000 7f9787bd0700 -1 osd.1 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T00:46:02.337 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:46:02 smithi121 podman[68438]: 2024-04-25 00:46:02.050773011 +0000 UTC m=+5.774539357 container died 4f10008a4b42a37635a710665438539ea1208486205c83fa08024e8c0cf9006f (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, RELEASE=squid-d2c45d9, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2) 2024-04-25T00:46:02.675 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:46:02 smithi121 podman[68438]: 2024-04-25 00:46:02.48858139 +0000 UTC m=+6.212347732 container cleanup 4f10008a4b42a37635a710665438539ea1208486205c83fa08024e8c0cf9006f (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-1, io.buildah.version=1.33.5, RELEASE=squid-d2c45d9, ceph=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-25T00:46:02.675 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:46:02 smithi121 bash[68438]: ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-1 2024-04-25T00:46:03.087 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:46:02 smithi121 podman[68455]: 2024-04-25 00:46:02.79022705 +0000 UTC m=+0.727875819 container remove 4f10008a4b42a37635a710665438539ea1208486205c83fa08024e8c0cf9006f (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-1, RELEASE=squid-d2c45d9, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-25T00:46:03.519 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:46:03 smithi121 podman[68553]: 2024-04-25 00:46:03.132655339 +0000 UTC m=+0.020946107 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901 2024-04-25T00:46:03.520 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:46:03 smithi121 podman[68553]: 2024-04-25 00:46:03.257806582 +0000 UTC m=+0.146114505 container create dc48928e4f9a3045e75758b5a9a50a198722e640e62bdec30c1c3c87b7b52fcb (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=squid-d2c45d9, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-04-25T00:46:03.838 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:46:03 smithi121 podman[68553]: 2024-04-25 00:46:03.519497092 +0000 UTC m=+0.407787847 container init dc48928e4f9a3045e75758b5a9a50a198722e640e62bdec30c1c3c87b7b52fcb (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-1-deactivate, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , RELEASE=squid-d2c45d9, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5) 2024-04-25T00:46:03.838 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:46:03 smithi121 podman[68553]: 2024-04-25 00:46:03.523056642 +0000 UTC m=+0.411347398 container start dc48928e4f9a3045e75758b5a9a50a198722e640e62bdec30c1c3c87b7b52fcb (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=squid-d2c45d9, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-04-25T00:46:03.838 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:46:03 smithi121 podman[68553]: 2024-04-25 00:46:03.643958809 +0000 UTC m=+0.532249588 container attach dc48928e4f9a3045e75758b5a9a50a198722e640e62bdec30c1c3c87b7b52fcb (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, RELEASE=squid-d2c45d9, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T00:46:04.337 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 25 00:46:04 smithi121 podman[68553]: 2024-04-25 00:46:04.013640708 +0000 UTC m=+0.901931458 container died dc48928e4f9a3045e75758b5a9a50a198722e640e62bdec30c1c3c87b7b52fcb (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-d2c45d9, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-25T00:46:04.728 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.1.service' 2024-04-25T00:46:04.769 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:46:04.769 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-25T00:46:04.769 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-25T00:46:04.769 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.2 2024-04-25T00:46:05.106 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:46:04 smithi130 systemd[1]: Stopping Ceph osd.2 for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:46:05.391 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:46:05 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-2[37663]: 2024-04-25T00:46:05.105+0000 7f4b62bdf700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:46:05.391 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:46:05 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-2[37663]: 2024-04-25T00:46:05.105+0000 7f4b62bdf700 -1 osd.2 111 *** Got signal Terminated *** 2024-04-25T00:46:05.392 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:46:05 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-2[37663]: 2024-04-25T00:46:05.105+0000 7f4b62bdf700 -1 osd.2 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T00:46:11.142 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:46:10 smithi130 podman[69528]: 2024-04-25 00:46:10.837037855 +0000 UTC m=+5.874135278 container died 6306faf4c41536317545281e83883d204fe181fbcec6fc431f3339735c2949a3 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-d2c45d9, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-25T00:46:11.642 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:46:11 smithi130 podman[69528]: 2024-04-25 00:46:11.389680075 +0000 UTC m=+6.426777495 container cleanup 6306faf4c41536317545281e83883d204fe181fbcec6fc431f3339735c2949a3 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-d2c45d9, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-25T00:46:11.642 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:46:11 smithi130 bash[69528]: ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-2 2024-04-25T00:46:12.048 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:46:11 smithi130 podman[69558]: 2024-04-25 00:46:11.724984699 +0000 UTC m=+0.880565232 container remove 6306faf4c41536317545281e83883d204fe181fbcec6fc431f3339735c2949a3 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-2, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-d2c45d9, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux ) 2024-04-25T00:46:12.354 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:46:12 smithi130 podman[69649]: 2024-04-25 00:46:11.952378901 +0000 UTC m=+0.018367965 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901 2024-04-25T00:46:12.354 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:46:12 smithi130 podman[69649]: 2024-04-25 00:46:12.092858222 +0000 UTC m=+0.158847262 container create 39a80b336399b8dee0f71c8700b747d11c5999947119272871d8b3b655222abd (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-d2c45d9, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=) 2024-04-25T00:46:12.642 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:46:12 smithi130 podman[69649]: 2024-04-25 00:46:12.354233778 +0000 UTC m=+0.420222828 container init 39a80b336399b8dee0f71c8700b747d11c5999947119272871d8b3b655222abd (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-d2c45d9, org.label-schema.schema-version=1.0) 2024-04-25T00:46:12.642 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:46:12 smithi130 podman[69649]: 2024-04-25 00:46:12.357683712 +0000 UTC m=+0.423672753 container start 39a80b336399b8dee0f71c8700b747d11c5999947119272871d8b3b655222abd (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-d2c45d9, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424) 2024-04-25T00:46:12.642 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:46:12 smithi130 podman[69649]: 2024-04-25 00:46:12.462696117 +0000 UTC m=+0.528685168 container attach 39a80b336399b8dee0f71c8700b747d11c5999947119272871d8b3b655222abd (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.build-date=20240424, RELEASE=squid-d2c45d9, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T00:46:12.983 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 25 00:46:12 smithi130 podman[69649]: 2024-04-25 00:46:12.669338672 +0000 UTC m=+0.735327713 container died 39a80b336399b8dee0f71c8700b747d11c5999947119272871d8b3b655222abd (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=squid-d2c45d9, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, ceph=True) 2024-04-25T00:46:13.297 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.2.service' 2024-04-25T00:46:13.335 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:46:13.336 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-25T00:46:13.336 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-25T00:46:13.336 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.3 2024-04-25T00:46:13.617 INFO:journalctl@ceph.osd.3.smithi032.stdout:Apr 25 00:46:13 smithi032 systemd[1]: Stopping Ceph osd.3 for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:46:13.893 INFO:journalctl@ceph.osd.3.smithi032.stdout:Apr 25 00:46:13 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-3[50928]: 2024-04-25T00:46:13.617+0000 7f4ac56a2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:46:13.893 INFO:journalctl@ceph.osd.3.smithi032.stdout:Apr 25 00:46:13 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-3[50928]: 2024-04-25T00:46:13.617+0000 7f4ac56a2700 -1 osd.3 111 *** Got signal Terminated *** 2024-04-25T00:46:13.893 INFO:journalctl@ceph.osd.3.smithi032.stdout:Apr 25 00:46:13 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-3[50928]: 2024-04-25T00:46:13.617+0000 7f4ac56a2700 -1 osd.3 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T00:46:14.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:14 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:14.701+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:15.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:14 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:14.998+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:16.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:15 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:15.750+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:16.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:15 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:15.977+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:16.391 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:15 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:15.943+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:17.215 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:16 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:16.749+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:17.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:16 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:16.973+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:17.391 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:16 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:16.972+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:17.498 INFO:journalctl@ceph.osd.3.smithi032.stdout:Apr 25 00:46:17 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-3[50928]: 2024-04-25T00:46:17.215+0000 7f4abdc8e700 -1 osd.3 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.694587+0000 front 2024-04-25T00:45:51.694523+0000 (oldest deadline 2024-04-25T00:46:16.295552+0000) 2024-04-25T00:46:17.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:17 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:17.727+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:18.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:17 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:17.941+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:18.391 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:18 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:18.011+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:18.498 INFO:journalctl@ceph.osd.3.smithi032.stdout:Apr 25 00:46:18 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-3[50928]: 2024-04-25T00:46:18.242+0000 7f4abdc8e700 -1 osd.3 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.694587+0000 front 2024-04-25T00:45:51.694523+0000 (oldest deadline 2024-04-25T00:46:16.295552+0000) 2024-04-25T00:46:18.892 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:18 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:18.586+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:47.567414+0000 front 2024-04-25T00:45:47.567436+0000 (oldest deadline 2024-04-25T00:46:18.167905+0000) 2024-04-25T00:46:18.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:18 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:18.729+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:19.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:18 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:18.931+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:19.391 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:19 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:19.034+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:19.652 INFO:journalctl@ceph.osd.3.smithi032.stdout:Apr 25 00:46:19 smithi032 podman[93821]: 2024-04-25 00:46:19.373954462 +0000 UTC m=+5.836944670 container died 909f4433856eebbdf8b1a726bff8c8bba959fd95512a9854cb7cf6ce3b623bee (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-3, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, RELEASE=squid-d2c45d9, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424) 2024-04-25T00:46:19.892 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:19 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:19.607+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:47.567414+0000 front 2024-04-25T00:45:47.567436+0000 (oldest deadline 2024-04-25T00:46:18.167905+0000) 2024-04-25T00:46:19.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:19 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:19.699+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:19.998 INFO:journalctl@ceph.osd.3.smithi032.stdout:Apr 25 00:46:19 smithi032 podman[93821]: 2024-04-25 00:46:19.751418549 +0000 UTC m=+6.214408739 container cleanup 909f4433856eebbdf8b1a726bff8c8bba959fd95512a9854cb7cf6ce3b623bee (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, RELEASE=squid-d2c45d9, io.buildah.version=1.33.5) 2024-04-25T00:46:19.998 INFO:journalctl@ceph.osd.3.smithi032.stdout:Apr 25 00:46:19 smithi032 bash[93821]: ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-3 2024-04-25T00:46:20.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:19 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:19.950+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:20.380 INFO:journalctl@ceph.osd.3.smithi032.stdout:Apr 25 00:46:20 smithi032 podman[93850]: 2024-04-25 00:46:20.061604173 +0000 UTC m=+0.685149053 container remove 909f4433856eebbdf8b1a726bff8c8bba959fd95512a9854cb7cf6ce3b623bee (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-3, RELEASE=squid-d2c45d9, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-25T00:46:20.391 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:20 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:20.076+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:20.648 INFO:journalctl@ceph.osd.3.smithi032.stdout:Apr 25 00:46:20 smithi032 podman[93943]: 2024-04-25 00:46:20.284231074 +0000 UTC m=+0.018614571 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901 2024-04-25T00:46:20.648 INFO:journalctl@ceph.osd.3.smithi032.stdout:Apr 25 00:46:20 smithi032 podman[93943]: 2024-04-25 00:46:20.412635343 +0000 UTC m=+0.147018839 container create f3da2574402df5b9de5266b19b979374cf3a890ab744a9b9f85dca57b9fb2bb3 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-d2c45d9, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=) 2024-04-25T00:46:20.649 INFO:journalctl@ceph.osd.3.smithi032.stdout:Apr 25 00:46:20 smithi032 podman[93943]: 2024-04-25 00:46:20.648712981 +0000 UTC m=+0.383096477 container init f3da2574402df5b9de5266b19b979374cf3a890ab744a9b9f85dca57b9fb2bb3 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-d2c45d9, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-25T00:46:20.892 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:20 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:20.627+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:47.567414+0000 front 2024-04-25T00:45:47.567436+0000 (oldest deadline 2024-04-25T00:46:18.167905+0000) 2024-04-25T00:46:20.947 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:20 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:20.728+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:20.947 INFO:journalctl@ceph.osd.3.smithi032.stdout:Apr 25 00:46:20 smithi032 podman[93943]: 2024-04-25 00:46:20.652241428 +0000 UTC m=+0.386624932 container start f3da2574402df5b9de5266b19b979374cf3a890ab744a9b9f85dca57b9fb2bb3 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, RELEASE=squid-d2c45d9, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-25T00:46:20.947 INFO:journalctl@ceph.osd.3.smithi032.stdout:Apr 25 00:46:20 smithi032 podman[93943]: 2024-04-25 00:46:20.765759655 +0000 UTC m=+0.500143166 container attach f3da2574402df5b9de5266b19b979374cf3a890ab744a9b9f85dca57b9fb2bb3 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=squid-d2c45d9, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T00:46:21.248 INFO:journalctl@ceph.osd.3.smithi032.stdout:Apr 25 00:46:20 smithi032 podman[93943]: 2024-04-25 00:46:20.947429812 +0000 UTC m=+0.681813308 container died f3da2574402df5b9de5266b19b979374cf3a890ab744a9b9f85dca57b9fb2bb3 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-3-deactivate, RELEASE=squid-d2c45d9, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0) 2024-04-25T00:46:21.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:20 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:20.902+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:21.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:21 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:21.065+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:21.582 DEBUG:teuthology.orchestra.run.smithi032:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.3.service' 2024-04-25T00:46:21.629 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:46:21.629 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-25T00:46:21.629 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-25T00:46:21.629 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.4 2024-04-25T00:46:21.891 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:21 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:21.611+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:47.567414+0000 front 2024-04-25T00:45:47.567436+0000 (oldest deadline 2024-04-25T00:46:18.167905+0000) 2024-04-25T00:46:21.941 INFO:journalctl@ceph.osd.4.smithi121.stdout:Apr 25 00:46:21 smithi121 systemd[1]: Stopping Ceph osd.4 for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:46:21.941 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:21 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:21.878+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:21.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:21 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:21.760+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:22.337 INFO:journalctl@ceph.osd.4.smithi121.stdout:Apr 25 00:46:21 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-4[43300]: 2024-04-25T00:46:21.940+0000 7f797bb3a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:46:22.337 INFO:journalctl@ceph.osd.4.smithi121.stdout:Apr 25 00:46:21 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-4[43300]: 2024-04-25T00:46:21.940+0000 7f797bb3a700 -1 osd.4 111 *** Got signal Terminated *** 2024-04-25T00:46:22.338 INFO:journalctl@ceph.osd.4.smithi121.stdout:Apr 25 00:46:21 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-4[43300]: 2024-04-25T00:46:21.940+0000 7f797bb3a700 -1 osd.4 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T00:46:22.391 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:22 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:22.102+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:22.891 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:22 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:22.640+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:47.567414+0000 front 2024-04-25T00:45:47.567436+0000 (oldest deadline 2024-04-25T00:46:18.167905+0000) 2024-04-25T00:46:22.892 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:22 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:22.640+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.168480+0000 front 2024-04-25T00:45:58.168573+0000 (oldest deadline 2024-04-25T00:46:22.268508+0000) 2024-04-25T00:46:22.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:22 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:22.729+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:22.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:22 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:22.729+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:23.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:22 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:22.927+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:23.349 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:23 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:23.082+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:23.891 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:23 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:23.607+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:47.567414+0000 front 2024-04-25T00:45:47.567436+0000 (oldest deadline 2024-04-25T00:46:18.167905+0000) 2024-04-25T00:46:23.892 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:23 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:23.607+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.168480+0000 front 2024-04-25T00:45:58.168573+0000 (oldest deadline 2024-04-25T00:46:22.268508+0000) 2024-04-25T00:46:24.250 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:23 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:23.761+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:24.250 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:23 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:23.761+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:24.250 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:23 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:23.779+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.127034+0000 front 2024-04-25T00:45:58.127011+0000 (oldest deadline 2024-04-25T00:46:22.827000+0000) 2024-04-25T00:46:24.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:23 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:23.962+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:24.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:23 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:23.962+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:24.391 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:24 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:24.059+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:24.892 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:24 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:24.589+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:47.567414+0000 front 2024-04-25T00:45:47.567436+0000 (oldest deadline 2024-04-25T00:46:18.167905+0000) 2024-04-25T00:46:24.892 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:24 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:24.589+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.168480+0000 front 2024-04-25T00:45:58.168573+0000 (oldest deadline 2024-04-25T00:46:22.268508+0000) 2024-04-25T00:46:24.998 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:24 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:24.740+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.127034+0000 front 2024-04-25T00:45:58.127011+0000 (oldest deadline 2024-04-25T00:46:22.827000+0000) 2024-04-25T00:46:24.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:24 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:24.809+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:24.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:24 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:24.809+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:25.337 INFO:journalctl@ceph.osd.4.smithi121.stdout:Apr 25 00:46:24 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-4[43300]: 2024-04-25T00:46:24.996+0000 7f797812e700 -1 osd.4 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.013978+0000 front 2024-04-25T00:46:01.014007+0000 (oldest deadline 2024-04-25T00:46:24.514175+0000) 2024-04-25T00:46:25.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:24 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:24.996+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:25.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:24 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:24.996+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:25.391 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:25 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:25.067+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:25.392 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:25 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:25.297+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:25.837 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:25 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:25.475+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:25.891 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:25 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:25.607+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:47.567414+0000 front 2024-04-25T00:45:47.567436+0000 (oldest deadline 2024-04-25T00:46:18.167905+0000) 2024-04-25T00:46:25.892 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:25 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:25.607+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.168480+0000 front 2024-04-25T00:45:58.168573+0000 (oldest deadline 2024-04-25T00:46:22.268508+0000) 2024-04-25T00:46:26.248 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:25 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:25.778+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.127034+0000 front 2024-04-25T00:45:58.127011+0000 (oldest deadline 2024-04-25T00:46:22.827000+0000) 2024-04-25T00:46:26.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:25 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:25.778+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:26.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:25 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:25.778+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:26.337 INFO:journalctl@ceph.osd.4.smithi121.stdout:Apr 25 00:46:25 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-4[43300]: 2024-04-25T00:46:25.993+0000 7f797812e700 -1 osd.4 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.013978+0000 front 2024-04-25T00:46:01.014007+0000 (oldest deadline 2024-04-25T00:46:24.514175+0000) 2024-04-25T00:46:26.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:26 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:26.035+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:26.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:26 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:26.035+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:26.342 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:26 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:26.076+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:26.342 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:26 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:26.076+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:26.641 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:26 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:26.341+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:26.642 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:26 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:26.587+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:47.567414+0000 front 2024-04-25T00:45:47.567436+0000 (oldest deadline 2024-04-25T00:46:18.167905+0000) 2024-04-25T00:46:26.642 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:26 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:26.587+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.168480+0000 front 2024-04-25T00:45:58.168573+0000 (oldest deadline 2024-04-25T00:46:22.268508+0000) 2024-04-25T00:46:26.837 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:26 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:26.448+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:26.998 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:26 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:26.811+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.127034+0000 front 2024-04-25T00:45:58.127011+0000 (oldest deadline 2024-04-25T00:46:22.827000+0000) 2024-04-25T00:46:26.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:26 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:26.729+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:26.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:26 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:26.729+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:27.296 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:27 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:27.041+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:27.296 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:27 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:27.041+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:27.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:27 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:27.033+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:27.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:27 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:27.034+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:27.600 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:27 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:27.295+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:27.837 INFO:journalctl@ceph.osd.4.smithi121.stdout:Apr 25 00:46:27 smithi121 podman[68662]: 2024-04-25 00:46:27.650368971 +0000 UTC m=+5.820061730 container died 5cace27307ef8b10ad06af55a6cb56a028c0ad4e498162161b5d0c9cff920171 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-4, RELEASE=squid-d2c45d9, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T00:46:27.838 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:27 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:27.409+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:27.891 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:27 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:27.600+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:47.567414+0000 front 2024-04-25T00:45:47.567436+0000 (oldest deadline 2024-04-25T00:46:18.167905+0000) 2024-04-25T00:46:27.891 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:27 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:27.600+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.168480+0000 front 2024-04-25T00:45:58.168573+0000 (oldest deadline 2024-04-25T00:46:22.268508+0000) 2024-04-25T00:46:27.998 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:27 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:27.765+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.127034+0000 front 2024-04-25T00:45:58.127011+0000 (oldest deadline 2024-04-25T00:46:22.827000+0000) 2024-04-25T00:46:27.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:27 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:27.721+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:27.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:27 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:27.721+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:28.304 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:28 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:28.051+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:28.304 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:28 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:28.051+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:28.317 INFO:journalctl@ceph.osd.4.smithi121.stdout:Apr 25 00:46:28 smithi121 podman[68662]: 2024-04-25 00:46:28.007413186 +0000 UTC m=+6.177105934 container cleanup 5cace27307ef8b10ad06af55a6cb56a028c0ad4e498162161b5d0c9cff920171 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-4, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-d2c45d9) 2024-04-25T00:46:28.317 INFO:journalctl@ceph.osd.4.smithi121.stdout:Apr 25 00:46:28 smithi121 bash[68662]: ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-4 2024-04-25T00:46:28.317 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:28 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:28.019+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:28.317 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:28 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:28.019+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:28.588 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:28 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:28.369+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:28.588 INFO:journalctl@ceph.osd.4.smithi121.stdout:Apr 25 00:46:28 smithi121 podman[68679]: 2024-04-25 00:46:28.317181694 +0000 UTC m=+0.670568500 container remove 5cace27307ef8b10ad06af55a6cb56a028c0ad4e498162161b5d0c9cff920171 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-4, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=squid-d2c45d9, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-25T00:46:28.638 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:28 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:28.304+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:28.892 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:28 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:28.637+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:47.567414+0000 front 2024-04-25T00:45:47.567436+0000 (oldest deadline 2024-04-25T00:46:18.167905+0000) 2024-04-25T00:46:28.892 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:28 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:28.637+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.168480+0000 front 2024-04-25T00:45:58.168573+0000 (oldest deadline 2024-04-25T00:46:22.268508+0000) 2024-04-25T00:46:28.922 INFO:journalctl@ceph.osd.4.smithi121.stdout:Apr 25 00:46:28 smithi121 podman[68772]: 2024-04-25 00:46:28.556458681 +0000 UTC m=+0.017995529 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901 2024-04-25T00:46:28.923 INFO:journalctl@ceph.osd.4.smithi121.stdout:Apr 25 00:46:28 smithi121 podman[68772]: 2024-04-25 00:46:28.677183972 +0000 UTC m=+0.138720848 container create c612928a018330f16da16d1fcd11ef2f391a70d0846aa8e92cfe6f8671ed2e42 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-d2c45d9, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-25T00:46:28.923 INFO:journalctl@ceph.osd.4.smithi121.stdout:Apr 25 00:46:28 smithi121 podman[68772]: 2024-04-25 00:46:28.922720807 +0000 UTC m=+0.384257638 container init c612928a018330f16da16d1fcd11ef2f391a70d0846aa8e92cfe6f8671ed2e42 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-d2c45d9, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240424) 2024-04-25T00:46:28.998 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:28 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:28.765+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.127034+0000 front 2024-04-25T00:45:58.127011+0000 (oldest deadline 2024-04-25T00:46:22.827000+0000) 2024-04-25T00:46:28.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:28 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:28.687+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:28.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:28 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:28.687+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:29.209 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:29 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:29.033+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:29.209 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:29 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:29.033+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:29.209 INFO:journalctl@ceph.osd.4.smithi121.stdout:Apr 25 00:46:28 smithi121 podman[68772]: 2024-04-25 00:46:28.926152959 +0000 UTC m=+0.387689795 container start c612928a018330f16da16d1fcd11ef2f391a70d0846aa8e92cfe6f8671ed2e42 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-d2c45d9, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-04-25T00:46:29.210 INFO:journalctl@ceph.osd.4.smithi121.stdout:Apr 25 00:46:29 smithi121 podman[68772]: 2024-04-25 00:46:29.005824231 +0000 UTC m=+0.467361065 container attach c612928a018330f16da16d1fcd11ef2f391a70d0846aa8e92cfe6f8671ed2e42 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-d2c45d9, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS) 2024-04-25T00:46:29.391 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:29 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:29.299+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:29.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:29 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:29.053+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:29.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:29 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:29.053+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:29.525 INFO:journalctl@ceph.osd.4.smithi121.stdout:Apr 25 00:46:29 smithi121 podman[68772]: 2024-04-25 00:46:29.210155823 +0000 UTC m=+0.671692660 container died c612928a018330f16da16d1fcd11ef2f391a70d0846aa8e92cfe6f8671ed2e42 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-4-deactivate, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-d2c45d9, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-25T00:46:29.526 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:29 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:29.322+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:29.826 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.4.service' 2024-04-25T00:46:29.864 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:46:29.864 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-25T00:46:29.864 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-25T00:46:29.864 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.5 2024-04-25T00:46:29.890 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:29 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:29.620+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:47.567414+0000 front 2024-04-25T00:45:47.567436+0000 (oldest deadline 2024-04-25T00:46:18.167905+0000) 2024-04-25T00:46:29.890 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:29 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:29.620+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.168480+0000 front 2024-04-25T00:45:58.168573+0000 (oldest deadline 2024-04-25T00:46:22.268508+0000) 2024-04-25T00:46:29.998 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:29 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:29.768+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.127034+0000 front 2024-04-25T00:45:58.127011+0000 (oldest deadline 2024-04-25T00:46:22.827000+0000) 2024-04-25T00:46:29.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:29 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:29.736+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:29.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:29 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:29.736+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:30.087 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:30 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:30.020+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:30.087 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:30 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:30.020+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:30.141 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:29 smithi130 systemd[1]: Stopping Ceph osd.5 for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:46:30.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:30 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:30.053+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:30.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:30 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:30.053+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:30.621 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:30 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:30.295+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:30.621 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:30 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:30.186+0000 7fbd1030e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:46:30.621 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:30 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:30.186+0000 7fbd1030e700 -1 osd.5 111 *** Got signal Terminated *** 2024-04-25T00:46:30.621 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:30 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:30.186+0000 7fbd1030e700 -1 osd.5 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T00:46:30.621 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:30 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:30.620+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:47.567414+0000 front 2024-04-25T00:45:47.567436+0000 (oldest deadline 2024-04-25T00:46:18.167905+0000) 2024-04-25T00:46:30.621 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:30 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:30.620+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.168480+0000 front 2024-04-25T00:45:58.168573+0000 (oldest deadline 2024-04-25T00:46:22.268508+0000) 2024-04-25T00:46:30.837 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:30 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:30.355+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:31.248 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:30 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:30.768+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.127034+0000 front 2024-04-25T00:45:58.127011+0000 (oldest deadline 2024-04-25T00:46:22.827000+0000) 2024-04-25T00:46:31.249 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:30 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:30.768+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:31.249 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:30 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:30.768+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:31.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:31 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:31.059+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:31.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:31 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:31.059+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:31.391 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:31 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:31.254+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:31.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:31 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:31.009+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:31.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:31 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:31.009+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:31.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:31 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:31.009+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:31.837 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:31 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:31.373+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:31.891 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:31 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:31.617+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:47.567414+0000 front 2024-04-25T00:45:47.567436+0000 (oldest deadline 2024-04-25T00:46:18.167905+0000) 2024-04-25T00:46:31.892 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:31 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:31.617+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.168480+0000 front 2024-04-25T00:45:58.168573+0000 (oldest deadline 2024-04-25T00:46:22.268508+0000) 2024-04-25T00:46:31.892 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:31 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:31.617+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.169260+0000 front 2024-04-25T00:46:05.169353+0000 (oldest deadline 2024-04-25T00:46:31.069452+0000) 2024-04-25T00:46:32.248 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:31 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:31.764+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.127034+0000 front 2024-04-25T00:45:58.127011+0000 (oldest deadline 2024-04-25T00:46:22.827000+0000) 2024-04-25T00:46:32.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:31 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:31.800+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:32.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:31 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:31.800+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:32.296 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:31 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:31.964+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:32.296 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:31 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:31.964+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:32.296 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:31 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:31.964+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:32.400 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:32 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:32.096+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:32.400 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:32 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:32.096+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:32.587 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:32 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:32.295+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:32.669 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:32 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:32.400+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:32.891 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:32 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:32.587+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:47.567414+0000 front 2024-04-25T00:45:47.567436+0000 (oldest deadline 2024-04-25T00:46:18.167905+0000) 2024-04-25T00:46:32.892 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:32 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:32.587+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.168480+0000 front 2024-04-25T00:45:58.168573+0000 (oldest deadline 2024-04-25T00:46:22.268508+0000) 2024-04-25T00:46:32.892 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:32 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:32.587+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.169260+0000 front 2024-04-25T00:46:05.169353+0000 (oldest deadline 2024-04-25T00:46:31.069452+0000) 2024-04-25T00:46:32.998 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:32 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:32.738+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.127034+0000 front 2024-04-25T00:45:58.127011+0000 (oldest deadline 2024-04-25T00:46:22.827000+0000) 2024-04-25T00:46:32.998 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:32 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:32.738+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.028931+0000 front 2024-04-25T00:46:08.028973+0000 (oldest deadline 2024-04-25T00:46:32.728749+0000) 2024-04-25T00:46:32.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:32 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:32.813+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:32.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:32 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:32.813+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:33.313 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:32 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:32.942+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:33.313 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:32 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:32.942+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:33.313 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:32 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:32.942+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:33.382 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:33 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:33.102+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:33.382 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:33 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:33.102+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:33.382 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:33 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:33.103+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:33.624 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:33 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:33.313+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:33.624 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:33 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:33.313+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:33.837 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:33 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:33.381+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:33.892 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:33 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:33.623+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:47.567414+0000 front 2024-04-25T00:45:47.567436+0000 (oldest deadline 2024-04-25T00:46:18.167905+0000) 2024-04-25T00:46:33.892 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:33 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:33.623+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.168480+0000 front 2024-04-25T00:45:58.168573+0000 (oldest deadline 2024-04-25T00:46:22.268508+0000) 2024-04-25T00:46:33.892 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:33 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:33.623+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.169260+0000 front 2024-04-25T00:46:05.169353+0000 (oldest deadline 2024-04-25T00:46:31.069452+0000) 2024-04-25T00:46:34.248 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:33 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:33.787+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.127034+0000 front 2024-04-25T00:45:58.127011+0000 (oldest deadline 2024-04-25T00:46:22.827000+0000) 2024-04-25T00:46:34.249 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:33 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:33.787+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.028931+0000 front 2024-04-25T00:46:08.028973+0000 (oldest deadline 2024-04-25T00:46:32.728749+0000) 2024-04-25T00:46:34.249 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:33 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:33.838+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:34.249 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:33 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:33.838+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:34.289 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:33 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:33.905+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:34.289 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:33 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:33.905+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:34.289 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:33 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:33.905+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:34.587 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:34 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:34.117+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:34.587 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:34 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:34.117+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:34.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:34 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:34.117+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:34.588 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:34 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:34.332+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:34.588 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:34 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:34.332+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:04.920336+0000 front 2024-04-25T00:46:04.920235+0000 (oldest deadline 2024-04-25T00:46:34.320708+0000) 2024-04-25T00:46:34.623 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:34 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:34.288+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:34.623 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:34 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:34.288+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:34.892 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:34 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:34.623+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:47.567414+0000 front 2024-04-25T00:45:47.567436+0000 (oldest deadline 2024-04-25T00:46:18.167905+0000) 2024-04-25T00:46:34.892 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:34 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:34.623+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.168480+0000 front 2024-04-25T00:45:58.168573+0000 (oldest deadline 2024-04-25T00:46:22.268508+0000) 2024-04-25T00:46:34.892 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:34 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5[44614]: 2024-04-25T00:46:34.623+0000 7fbd0c902700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.169260+0000 front 2024-04-25T00:46:05.169353+0000 (oldest deadline 2024-04-25T00:46:31.069452+0000) 2024-04-25T00:46:35.248 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:34 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:34.754+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.127034+0000 front 2024-04-25T00:45:58.127011+0000 (oldest deadline 2024-04-25T00:46:22.827000+0000) 2024-04-25T00:46:35.248 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:34 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:34.754+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.028931+0000 front 2024-04-25T00:46:08.028973+0000 (oldest deadline 2024-04-25T00:46:32.728749+0000) 2024-04-25T00:46:35.249 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:34 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:34.852+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:35.249 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:34 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:34.852+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:35.249 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:34 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:34.852+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:35.291 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:34 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:34.939+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:35.291 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:34 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:34.939+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:35.291 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:34 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:34.939+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:35.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:35 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:35.081+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:35.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:35 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:35.081+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:35.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:35 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:35.081+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:35.641 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:35 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:35.290+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:35.641 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:35 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:35.290+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:35.837 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:35 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:35.382+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:35.837 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:35 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:35.382+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:04.920336+0000 front 2024-04-25T00:46:04.920235+0000 (oldest deadline 2024-04-25T00:46:34.320708+0000) 2024-04-25T00:46:36.129 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:35 smithi130 podman[69756]: 2024-04-25 00:46:35.8349682 +0000 UTC m=+5.772583923 container died 034d093c0953e3269b9bd8f0d4caeff46450d2df4460497eac2993ac45562b78 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-d2c45d9, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-04-25T00:46:36.130 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:35 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:35.984+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:36.130 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:35 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:35.984+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:36.130 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:35 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:35.984+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:36.248 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:35 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:35.755+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.127034+0000 front 2024-04-25T00:45:58.127011+0000 (oldest deadline 2024-04-25T00:46:22.827000+0000) 2024-04-25T00:46:36.248 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:35 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:35.755+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.028931+0000 front 2024-04-25T00:46:08.028973+0000 (oldest deadline 2024-04-25T00:46:32.728749+0000) 2024-04-25T00:46:36.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:35 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:35.840+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:36.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:35 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:35.840+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:36.249 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:35 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:35.840+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:36.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:36 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:36.056+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:36.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:36 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:36.056+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:36.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:36 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:36.056+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:36.391 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:36 smithi130 podman[69756]: 2024-04-25 00:46:36.236389756 +0000 UTC m=+6.174005478 container cleanup 034d093c0953e3269b9bd8f0d4caeff46450d2df4460497eac2993ac45562b78 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-d2c45d9, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T00:46:36.392 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:36 smithi130 bash[69756]: ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5 2024-04-25T00:46:36.392 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:36 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:36.265+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:36.392 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:36 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:36.265+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:36.837 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:36 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:36.360+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:36.838 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:36 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:36.360+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:04.920336+0000 front 2024-04-25T00:46:04.920235+0000 (oldest deadline 2024-04-25T00:46:34.320708+0000) 2024-04-25T00:46:36.842 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:36 smithi130 podman[69773]: 2024-04-25 00:46:36.488045573 +0000 UTC m=+0.655058550 container remove 034d093c0953e3269b9bd8f0d4caeff46450d2df4460497eac2993ac45562b78 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-d2c45d9, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T00:46:37.142 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:36 smithi130 podman[69865]: 2024-04-25 00:46:36.746111674 +0000 UTC m=+0.018386533 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901 2024-04-25T00:46:37.142 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:36 smithi130 podman[69865]: 2024-04-25 00:46:36.847407348 +0000 UTC m=+0.119682191 container create 4ce22e69bc1a469b92f926be65142c9ac39713979c17ba3a1d1c457fcbffb1a8 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-d2c45d9, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-04-25T00:46:37.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:36 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:36.939+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:37.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:36 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:36.939+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:37.143 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:36 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:36.939+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:37.248 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:36 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:36.776+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.127034+0000 front 2024-04-25T00:45:58.127011+0000 (oldest deadline 2024-04-25T00:46:22.827000+0000) 2024-04-25T00:46:37.248 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:36 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:36.776+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.028931+0000 front 2024-04-25T00:46:08.028973+0000 (oldest deadline 2024-04-25T00:46:32.728749+0000) 2024-04-25T00:46:37.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:36 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:36.799+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:37.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:36 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:36.799+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:37.249 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:36 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:36.799+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:37.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:37 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:37.078+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:37.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:37 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:37.078+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:37.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:37 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:37.078+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:37.445 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:37 smithi130 podman[69865]: 2024-04-25 00:46:37.150114026 +0000 UTC m=+0.422388880 container init 4ce22e69bc1a469b92f926be65142c9ac39713979c17ba3a1d1c457fcbffb1a8 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5-deactivate, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-d2c45d9, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True) 2024-04-25T00:46:37.445 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:37 smithi130 podman[69865]: 2024-04-25 00:46:37.153394955 +0000 UTC m=+0.425669802 container start 4ce22e69bc1a469b92f926be65142c9ac39713979c17ba3a1d1c457fcbffb1a8 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=squid-d2c45d9, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-25T00:46:37.445 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:37 smithi130 podman[69865]: 2024-04-25 00:46:37.266946684 +0000 UTC m=+0.539221539 container attach 4ce22e69bc1a469b92f926be65142c9ac39713979c17ba3a1d1c457fcbffb1a8 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=squid-d2c45d9, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T00:46:37.446 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:37 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:37.225+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:37.446 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:37 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:37.225+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:37.761 INFO:journalctl@ceph.osd.5.smithi130.stdout:Apr 25 00:46:37 smithi130 podman[69865]: 2024-04-25 00:46:37.445604447 +0000 UTC m=+0.717879290 container died 4ce22e69bc1a469b92f926be65142c9ac39713979c17ba3a1d1c457fcbffb1a8 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, RELEASE=squid-d2c45d9, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T00:46:37.837 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:37 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:37.388+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:37.837 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:37 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:37.388+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:04.920336+0000 front 2024-04-25T00:46:04.920235+0000 (oldest deadline 2024-04-25T00:46:34.320708+0000) 2024-04-25T00:46:38.037 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:37 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:37.975+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:38.037 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:37 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:37.975+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:38.037 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:37 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:37.975+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:38.059 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.5.service' 2024-04-25T00:46:38.136 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:46:38.137 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-25T00:46:38.137 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-25T00:46:38.137 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.6 2024-04-25T00:46:38.163 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:37 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:37.789+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.127034+0000 front 2024-04-25T00:45:58.127011+0000 (oldest deadline 2024-04-25T00:46:22.827000+0000) 2024-04-25T00:46:38.164 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:37 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:37.789+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.028931+0000 front 2024-04-25T00:46:08.028973+0000 (oldest deadline 2024-04-25T00:46:32.728749+0000) 2024-04-25T00:46:38.164 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:37 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:37.777+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:38.164 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:37 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:37.777+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:38.164 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:37 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:37.777+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:38.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:38 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:38.072+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:38.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:38 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:38.072+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:38.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:38 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:38.072+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:38.391 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:38 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:38.219+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:38.391 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:38 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:38.219+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:38.440 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:38 smithi032 systemd[1]: Stopping Ceph osd.6 for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:46:38.748 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:38 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:38.439+0000 7efd3d794700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:46:38.748 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:38 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:38.439+0000 7efd3d794700 -1 osd.6 111 *** Got signal Terminated *** 2024-04-25T00:46:38.748 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:38 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:38.439+0000 7efd3d794700 -1 osd.6 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T00:46:38.837 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:38 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:38.354+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:38.837 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:38 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:38.354+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:04.920336+0000 front 2024-04-25T00:46:04.920235+0000 (oldest deadline 2024-04-25T00:46:34.320708+0000) 2024-04-25T00:46:39.248 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:38 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:38.816+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.127034+0000 front 2024-04-25T00:45:58.127011+0000 (oldest deadline 2024-04-25T00:46:22.827000+0000) 2024-04-25T00:46:39.248 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:38 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:38.816+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.028931+0000 front 2024-04-25T00:46:08.028973+0000 (oldest deadline 2024-04-25T00:46:32.728749+0000) 2024-04-25T00:46:39.249 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:38 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:38.799+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:39.249 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:38 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:38.799+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:39.249 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:38 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:38.799+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:39.392 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:39 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:39.233+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:39.392 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:39 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:39.233+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:39.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:39 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:39.012+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:39.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:39 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:39.012+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:39.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:39 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:39.012+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:39.587 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:39 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:39.111+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:39.587 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:39 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:39.111+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:39.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:39 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:39.111+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:39.588 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:39 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:39.318+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:39.588 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:39 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:39.318+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:04.920336+0000 front 2024-04-25T00:46:04.920235+0000 (oldest deadline 2024-04-25T00:46:34.320708+0000) 2024-04-25T00:46:40.248 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:39 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:39.777+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.127034+0000 front 2024-04-25T00:45:58.127011+0000 (oldest deadline 2024-04-25T00:46:22.827000+0000) 2024-04-25T00:46:40.249 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:39 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:39.777+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.028931+0000 front 2024-04-25T00:46:08.028973+0000 (oldest deadline 2024-04-25T00:46:32.728749+0000) 2024-04-25T00:46:40.249 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:39 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:39.826+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:40.249 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:39 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:39.826+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:40.249 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:39 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:39.826+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:40.391 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:40 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:40.188+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:40.391 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:40 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:40.188+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:40.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:39 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:39.994+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:40.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:39 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:39.994+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:40.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:39 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:39.994+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:40.587 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:40 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:40.292+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:40.587 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:40 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:40.292+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:04.920336+0000 front 2024-04-25T00:46:04.920235+0000 (oldest deadline 2024-04-25T00:46:34.320708+0000) 2024-04-25T00:46:40.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:40 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:40.145+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:40.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:40 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:40.145+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:40.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:40 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:40.145+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:41.248 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:40 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:40.801+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.127034+0000 front 2024-04-25T00:45:58.127011+0000 (oldest deadline 2024-04-25T00:46:22.827000+0000) 2024-04-25T00:46:41.248 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:40 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:40.801+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.028931+0000 front 2024-04-25T00:46:08.028973+0000 (oldest deadline 2024-04-25T00:46:32.728749+0000) 2024-04-25T00:46:41.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:40 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:40.803+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:41.249 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:40 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:40.803+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:41.249 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:40 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:40.803+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:41.249 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:40 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:40.803+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:46:41.391 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:41 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:41.190+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:41.392 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:41 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:41.190+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:41.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:40 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:40.985+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:41.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:40 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:40.985+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:41.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:40 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:40.985+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:41.587 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:41 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:41.268+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:41.587 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:41 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:41.268+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:04.920336+0000 front 2024-04-25T00:46:04.920235+0000 (oldest deadline 2024-04-25T00:46:34.320708+0000) 2024-04-25T00:46:41.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:41 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:41.168+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:41.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:41 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:41.168+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:41.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:41 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:41.168+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:41.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:41 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:41.168+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:46:42.250 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:41 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:41.790+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.127034+0000 front 2024-04-25T00:45:58.127011+0000 (oldest deadline 2024-04-25T00:46:22.827000+0000) 2024-04-25T00:46:42.250 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:41 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:41.790+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.028931+0000 front 2024-04-25T00:46:08.028973+0000 (oldest deadline 2024-04-25T00:46:32.728749+0000) 2024-04-25T00:46:42.250 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:41 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:41.775+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:42.250 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:41 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:41.775+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:42.251 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:41 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:41.775+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:42.251 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:41 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:41.775+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:46:42.392 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:42 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:42.185+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:42.392 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:42 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:42.185+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:42.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:41 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:41.945+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:42.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:41 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:41.945+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:42.393 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:41 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:41.945+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:42.587 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:42 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:42.224+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:42.588 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:42 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:42.224+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:04.920336+0000 front 2024-04-25T00:46:04.920235+0000 (oldest deadline 2024-04-25T00:46:34.320708+0000) 2024-04-25T00:46:42.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:42 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:42.123+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:42.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:42 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:42.123+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:42.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:42 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:42.123+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:42.589 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:42 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:42.123+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:46:43.149 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:42 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:42.814+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.127034+0000 front 2024-04-25T00:45:58.127011+0000 (oldest deadline 2024-04-25T00:46:22.827000+0000) 2024-04-25T00:46:43.149 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:42 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6[57784]: 2024-04-25T00:46:42.814+0000 7efd35d80700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.028931+0000 front 2024-04-25T00:46:08.028973+0000 (oldest deadline 2024-04-25T00:46:32.728749+0000) 2024-04-25T00:46:43.150 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:42 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:42.765+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:43.150 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:42 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:42.765+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:43.150 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:42 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:42.765+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:43.150 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:42 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:42.765+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:46:43.207 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:42 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:42.896+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:43.207 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:42 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:42.896+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:43.207 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:42 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:42.896+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:43.587 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:43 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:43.198+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:43.588 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:43 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:43.198+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:04.920336+0000 front 2024-04-25T00:46:04.920235+0000 (oldest deadline 2024-04-25T00:46:34.320708+0000) 2024-04-25T00:46:43.588 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:43 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:43.198+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:17.722394+0000 front 2024-04-25T00:46:17.722356+0000 (oldest deadline 2024-04-25T00:46:42.422440+0000) 2024-04-25T00:46:43.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:43 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:43.114+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:43.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:43 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:43.114+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:43.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:43 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:43.114+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:43.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:43 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:43.114+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:46:43.641 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:43 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:43.206+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:43.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:43 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:43.206+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:43.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:43 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:43.746+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:43.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:43 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:43.746+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:43.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:43 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:43.746+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:43.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:43 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:43.746+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:46:44.141 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:43 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:43.867+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:44.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:43 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:43.867+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:44.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:43 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:43.867+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:44.337 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:44 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:44.218+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:44.338 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:44 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:44.218+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:04.920336+0000 front 2024-04-25T00:46:04.920235+0000 (oldest deadline 2024-04-25T00:46:34.320708+0000) 2024-04-25T00:46:44.338 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:44 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:44.218+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:17.722394+0000 front 2024-04-25T00:46:17.722356+0000 (oldest deadline 2024-04-25T00:46:42.422440+0000) 2024-04-25T00:46:44.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:44 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:44.077+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:44.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:44 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:44.077+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:44.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:44 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:44.077+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:44.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:44 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:44.077+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:46:44.434 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:44 smithi032 podman[94050]: 2024-04-25 00:46:44.201096789 +0000 UTC m=+5.867384409 container died 80bb66923df24f63c431dabf9fd2d4c4755a3d822cef8ef44b6d83f493f6bb68 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=squid-d2c45d9, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-25T00:46:44.641 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:44 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:44.223+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:44.641 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:44 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:44.223+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:44.748 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:44 smithi032 podman[94050]: 2024-04-25 00:46:44.565483479 +0000 UTC m=+6.231771107 container cleanup 80bb66923df24f63c431dabf9fd2d4c4755a3d822cef8ef44b6d83f493f6bb68 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-d2c45d9, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS) 2024-04-25T00:46:44.748 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:44 smithi032 bash[94050]: ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6 2024-04-25T00:46:45.141 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:44 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:44.874+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:45.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:44 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:44.874+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:45.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:44 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:44.874+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:45.206 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:44 smithi032 podman[94069]: 2024-04-25 00:46:44.858463559 +0000 UTC m=+0.651976129 container remove 80bb66923df24f63c431dabf9fd2d4c4755a3d822cef8ef44b6d83f493f6bb68 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-d2c45d9, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS) 2024-04-25T00:46:45.207 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:44 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:44.793+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:45.207 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:44 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:44.793+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:45.207 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:44 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:44.793+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:45.207 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:44 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:44.793+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:46:45.498 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:45 smithi032 podman[94168]: 2024-04-25 00:46:45.110210636 +0000 UTC m=+0.018717924 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901 2024-04-25T00:46:45.498 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:45 smithi032 podman[94168]: 2024-04-25 00:46:45.259398436 +0000 UTC m=+0.167905698 container create 92cfae3efec6c07172982104cb954242af1d4853d35511d902b0a6618e026003 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-d2c45d9, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T00:46:45.587 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:45 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:45.196+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:45.588 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:45 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:45.196+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:04.920336+0000 front 2024-04-25T00:46:04.920235+0000 (oldest deadline 2024-04-25T00:46:34.320708+0000) 2024-04-25T00:46:45.588 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:45 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:45.196+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:17.722394+0000 front 2024-04-25T00:46:17.722356+0000 (oldest deadline 2024-04-25T00:46:42.422440+0000) 2024-04-25T00:46:45.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:45 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:45.104+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:45.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:45 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:45.104+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:45.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:45 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:45.104+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:45.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:45 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:45.104+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:46:45.641 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:45 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:45.252+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:45.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:45 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:45.252+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:45.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:45 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:45.252+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:17.644799+0000 front 2024-04-25T00:46:17.644829+0000 (oldest deadline 2024-04-25T00:46:44.645374+0000) 2024-04-25T00:46:45.834 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:45 smithi032 podman[94168]: 2024-04-25 00:46:45.520537671 +0000 UTC m=+0.429044932 container init 92cfae3efec6c07172982104cb954242af1d4853d35511d902b0a6618e026003 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-d2c45d9, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T00:46:45.834 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:45 smithi032 podman[94168]: 2024-04-25 00:46:45.524208281 +0000 UTC m=+0.432715543 container start 92cfae3efec6c07172982104cb954242af1d4853d35511d902b0a6618e026003 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-d2c45d9, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424) 2024-04-25T00:46:45.834 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:45 smithi032 podman[94168]: 2024-04-25 00:46:45.612263007 +0000 UTC m=+0.520770283 container attach 92cfae3efec6c07172982104cb954242af1d4853d35511d902b0a6618e026003 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-d2c45d9, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-25T00:46:45.835 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:45 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:45.750+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:45.835 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:45 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:45.750+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:45.835 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:45 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:45.750+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:45.835 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:45 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:45.750+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:46:46.134 INFO:journalctl@ceph.osd.6.smithi032.stdout:Apr 25 00:46:45 smithi032 podman[94168]: 2024-04-25 00:46:45.834455563 +0000 UTC m=+0.742962827 container died 92cfae3efec6c07172982104cb954242af1d4853d35511d902b0a6618e026003 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=squid-d2c45d9, org.label-schema.build-date=20240424, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True) 2024-04-25T00:46:46.141 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:45 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:45.876+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:46.141 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:45 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:45.876+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:46.141 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:45 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:45.876+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:46.462 DEBUG:teuthology.orchestra.run.smithi032:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.6.service' 2024-04-25T00:46:46.547 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:46:46.547 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-25T00:46:46.547 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-25T00:46:46.547 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.7 2024-04-25T00:46:46.576 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:46 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:46.180+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:46.576 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:46 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:46.180+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:04.920336+0000 front 2024-04-25T00:46:04.920235+0000 (oldest deadline 2024-04-25T00:46:34.320708+0000) 2024-04-25T00:46:46.576 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:46 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:46.180+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:17.722394+0000 front 2024-04-25T00:46:17.722356+0000 (oldest deadline 2024-04-25T00:46:42.422440+0000) 2024-04-25T00:46:46.576 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:46 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:46.105+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:46.576 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:46 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:46.105+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:46.576 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:46 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:46.105+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:46.576 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:46 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:46.105+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:46:46.641 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:46 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:46.268+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:46.641 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:46 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:46.268+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:46.641 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:46 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:46.268+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:17.644799+0000 front 2024-04-25T00:46:17.644829+0000 (oldest deadline 2024-04-25T00:46:44.645374+0000) 2024-04-25T00:46:46.837 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:46 smithi121 systemd[1]: Stopping Ceph osd.7 for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:46:47.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:46 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:46.780+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:47.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:46 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:46.780+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:47.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:46 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:46.780+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:47.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:46 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:46.780+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:46:47.257 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:46 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:46.903+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:47.257 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:46 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:46.903+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:47.257 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:46 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:46.903+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:47.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:47 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:47.083+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:47.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:47 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:47.083+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:47.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:47 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:47.083+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:47.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:47 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:47.083+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:46:47.338 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:46 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:46.919+0000 7f0c390ac700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:46:47.338 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:46 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:46.919+0000 7f0c390ac700 -1 osd.7 111 *** Got signal Terminated *** 2024-04-25T00:46:47.338 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:46 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:46.919+0000 7f0c390ac700 -1 osd.7 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T00:46:47.338 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:47 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:47.163+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:47.338 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:47 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:47.163+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:04.920336+0000 front 2024-04-25T00:46:04.920235+0000 (oldest deadline 2024-04-25T00:46:34.320708+0000) 2024-04-25T00:46:47.338 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:47 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:47.163+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:17.722394+0000 front 2024-04-25T00:46:17.722356+0000 (oldest deadline 2024-04-25T00:46:42.422440+0000) 2024-04-25T00:46:47.641 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:47 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:47.256+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:47.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:47 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:47.257+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:47.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:47 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:47.257+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:17.644799+0000 front 2024-04-25T00:46:17.644829+0000 (oldest deadline 2024-04-25T00:46:44.645374+0000) 2024-04-25T00:46:47.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:47 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:47.736+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:47.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:47 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:47.736+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:47.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:47 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:47.736+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:47.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:47 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:47.736+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:46:48.224 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:47 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:47.935+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:48.224 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:47 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:47.935+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:48.225 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:47 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:47.935+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:48.587 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:48 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:48.199+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:48.587 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:48 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:48.199+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:04.920336+0000 front 2024-04-25T00:46:04.920235+0000 (oldest deadline 2024-04-25T00:46:34.320708+0000) 2024-04-25T00:46:48.588 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:48 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:48.199+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:17.722394+0000 front 2024-04-25T00:46:17.722356+0000 (oldest deadline 2024-04-25T00:46:42.422440+0000) 2024-04-25T00:46:48.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:48 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:48.129+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:48.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:48 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:48.129+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:48.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:48 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:48.129+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:48.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:48 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:48.129+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:46:48.641 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:48 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:48.224+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:48.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:48 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:48.224+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:48.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:48 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:48.224+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:17.644799+0000 front 2024-04-25T00:46:17.644829+0000 (oldest deadline 2024-04-25T00:46:44.645374+0000) 2024-04-25T00:46:49.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:48 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:48.754+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:49.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:48 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:48.754+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:49.249 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:48 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:48.754+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:49.249 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:48 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:48.754+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:46:49.253 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:48 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:48.913+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:49.253 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:48 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:48.913+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:49.253 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:48 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:48.913+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:49.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:49 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:49.079+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:49.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:49 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:49.079+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:49.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:49 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:49.079+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:49.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:49 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:49.079+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:46:49.338 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:49 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:49.235+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:49.338 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:49 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:49.235+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:04.920336+0000 front 2024-04-25T00:46:04.920235+0000 (oldest deadline 2024-04-25T00:46:34.320708+0000) 2024-04-25T00:46:49.338 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:49 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:49.235+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:17.722394+0000 front 2024-04-25T00:46:17.722356+0000 (oldest deadline 2024-04-25T00:46:42.422440+0000) 2024-04-25T00:46:49.338 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:49 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:49.235+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:25.323285+0000 front 2024-04-25T00:46:25.323359+0000 (oldest deadline 2024-04-25T00:46:48.823419+0000) 2024-04-25T00:46:49.641 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:49 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:49.253+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:49.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:49 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:49.253+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:49.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:49 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:49.253+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:17.644799+0000 front 2024-04-25T00:46:17.644829+0000 (oldest deadline 2024-04-25T00:46:44.645374+0000) 2024-04-25T00:46:49.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:49 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:49.731+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:49.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:49 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:49.731+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:49.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:49 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:49.731+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:49.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:49 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:49.731+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:46:49.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:49 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:49.731+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:23.922801+0000 front 2024-04-25T00:46:23.922846+0000 (oldest deadline 2024-04-25T00:46:49.222546+0000) 2024-04-25T00:46:50.298 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:49 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:49.916+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:50.298 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:49 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:49.916+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:50.299 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:49 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:49.916+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:50.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:50 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:50.084+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:50.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:50 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:50.084+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:50.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:50 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:50.084+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:50.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:50 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:50.084+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:46:50.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:50 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:50.084+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:46:50.338 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:50 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:50.217+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:50.338 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:50 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:50.217+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:04.920336+0000 front 2024-04-25T00:46:04.920235+0000 (oldest deadline 2024-04-25T00:46:34.320708+0000) 2024-04-25T00:46:50.338 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:50 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:50.217+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:17.722394+0000 front 2024-04-25T00:46:17.722356+0000 (oldest deadline 2024-04-25T00:46:42.422440+0000) 2024-04-25T00:46:50.339 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:50 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:50.217+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:25.323285+0000 front 2024-04-25T00:46:25.323359+0000 (oldest deadline 2024-04-25T00:46:48.823419+0000) 2024-04-25T00:46:50.641 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:50 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:50.297+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:50.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:50 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:50.297+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:50.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:50 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:50.297+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:17.644799+0000 front 2024-04-25T00:46:17.644829+0000 (oldest deadline 2024-04-25T00:46:44.645374+0000) 2024-04-25T00:46:50.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:50 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:50.298+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.646192+0000 front 2024-04-25T00:46:24.646076+0000 (oldest deadline 2024-04-25T00:46:49.846530+0000) 2024-04-25T00:46:51.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:50 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:50.762+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:51.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:50 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:50.762+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:51.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:50 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:50.762+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:51.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:50 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:50.762+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:46:51.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:50 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:50.762+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:23.922801+0000 front 2024-04-25T00:46:23.922846+0000 (oldest deadline 2024-04-25T00:46:49.222546+0000) 2024-04-25T00:46:51.340 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:50 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:50.909+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:51.340 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:50 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:50.909+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:51.340 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:50 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:50.909+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:51.340 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:50 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:50.909+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:46:51.587 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:51 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:51.207+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.819466+0000 front 2024-04-25T00:46:00.819498+0000 (oldest deadline 2024-04-25T00:46:24.919645+0000) 2024-04-25T00:46:51.587 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:51 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:51.207+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:04.920336+0000 front 2024-04-25T00:46:04.920235+0000 (oldest deadline 2024-04-25T00:46:34.320708+0000) 2024-04-25T00:46:51.588 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:51 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:51.207+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:17.722394+0000 front 2024-04-25T00:46:17.722356+0000 (oldest deadline 2024-04-25T00:46:42.422440+0000) 2024-04-25T00:46:51.588 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:51 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7[49836]: 2024-04-25T00:46:51.207+0000 7f0c31698700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:25.323285+0000 front 2024-04-25T00:46:25.323359+0000 (oldest deadline 2024-04-25T00:46:48.823419+0000) 2024-04-25T00:46:51.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:51 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:51.118+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:51.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:51 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:51.118+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:51.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:51 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:51.118+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:51.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:51 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:51.118+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:46:51.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:51 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:51.118+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:46:51.641 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:51 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:51.339+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:51.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:51 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:51.339+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:51.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:51 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:51.339+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:17.644799+0000 front 2024-04-25T00:46:17.644829+0000 (oldest deadline 2024-04-25T00:46:44.645374+0000) 2024-04-25T00:46:51.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:51 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:51.339+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.646192+0000 front 2024-04-25T00:46:24.646076+0000 (oldest deadline 2024-04-25T00:46:49.846530+0000) 2024-04-25T00:46:51.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:51 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:51.726+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:51.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:51 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:51.726+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:51.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:51 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:51.726+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:51.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:51 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:51.726+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:46:51.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:51 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:51.726+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:23.922801+0000 front 2024-04-25T00:46:23.922846+0000 (oldest deadline 2024-04-25T00:46:49.222546+0000) 2024-04-25T00:46:52.141 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:51 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:51.888+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:52.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:51 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:51.888+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:52.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:51 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:51.888+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:52.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:51 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:51.888+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:46:52.587 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:52 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:52.116+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:52.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:52 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:52.116+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:52.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:52 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:52.116+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:52.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:52 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:52.116+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:46:52.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:52 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:52.116+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:46:52.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:52 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:52.371+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:52.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:52 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:52.371+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:52.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:52 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:52.371+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:17.644799+0000 front 2024-04-25T00:46:17.644829+0000 (oldest deadline 2024-04-25T00:46:44.645374+0000) 2024-04-25T00:46:52.643 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:52 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:52.371+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.646192+0000 front 2024-04-25T00:46:24.646076+0000 (oldest deadline 2024-04-25T00:46:49.846530+0000) 2024-04-25T00:46:52.919 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:52 smithi121 podman[68892]: 2024-04-25 00:46:52.600286758 +0000 UTC m=+5.850379815 container died 1f5a12fc1a3c375fb143c838f058a72164fd2679606c85f9b07802784d4640e5 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, RELEASE=squid-d2c45d9, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-04-25T00:46:53.141 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:52 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:52.841+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:53.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:52 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:52.841+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:53.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:52 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:52.841+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:53.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:52 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:52.841+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:46:53.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:52 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:52.751+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:53.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:52 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:52.751+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:53.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:52 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:52.751+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:53.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:52 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:52.751+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:46:53.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:52 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:52.751+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:23.922801+0000 front 2024-04-25T00:46:23.922846+0000 (oldest deadline 2024-04-25T00:46:49.222546+0000) 2024-04-25T00:46:53.337 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:52 smithi121 podman[68892]: 2024-04-25 00:46:52.919194163 +0000 UTC m=+6.169287220 container cleanup 1f5a12fc1a3c375fb143c838f058a72164fd2679606c85f9b07802784d4640e5 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-d2c45d9, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-25T00:46:53.337 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:52 smithi121 bash[68892]: ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7 2024-04-25T00:46:53.337 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:53 smithi121 podman[68909]: 2024-04-25 00:46:53.153892736 +0000 UTC m=+0.549908348 container remove 1f5a12fc1a3c375fb143c838f058a72164fd2679606c85f9b07802784d4640e5 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7, org.label-schema.build-date=20240424, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-d2c45d9, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2) 2024-04-25T00:46:53.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:53 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:53.069+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:53.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:53 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:53.069+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:53.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:53 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:53.069+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:53.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:53 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:53.069+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:46:53.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:53 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:53.069+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:46:53.641 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:53 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:53.325+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:53.641 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:53 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:53.325+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:53.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:53 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:53.325+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:17.644799+0000 front 2024-04-25T00:46:17.644829+0000 (oldest deadline 2024-04-25T00:46:44.645374+0000) 2024-04-25T00:46:53.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:53 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:53.325+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.646192+0000 front 2024-04-25T00:46:24.646076+0000 (oldest deadline 2024-04-25T00:46:49.846530+0000) 2024-04-25T00:46:53.832 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:53 smithi121 podman[68999]: 2024-04-25 00:46:53.480079257 +0000 UTC m=+0.017938489 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901 2024-04-25T00:46:53.832 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:53 smithi121 podman[68999]: 2024-04-25 00:46:53.588051725 +0000 UTC m=+0.125910943 container create 3158b646f7eddccb30c2a074acd5cb6778f9fc624a758b63e8b991e382d087c6 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-d2c45d9, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-25T00:46:53.833 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:53 smithi121 podman[68999]: 2024-04-25 00:46:53.832637577 +0000 UTC m=+0.370496794 container init 3158b646f7eddccb30c2a074acd5cb6778f9fc624a758b63e8b991e382d087c6 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, RELEASE=squid-d2c45d9, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T00:46:54.087 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:53 smithi121 podman[68999]: 2024-04-25 00:46:53.83659709 +0000 UTC m=+0.374456310 container start 3158b646f7eddccb30c2a074acd5cb6778f9fc624a758b63e8b991e382d087c6 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7-deactivate, RELEASE=squid-d2c45d9, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, io.buildah.version=1.33.5) 2024-04-25T00:46:54.088 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:53 smithi121 podman[68999]: 2024-04-25 00:46:53.941143875 +0000 UTC m=+0.479003114 container attach 3158b646f7eddccb30c2a074acd5cb6778f9fc624a758b63e8b991e382d087c6 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7-deactivate, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-d2c45d9, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-25T00:46:54.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:54 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:54.064+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:54.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:54 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:54.064+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:54.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:54 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:54.064+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:54.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:54 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:54.064+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:46:54.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:54 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:54.064+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:46:54.141 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:53 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:53.867+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:54.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:53 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:53.867+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:54.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:53 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:53.867+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:54.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:53 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:53.867+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:46:54.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:53 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:53.776+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:54.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:53 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:53.776+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:54.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:53 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:53.776+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:54.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:53 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:53.776+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:46:54.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:53 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:53.776+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:23.922801+0000 front 2024-04-25T00:46:23.922846+0000 (oldest deadline 2024-04-25T00:46:49.222546+0000) 2024-04-25T00:46:54.396 INFO:journalctl@ceph.osd.7.smithi121.stdout:Apr 25 00:46:54 smithi121 podman[68999]: 2024-04-25 00:46:54.121368784 +0000 UTC m=+0.659228003 container died 3158b646f7eddccb30c2a074acd5cb6778f9fc624a758b63e8b991e382d087c6 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-d2c45d9, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-25T00:46:54.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:54 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:54.302+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:54.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:54 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:54.302+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:54.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:54 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:54.302+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:17.644799+0000 front 2024-04-25T00:46:17.644829+0000 (oldest deadline 2024-04-25T00:46:44.645374+0000) 2024-04-25T00:46:54.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:54 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:54.302+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.646192+0000 front 2024-04-25T00:46:24.646076+0000 (oldest deadline 2024-04-25T00:46:49.846530+0000) 2024-04-25T00:46:54.688 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.7.service' 2024-04-25T00:46:54.765 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:46:54.765 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-25T00:46:54.765 INFO:tasks.cephadm.osd.8:Stopping osd.8... 2024-04-25T00:46:54.765 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.8 2024-04-25T00:46:55.090 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:54 smithi130 systemd[1]: Stopping Ceph osd.8 for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:46:55.091 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:55 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:55.090+0000 7f487cd7b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.8 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:46:55.091 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:54 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:54.840+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:55.091 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:54 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:54.840+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:55.091 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:54 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:54.840+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:55.091 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:54 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:54.840+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:46:55.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:54 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:54.780+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:55.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:54 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:54.780+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:55.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:54 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:54.780+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:55.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:54 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:54.780+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:46:55.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:54 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:54.780+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:23.922801+0000 front 2024-04-25T00:46:23.922846+0000 (oldest deadline 2024-04-25T00:46:49.222546+0000) 2024-04-25T00:46:55.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:55 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:55.062+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:55.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:55 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:55.062+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:55.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:55 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:55.062+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:55.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:55 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:55.062+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:46:55.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:55 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:55.062+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:46:55.392 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:55 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:55.090+0000 7f487cd7b700 -1 osd.8 111 *** Got signal Terminated *** 2024-04-25T00:46:55.392 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:55 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:55.090+0000 7f487cd7b700 -1 osd.8 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T00:46:55.392 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:55 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:55.279+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:55.392 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:55 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:55.279+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:55.392 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:55 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:55.279+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:17.644799+0000 front 2024-04-25T00:46:17.644829+0000 (oldest deadline 2024-04-25T00:46:44.645374+0000) 2024-04-25T00:46:55.392 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:55 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:55.279+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.646192+0000 front 2024-04-25T00:46:24.646076+0000 (oldest deadline 2024-04-25T00:46:49.846530+0000) 2024-04-25T00:46:56.141 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:55 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:55.878+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:56.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:55 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:55.878+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:56.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:55 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:55.878+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:56.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:55 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:55.878+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:46:56.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:55 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:55.804+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:56.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:55 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:55.804+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:56.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:55 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:55.804+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:56.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:55 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:55.804+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:46:56.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:55 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:55.804+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:23.922801+0000 front 2024-04-25T00:46:23.922846+0000 (oldest deadline 2024-04-25T00:46:49.222546+0000) 2024-04-25T00:46:56.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:56 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:56.084+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:56.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:56 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:56.084+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:56.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:56 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:56.084+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:56.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:56 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:56.084+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:46:56.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:56 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:56.084+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:46:56.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:56 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:56.250+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:56.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:56 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:56.250+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:56.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:56 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:56.250+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:17.644799+0000 front 2024-04-25T00:46:17.644829+0000 (oldest deadline 2024-04-25T00:46:44.645374+0000) 2024-04-25T00:46:56.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:56 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:56.250+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.646192+0000 front 2024-04-25T00:46:24.646076+0000 (oldest deadline 2024-04-25T00:46:49.846530+0000) 2024-04-25T00:46:57.247 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:56 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:56.901+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:57.247 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:56 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:56.902+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:57.247 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:56 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:56.902+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:57.248 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:56 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:56.902+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:46:57.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:56 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:56.811+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:57.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:56 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:56.811+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:57.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:56 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:56.811+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:57.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:56 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:56.811+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:46:57.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:56 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:56.811+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:23.922801+0000 front 2024-04-25T00:46:23.922846+0000 (oldest deadline 2024-04-25T00:46:49.222546+0000) 2024-04-25T00:46:57.587 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:57 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:57.098+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:57.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:57 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:57.098+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:57.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:57 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:57.098+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:57.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:57 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:57.098+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:46:57.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:57 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:57.098+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:46:57.588 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:57 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:57.098+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:34.658309+0000 front 2024-04-25T00:46:34.658326+0000 (oldest deadline 2024-04-25T00:46:56.358119+0000) 2024-04-25T00:46:57.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:57 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:57.247+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:57.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:57 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:57.247+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:57.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:57 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:57.247+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:17.644799+0000 front 2024-04-25T00:46:17.644829+0000 (oldest deadline 2024-04-25T00:46:44.645374+0000) 2024-04-25T00:46:57.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:57 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:57.247+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.646192+0000 front 2024-04-25T00:46:24.646076+0000 (oldest deadline 2024-04-25T00:46:49.846530+0000) 2024-04-25T00:46:58.229 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:57 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:57.922+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:58.229 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:57 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:57.922+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:58.229 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:57 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:57.922+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:58.229 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:57 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:57.922+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:46:58.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:57 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:57.796+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:58.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:57 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:57.796+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:58.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:57 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:57.796+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:58.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:57 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:57.796+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:46:58.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:57 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:57.796+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:23.922801+0000 front 2024-04-25T00:46:23.922846+0000 (oldest deadline 2024-04-25T00:46:49.222546+0000) 2024-04-25T00:46:58.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:58 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:58.060+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:58.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:58 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:58.060+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:58.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:58 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:58.060+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:58.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:58 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:58.060+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:46:58.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:58 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:58.060+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:46:58.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:58 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:58.060+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:34.658309+0000 front 2024-04-25T00:46:34.658326+0000 (oldest deadline 2024-04-25T00:46:56.358119+0000) 2024-04-25T00:46:58.641 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:58 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:58.228+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:58.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:58 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:58.228+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:58.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:58 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:58.228+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:17.644799+0000 front 2024-04-25T00:46:17.644829+0000 (oldest deadline 2024-04-25T00:46:44.645374+0000) 2024-04-25T00:46:58.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:58 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:58.228+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.646192+0000 front 2024-04-25T00:46:24.646076+0000 (oldest deadline 2024-04-25T00:46:49.846530+0000) 2024-04-25T00:46:59.141 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:58 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:58.890+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:46:59.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:58 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:58.890+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:46:59.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:58 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:58.890+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:46:59.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:58 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:58.890+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:46:59.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:58 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:58.767+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:59.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:58 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:58.767+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:59.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:58 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:58.767+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:59.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:58 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:58.767+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:46:59.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:58 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:58.767+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:23.922801+0000 front 2024-04-25T00:46:23.922846+0000 (oldest deadline 2024-04-25T00:46:49.222546+0000) 2024-04-25T00:46:59.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:59 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:59.012+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:46:59.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:59 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:59.012+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:46:59.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:59 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:59.012+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:46:59.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:59 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:59.012+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:46:59.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:59 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:59.012+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:46:59.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:46:59 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:46:59.012+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:34.658309+0000 front 2024-04-25T00:46:34.658326+0000 (oldest deadline 2024-04-25T00:46:56.358119+0000) 2024-04-25T00:46:59.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:59 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:59.265+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.842160+0000 front 2024-04-25T00:46:00.842296+0000 (oldest deadline 2024-04-25T00:46:24.942145+0000) 2024-04-25T00:46:59.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:59 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:59.265+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:06.642887+0000 front 2024-04-25T00:46:06.642981+0000 (oldest deadline 2024-04-25T00:46:32.443486+0000) 2024-04-25T00:46:59.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:59 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:59.265+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:17.644799+0000 front 2024-04-25T00:46:17.644829+0000 (oldest deadline 2024-04-25T00:46:44.645374+0000) 2024-04-25T00:46:59.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:59 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:59.265+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.646192+0000 front 2024-04-25T00:46:24.646076+0000 (oldest deadline 2024-04-25T00:46:49.846530+0000) 2024-04-25T00:46:59.642 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:46:59 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8[51307]: 2024-04-25T00:46:59.265+0000 7f4875367700 -1 osd.8 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:34.948395+0000 front 2024-04-25T00:46:34.948433+0000 (oldest deadline 2024-04-25T00:46:59.048573+0000) 2024-04-25T00:46:59.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:59 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:59.745+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:46:59.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:59 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:59.745+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:46:59.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:59 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:59.745+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:46:59.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:59 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:59.745+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:46:59.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:46:59 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:46:59.745+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:23.922801+0000 front 2024-04-25T00:46:23.922846+0000 (oldest deadline 2024-04-25T00:46:49.222546+0000) 2024-04-25T00:47:00.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:00 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:00.034+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:47:00.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:00 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:00.034+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:47:00.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:00 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:00.034+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:47:00.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:00 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:00.034+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:47:00.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:00 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:00.034+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:47:00.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:00 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:00.034+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:34.658309+0000 front 2024-04-25T00:46:34.658326+0000 (oldest deadline 2024-04-25T00:46:56.358119+0000) 2024-04-25T00:47:00.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:59 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:59.935+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:00.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:59 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:59.935+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:00.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:59 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:59.935+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:00.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:46:59 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:46:59.935+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:00.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:00 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:00.708+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:47:00.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:00 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:00.708+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:47:00.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:00 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:00.708+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:47:00.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:00 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:00.708+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:47:00.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:00 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:00.708+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:23.922801+0000 front 2024-04-25T00:46:23.922846+0000 (oldest deadline 2024-04-25T00:46:49.222546+0000) 2024-04-25T00:47:01.008 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:47:00 smithi130 podman[69972]: 2024-04-25 00:47:00.709354523 +0000 UTC m=+5.749585926 container died a4bbcd14f447c13294296e61c0c1c29ca783bd298e9c1024b613acd5ce24de10 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-d2c45d9, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2) 2024-04-25T00:47:01.008 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:00 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:00.897+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:01.008 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:00 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:00.897+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:01.008 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:00 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:00.897+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:01.008 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:00 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:00.897+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:01.008 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:00 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:00.897+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.836754+0000 front 2024-04-25T00:46:30.836803+0000 (oldest deadline 2024-04-25T00:47:00.237230+0000) 2024-04-25T00:47:01.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:01 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:01.002+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:47:01.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:01 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:01.002+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:47:01.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:01 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:01.002+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:47:01.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:01 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:01.002+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:47:01.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:01 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:01.002+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:47:01.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:01 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:01.002+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:34.658309+0000 front 2024-04-25T00:46:34.658326+0000 (oldest deadline 2024-04-25T00:46:56.358119+0000) 2024-04-25T00:47:01.341 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:47:01 smithi130 podman[69972]: 2024-04-25 00:47:01.107048498 +0000 UTC m=+6.147279910 container cleanup a4bbcd14f447c13294296e61c0c1c29ca783bd298e9c1024b613acd5ce24de10 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=squid-d2c45d9, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux ) 2024-04-25T00:47:01.341 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:47:01 smithi130 bash[69972]: ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8 2024-04-25T00:47:01.641 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:47:01 smithi130 podman[69991]: 2024-04-25 00:47:01.341407763 +0000 UTC m=+0.628218944 container remove a4bbcd14f447c13294296e61c0c1c29ca783bd298e9c1024b613acd5ce24de10 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, RELEASE=squid-d2c45d9, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-04-25T00:47:01.995 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:47:01 smithi130 podman[70080]: 2024-04-25 00:47:01.569617197 +0000 UTC m=+0.017983598 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901 2024-04-25T00:47:01.996 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:47:01 smithi130 podman[70080]: 2024-04-25 00:47:01.701150079 +0000 UTC m=+0.149516506 container create 3edf091d10bc5563bce0534e710d2f67ef9a489c4e5c14ec23b2d7491e2aea7c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8-deactivate, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240424, RELEASE=squid-d2c45d9, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-25T00:47:01.996 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:47:01 smithi130 podman[70080]: 2024-04-25 00:47:01.895532144 +0000 UTC m=+0.343898529 container init 3edf091d10bc5563bce0534e710d2f67ef9a489c4e5c14ec23b2d7491e2aea7c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240424, RELEASE=squid-d2c45d9, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-04-25T00:47:01.996 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:47:01 smithi130 podman[70080]: 2024-04-25 00:47:01.898576466 +0000 UTC m=+0.346942854 container start 3edf091d10bc5563bce0534e710d2f67ef9a489c4e5c14ec23b2d7491e2aea7c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8-deactivate, org.label-schema.build-date=20240424, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-d2c45d9, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T00:47:01.996 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:47:01 smithi130 podman[70080]: 2024-04-25 00:47:01.995469032 +0000 UTC m=+0.443835424 container attach 3edf091d10bc5563bce0534e710d2f67ef9a489c4e5c14ec23b2d7491e2aea7c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, RELEASE=squid-d2c45d9, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T00:47:01.996 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:01 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:01.913+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:01.996 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:01 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:01.913+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:01.997 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:01 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:01.913+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:01.997 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:01 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:01.913+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:01.997 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:01 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:01.913+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.836754+0000 front 2024-04-25T00:46:30.836803+0000 (oldest deadline 2024-04-25T00:47:00.237230+0000) 2024-04-25T00:47:01.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:01 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:01.684+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:47:01.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:01 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:01.684+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:47:01.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:01 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:01.684+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:47:01.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:01 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:01.684+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:47:01.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:01 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:01.684+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:23.922801+0000 front 2024-04-25T00:46:23.922846+0000 (oldest deadline 2024-04-25T00:46:49.222546+0000) 2024-04-25T00:47:01.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:01 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:01.684+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.923692+0000 front 2024-04-25T00:46:30.923829+0000 (oldest deadline 2024-04-25T00:47:00.924177+0000) 2024-04-25T00:47:02.239 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:01 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:01.962+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:47:02.239 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:01 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:01.962+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:47:02.239 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:01 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:01.962+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:47:02.239 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:01 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:01.962+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:47:02.239 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:01 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:01.962+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:47:02.239 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:01 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:01.962+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:34.658309+0000 front 2024-04-25T00:46:34.658326+0000 (oldest deadline 2024-04-25T00:46:56.358119+0000) 2024-04-25T00:47:02.391 INFO:journalctl@ceph.osd.8.smithi130.stdout:Apr 25 00:47:02 smithi130 podman[70080]: 2024-04-25 00:47:02.184023613 +0000 UTC m=+0.632390007 container died 3edf091d10bc5563bce0534e710d2f67ef9a489c4e5c14ec23b2d7491e2aea7c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-8-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_CLEAN=True, RELEASE=squid-d2c45d9, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-25T00:47:02.781 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.8.service' 2024-04-25T00:47:02.818 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:47:02.819 INFO:tasks.cephadm.osd.8:Stopped osd.8 2024-04-25T00:47:02.819 INFO:tasks.cephadm.osd.9:Stopping osd.9... 2024-04-25T00:47:02.819 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.9 2024-04-25T00:47:02.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:02 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:02.656+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:47:02.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:02 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:02.656+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:47:02.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:02 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:02.656+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:47:02.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:02 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:02.656+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:47:02.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:02 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:02.656+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:23.922801+0000 front 2024-04-25T00:46:23.922846+0000 (oldest deadline 2024-04-25T00:46:49.222546+0000) 2024-04-25T00:47:02.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:02 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:02.656+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.923692+0000 front 2024-04-25T00:46:30.923829+0000 (oldest deadline 2024-04-25T00:47:00.924177+0000) 2024-04-25T00:47:02.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:02 smithi032 systemd[1]: Stopping Ceph osd.9 for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:47:03.141 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:02 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:02.928+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:03.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:02 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:02.928+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:03.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:02 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:02.928+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:03.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:02 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:02.928+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:03.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:02 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:02.928+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.836754+0000 front 2024-04-25T00:46:30.836803+0000 (oldest deadline 2024-04-25T00:47:00.237230+0000) 2024-04-25T00:47:03.250 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:02 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:02.916+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:47:03.251 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:02 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:02.916+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:47:03.251 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:02 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:02.916+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:47:03.251 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:02 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:02.916+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:47:03.251 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:02 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:02.916+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:47:03.251 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:02 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:02.916+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:34.658309+0000 front 2024-04-25T00:46:34.658326+0000 (oldest deadline 2024-04-25T00:46:56.358119+0000) 2024-04-25T00:47:03.498 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:03 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:03.145+0000 7f3a450db700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.9 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:47:03.498 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:03 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:03.145+0000 7f3a450db700 -1 osd.9 111 *** Got signal Terminated *** 2024-04-25T00:47:03.498 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:03 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:03.145+0000 7f3a450db700 -1 osd.9 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T00:47:03.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:03 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:03.640+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:47:03.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:03 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:03.640+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:47:03.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:03 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:03.640+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:47:03.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:03 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:03.640+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:47:03.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:03 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:03.640+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:23.922801+0000 front 2024-04-25T00:46:23.922846+0000 (oldest deadline 2024-04-25T00:46:49.222546+0000) 2024-04-25T00:47:03.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:03 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:03.640+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.923692+0000 front 2024-04-25T00:46:30.923829+0000 (oldest deadline 2024-04-25T00:47:00.924177+0000) 2024-04-25T00:47:04.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:03 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:03.878+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:47:04.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:03 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:03.878+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:47:04.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:03 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:03.878+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:47:04.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:03 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:03.878+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:47:04.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:03 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:03.878+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:47:04.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:03 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:03.878+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:34.658309+0000 front 2024-04-25T00:46:34.658326+0000 (oldest deadline 2024-04-25T00:46:56.358119+0000) 2024-04-25T00:47:04.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:03 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:03.973+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:04.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:03 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:03.973+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:04.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:03 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:03.973+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:04.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:03 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:03.973+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:04.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:03 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:03.973+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.836754+0000 front 2024-04-25T00:46:30.836803+0000 (oldest deadline 2024-04-25T00:47:00.237230+0000) 2024-04-25T00:47:04.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:04 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:04.610+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:47:04.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:04 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:04.610+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:47:04.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:04 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:04.610+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:47:04.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:04 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:04.610+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:47:04.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:04 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:04.610+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:23.922801+0000 front 2024-04-25T00:46:23.922846+0000 (oldest deadline 2024-04-25T00:46:49.222546+0000) 2024-04-25T00:47:04.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:04 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:04.610+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.923692+0000 front 2024-04-25T00:46:30.923829+0000 (oldest deadline 2024-04-25T00:47:00.924177+0000) 2024-04-25T00:47:05.293 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:04 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:04.967+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:05.293 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:04 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:04.967+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:05.293 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:04 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:04.967+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:05.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:04 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:04.967+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:05.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:04 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:04.967+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.836754+0000 front 2024-04-25T00:46:30.836803+0000 (oldest deadline 2024-04-25T00:47:00.237230+0000) 2024-04-25T00:47:05.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:04 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:04.889+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:47:05.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:04 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:04.889+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:47:05.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:04 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:04.889+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:47:05.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:04 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:04.889+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:47:05.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:04 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:04.889+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:47:05.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:04 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:04.889+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:34.658309+0000 front 2024-04-25T00:46:34.658326+0000 (oldest deadline 2024-04-25T00:46:56.358119+0000) 2024-04-25T00:47:05.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:05 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:05.618+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:47:05.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:05 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:05.618+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:47:05.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:05 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:05.618+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:47:05.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:05 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:05.618+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:47:05.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:05 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:05.618+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:23.922801+0000 front 2024-04-25T00:46:23.922846+0000 (oldest deadline 2024-04-25T00:46:49.222546+0000) 2024-04-25T00:47:05.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:05 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:05.618+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.923692+0000 front 2024-04-25T00:46:30.923829+0000 (oldest deadline 2024-04-25T00:47:00.924177+0000) 2024-04-25T00:47:06.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:05 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:05.854+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:47:06.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:05 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:05.854+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:47:06.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:05 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:05.854+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:47:06.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:05 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:05.854+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:47:06.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:05 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:05.854+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:47:06.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:05 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:05.854+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:34.658309+0000 front 2024-04-25T00:46:34.658326+0000 (oldest deadline 2024-04-25T00:46:56.358119+0000) 2024-04-25T00:47:06.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:05 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:05.930+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:06.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:05 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:05.930+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:06.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:05 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:05.930+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:06.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:05 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:05.930+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:06.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:05 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:05.930+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.836754+0000 front 2024-04-25T00:46:30.836803+0000 (oldest deadline 2024-04-25T00:47:00.237230+0000) 2024-04-25T00:47:06.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:06 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:06.660+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:47:06.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:06 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:06.660+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:47:06.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:06 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:06.660+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:47:06.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:06 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:06.660+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:47:06.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:06 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:06.660+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:23.922801+0000 front 2024-04-25T00:46:23.922846+0000 (oldest deadline 2024-04-25T00:46:49.222546+0000) 2024-04-25T00:47:06.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:06 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:06.660+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.923692+0000 front 2024-04-25T00:46:30.923829+0000 (oldest deadline 2024-04-25T00:47:00.924177+0000) 2024-04-25T00:47:07.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:06 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:06.889+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:07.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:06 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:06.889+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:07.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:06 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:06.889+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:07.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:06 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:06.889+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:07.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:06 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:06.889+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.836754+0000 front 2024-04-25T00:46:30.836803+0000 (oldest deadline 2024-04-25T00:47:00.237230+0000) 2024-04-25T00:47:07.142 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:06 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:06.889+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:41.338170+0000 front 2024-04-25T00:46:41.338284+0000 (oldest deadline 2024-04-25T00:47:06.637963+0000) 2024-04-25T00:47:07.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:06 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:06.844+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:47:07.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:06 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:06.844+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:47:07.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:06 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:06.844+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:47:07.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:06 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:06.844+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:47:07.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:06 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:06.844+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:47:07.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:06 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:06.844+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:34.658309+0000 front 2024-04-25T00:46:34.658326+0000 (oldest deadline 2024-04-25T00:46:56.358119+0000) 2024-04-25T00:47:07.998 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:07 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:07.687+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.514212+0000 front 2024-04-25T00:45:52.514348+0000 (oldest deadline 2024-04-25T00:46:14.214385+0000) 2024-04-25T00:47:07.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:07 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:07.687+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:01.016706+0000 front 2024-04-25T00:46:01.016741+0000 (oldest deadline 2024-04-25T00:46:22.116973+0000) 2024-04-25T00:47:07.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:07 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:07.687+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.919759+0000 front 2024-04-25T00:46:08.919637+0000 (oldest deadline 2024-04-25T00:46:34.819519+0000) 2024-04-25T00:47:07.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:07 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:07.687+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:16.420981+0000 front 2024-04-25T00:46:16.420946+0000 (oldest deadline 2024-04-25T00:46:39.921079+0000) 2024-04-25T00:47:07.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:07 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:07.687+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:23.922801+0000 front 2024-04-25T00:46:23.922846+0000 (oldest deadline 2024-04-25T00:46:49.222546+0000) 2024-04-25T00:47:07.999 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:07 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9[64865]: 2024-04-25T00:47:07.687+0000 7f3a3d6c7700 -1 osd.9 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.923692+0000 front 2024-04-25T00:46:30.923829+0000 (oldest deadline 2024-04-25T00:47:00.924177+0000) 2024-04-25T00:47:08.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:07 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:07.856+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:47:08.339 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:07 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:07.856+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:47:08.339 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:07 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:07.856+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:47:08.339 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:07 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:07.856+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:47:08.339 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:07 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:07.856+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:47:08.339 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:07 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:07.856+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:34.658309+0000 front 2024-04-25T00:46:34.658326+0000 (oldest deadline 2024-04-25T00:46:56.358119+0000) 2024-04-25T00:47:08.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:07 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:07.936+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:08.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:07 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:07.936+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:08.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:07 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:07.936+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:08.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:07 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:07.936+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:08.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:07 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:07.936+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.836754+0000 front 2024-04-25T00:46:30.836803+0000 (oldest deadline 2024-04-25T00:47:00.237230+0000) 2024-04-25T00:47:08.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:07 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:07.936+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:41.338170+0000 front 2024-04-25T00:46:41.338284+0000 (oldest deadline 2024-04-25T00:47:06.637963+0000) 2024-04-25T00:47:09.237 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:08 smithi032 podman[94276]: 2024-04-25 00:47:08.858910524 +0000 UTC m=+5.840924797 container died b43d43ef360b0b39553891327951182521a399129f1bb8f8c723fd80727bf2a0 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-d2c45d9, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424) 2024-04-25T00:47:09.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:08 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:08.860+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:47:09.337 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:08 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:08.860+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:47:09.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:08 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:08.860+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:47:09.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:08 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:08.860+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:47:09.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:08 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:08.860+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:47:09.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:08 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:08.860+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:34.658309+0000 front 2024-04-25T00:46:34.658326+0000 (oldest deadline 2024-04-25T00:46:56.358119+0000) 2024-04-25T00:47:09.338 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:08 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:08.860+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:42.159508+0000 front 2024-04-25T00:46:42.159577+0000 (oldest deadline 2024-04-25T00:47:08.559666+0000) 2024-04-25T00:47:09.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:08 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:08.932+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:09.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:08 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:08.932+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:09.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:08 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:08.932+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:09.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:08 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:08.932+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:09.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:08 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:08.932+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.836754+0000 front 2024-04-25T00:46:30.836803+0000 (oldest deadline 2024-04-25T00:47:00.237230+0000) 2024-04-25T00:47:09.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:08 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:08.932+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:41.338170+0000 front 2024-04-25T00:46:41.338284+0000 (oldest deadline 2024-04-25T00:47:06.637963+0000) 2024-04-25T00:47:09.498 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:09 smithi032 podman[94276]: 2024-04-25 00:47:09.32017775 +0000 UTC m=+6.302192021 container cleanup b43d43ef360b0b39553891327951182521a399129f1bb8f8c723fd80727bf2a0 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-d2c45d9, org.label-schema.license=GPLv2) 2024-04-25T00:47:09.498 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:09 smithi032 bash[94276]: ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9 2024-04-25T00:47:09.940 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:09 smithi032 podman[94293]: 2024-04-25 00:47:09.596811877 +0000 UTC m=+0.735796302 container remove b43d43ef360b0b39553891327951182521a399129f1bb8f8c723fd80727bf2a0 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9, ceph=True, RELEASE=squid-d2c45d9, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T00:47:10.087 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:09 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:09.833+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:47:10.087 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:09 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:09.833+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:47:10.087 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:09 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:09.833+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:47:10.087 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:09 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:09.833+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:47:10.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:09 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:09.833+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:47:10.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:09 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:09.833+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:34.658309+0000 front 2024-04-25T00:46:34.658326+0000 (oldest deadline 2024-04-25T00:46:56.358119+0000) 2024-04-25T00:47:10.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:09 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:09.833+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:42.159508+0000 front 2024-04-25T00:46:42.159577+0000 (oldest deadline 2024-04-25T00:47:08.559666+0000) 2024-04-25T00:47:10.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:09 smithi032 podman[94383]: 2024-04-25 00:47:09.842998477 +0000 UTC m=+0.017857015 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901 2024-04-25T00:47:10.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:09 smithi032 podman[94383]: 2024-04-25 00:47:09.947538425 +0000 UTC m=+0.122397002 container create 4af2ae48d6dd132f1b28c8f655a6482221aa319eded9a007e2e4f6c7f194da0f (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=squid-d2c45d9, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-04-25T00:47:10.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:10 smithi032 podman[94383]: 2024-04-25 00:47:10.183535286 +0000 UTC m=+0.358393829 container init 4af2ae48d6dd132f1b28c8f655a6482221aa319eded9a007e2e4f6c7f194da0f (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=squid-d2c45d9, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-04-25T00:47:10.248 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:10 smithi032 podman[94383]: 2024-04-25 00:47:10.18708684 +0000 UTC m=+0.361945381 container start 4af2ae48d6dd132f1b28c8f655a6482221aa319eded9a007e2e4f6c7f194da0f (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=squid-d2c45d9, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-04-25T00:47:10.391 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:09 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:09.924+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:10.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:09 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:09.924+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:10.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:09 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:09.924+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:10.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:09 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:09.924+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:10.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:09 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:09.924+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.836754+0000 front 2024-04-25T00:46:30.836803+0000 (oldest deadline 2024-04-25T00:47:00.237230+0000) 2024-04-25T00:47:10.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:09 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:09.924+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:41.338170+0000 front 2024-04-25T00:46:41.338284+0000 (oldest deadline 2024-04-25T00:47:06.637963+0000) 2024-04-25T00:47:10.749 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:10 smithi032 podman[94383]: 2024-04-25 00:47:10.28354198 +0000 UTC m=+0.458400530 container attach 4af2ae48d6dd132f1b28c8f655a6482221aa319eded9a007e2e4f6c7f194da0f (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9-deactivate, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , RELEASE=squid-d2c45d9, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-25T00:47:10.749 INFO:journalctl@ceph.osd.9.smithi032.stdout:Apr 25 00:47:10 smithi032 podman[94383]: 2024-04-25 00:47:10.494564957 +0000 UTC m=+0.669423488 container died 4af2ae48d6dd132f1b28c8f655a6482221aa319eded9a007e2e4f6c7f194da0f (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-9-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=squid-d2c45d9, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-04-25T00:47:11.087 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:10 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:10.794+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:47:11.087 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:10 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:10.794+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:47:11.087 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:10 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:10.794+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:47:11.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:10 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:10.794+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:47:11.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:10 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:10.794+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:47:11.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:10 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:10.794+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:34.658309+0000 front 2024-04-25T00:46:34.658326+0000 (oldest deadline 2024-04-25T00:46:56.358119+0000) 2024-04-25T00:47:11.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:10 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:10.794+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:42.159508+0000 front 2024-04-25T00:46:42.159577+0000 (oldest deadline 2024-04-25T00:47:08.559666+0000) 2024-04-25T00:47:11.221 DEBUG:teuthology.orchestra.run.smithi032:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.9.service' 2024-04-25T00:47:11.268 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:47:11.268 INFO:tasks.cephadm.osd.9:Stopped osd.9 2024-04-25T00:47:11.268 INFO:tasks.cephadm.osd.10:Stopping osd.10... 2024-04-25T00:47:11.268 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.10 2024-04-25T00:47:11.391 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:10 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:10.962+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:11.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:10 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:10.962+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:11.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:10 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:10.962+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:11.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:10 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:10.962+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:11.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:10 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:10.962+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.836754+0000 front 2024-04-25T00:46:30.836803+0000 (oldest deadline 2024-04-25T00:47:00.237230+0000) 2024-04-25T00:47:11.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:10 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:10.962+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:41.338170+0000 front 2024-04-25T00:46:41.338284+0000 (oldest deadline 2024-04-25T00:47:06.637963+0000) 2024-04-25T00:47:11.572 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:11 smithi121 systemd[1]: Stopping Ceph osd.10 for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:47:11.838 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:11 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:11.571+0000 7fe96d481700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.10 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:47:11.838 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:11 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:11.571+0000 7fe96d481700 -1 osd.10 111 *** Got signal Terminated *** 2024-04-25T00:47:11.838 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:11 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:11.571+0000 7fe96d481700 -1 osd.10 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T00:47:11.838 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:11 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:11.766+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:47:11.838 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:11 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:11.766+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:47:11.838 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:11 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:11.766+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:47:11.838 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:11 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:11.766+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:47:11.838 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:11 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:11.766+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:47:11.838 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:11 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:11.766+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:34.658309+0000 front 2024-04-25T00:46:34.658326+0000 (oldest deadline 2024-04-25T00:46:56.358119+0000) 2024-04-25T00:47:11.838 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:11 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:11.766+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:42.159508+0000 front 2024-04-25T00:46:42.159577+0000 (oldest deadline 2024-04-25T00:47:08.559666+0000) 2024-04-25T00:47:12.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:11 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:11.986+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:12.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:11 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:11.986+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:12.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:11 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:11.986+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:12.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:11 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:11.986+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:12.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:11 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:11.986+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.836754+0000 front 2024-04-25T00:46:30.836803+0000 (oldest deadline 2024-04-25T00:47:00.237230+0000) 2024-04-25T00:47:12.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:11 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:11.986+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:41.338170+0000 front 2024-04-25T00:46:41.338284+0000 (oldest deadline 2024-04-25T00:47:06.637963+0000) 2024-04-25T00:47:13.087 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:12 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:12.742+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:47:13.087 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:12 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:12.742+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:47:13.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:12 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:12.742+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:47:13.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:12 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:12.742+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:47:13.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:12 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:12.742+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:47:13.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:12 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:12.742+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:34.658309+0000 front 2024-04-25T00:46:34.658326+0000 (oldest deadline 2024-04-25T00:46:56.358119+0000) 2024-04-25T00:47:13.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:12 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:12.742+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:42.159508+0000 front 2024-04-25T00:46:42.159577+0000 (oldest deadline 2024-04-25T00:47:08.559666+0000) 2024-04-25T00:47:13.391 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:12 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:12.966+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:13.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:12 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:12.966+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:13.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:12 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:12.966+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:13.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:12 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:12.966+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:13.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:12 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:12.966+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.836754+0000 front 2024-04-25T00:46:30.836803+0000 (oldest deadline 2024-04-25T00:47:00.237230+0000) 2024-04-25T00:47:13.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:12 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:12.966+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:41.338170+0000 front 2024-04-25T00:46:41.338284+0000 (oldest deadline 2024-04-25T00:47:06.637963+0000) 2024-04-25T00:47:14.087 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:13 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:13.741+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:47:14.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:13 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:13.741+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:47:14.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:13 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:13.741+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:47:14.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:13 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:13.741+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:47:14.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:13 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:13.741+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:47:14.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:13 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:13.741+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:34.658309+0000 front 2024-04-25T00:46:34.658326+0000 (oldest deadline 2024-04-25T00:46:56.358119+0000) 2024-04-25T00:47:14.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:13 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:13.741+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:42.159508+0000 front 2024-04-25T00:46:42.159577+0000 (oldest deadline 2024-04-25T00:47:08.559666+0000) 2024-04-25T00:47:14.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:14 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:14.014+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:14.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:14 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:14.014+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:14.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:14 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:14.014+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:14.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:14 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:14.014+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:14.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:14 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:14.014+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.836754+0000 front 2024-04-25T00:46:30.836803+0000 (oldest deadline 2024-04-25T00:47:00.237230+0000) 2024-04-25T00:47:14.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:14 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:14.014+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:41.338170+0000 front 2024-04-25T00:46:41.338284+0000 (oldest deadline 2024-04-25T00:47:06.637963+0000) 2024-04-25T00:47:15.087 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:14 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:14.777+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:47:15.087 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:14 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:14.777+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:47:15.087 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:14 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:14.777+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:47:15.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:14 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:14.777+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:47:15.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:14 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:14.777+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:47:15.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:14 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:14.777+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:34.658309+0000 front 2024-04-25T00:46:34.658326+0000 (oldest deadline 2024-04-25T00:46:56.358119+0000) 2024-04-25T00:47:15.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:14 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:14.777+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:42.159508+0000 front 2024-04-25T00:46:42.159577+0000 (oldest deadline 2024-04-25T00:47:08.559666+0000) 2024-04-25T00:47:15.391 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:14 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:14.973+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:15.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:14 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:14.973+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:15.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:14 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:14.973+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:15.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:14 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:14.973+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:15.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:14 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:14.973+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.836754+0000 front 2024-04-25T00:46:30.836803+0000 (oldest deadline 2024-04-25T00:47:00.237230+0000) 2024-04-25T00:47:15.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:14 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:14.973+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:41.338170+0000 front 2024-04-25T00:46:41.338284+0000 (oldest deadline 2024-04-25T00:47:06.637963+0000) 2024-04-25T00:47:15.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:14 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:14.973+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6820 osd.7 since back 2024-04-25T00:46:48.939192+0000 front 2024-04-25T00:46:48.939268+0000 (oldest deadline 2024-04-25T00:47:14.239151+0000) 2024-04-25T00:47:16.087 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:15 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:15.753+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:51.949535+0000 front 2024-04-25T00:45:51.949283+0000 (oldest deadline 2024-04-25T00:46:14.249405+0000) 2024-04-25T00:47:16.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:15 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:15.753+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:45:58.850944+0000 front 2024-04-25T00:45:58.850976+0000 (oldest deadline 2024-04-25T00:46:22.951288+0000) 2024-04-25T00:47:16.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:15 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:15.753+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:08.053404+0000 front 2024-04-25T00:46:08.053298+0000 (oldest deadline 2024-04-25T00:46:32.153286+0000) 2024-04-25T00:47:16.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:15 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:15.753+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6814 osd.3 since back 2024-04-25T00:46:18.355642+0000 front 2024-04-25T00:46:18.355607+0000 (oldest deadline 2024-04-25T00:46:40.655524+0000) 2024-04-25T00:47:16.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:15 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:15.753+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:26.556486+0000 front 2024-04-25T00:46:26.556512+0000 (oldest deadline 2024-04-25T00:46:50.056533+0000) 2024-04-25T00:47:16.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:15 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:15.753+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:34.658309+0000 front 2024-04-25T00:46:34.658326+0000 (oldest deadline 2024-04-25T00:46:56.358119+0000) 2024-04-25T00:47:16.088 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:15 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10[56899]: 2024-04-25T00:47:15.753+0000 7fe969a75700 -1 osd.10 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:42.159508+0000 front 2024-04-25T00:46:42.159577+0000 (oldest deadline 2024-04-25T00:47:08.559666+0000) 2024-04-25T00:47:16.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:16 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:16.008+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:16.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:16 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:16.008+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:16.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:16 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:16.008+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:16.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:16 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:16.008+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:16.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:16 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:16.008+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.836754+0000 front 2024-04-25T00:46:30.836803+0000 (oldest deadline 2024-04-25T00:47:00.237230+0000) 2024-04-25T00:47:16.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:16 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:16.008+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:41.338170+0000 front 2024-04-25T00:46:41.338284+0000 (oldest deadline 2024-04-25T00:47:06.637963+0000) 2024-04-25T00:47:16.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:16 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:16.008+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6820 osd.7 since back 2024-04-25T00:46:48.939192+0000 front 2024-04-25T00:46:48.939268+0000 (oldest deadline 2024-04-25T00:47:14.239151+0000) 2024-04-25T00:47:17.391 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:16 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:16.973+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:17.391 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:16 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:16.973+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:17.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:16 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:16.973+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:17.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:16 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:16.973+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:17.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:16 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:16.973+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.836754+0000 front 2024-04-25T00:46:30.836803+0000 (oldest deadline 2024-04-25T00:47:00.237230+0000) 2024-04-25T00:47:17.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:16 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:16.973+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:41.338170+0000 front 2024-04-25T00:46:41.338284+0000 (oldest deadline 2024-04-25T00:47:06.637963+0000) 2024-04-25T00:47:17.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:16 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:16.973+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6820 osd.7 since back 2024-04-25T00:46:48.939192+0000 front 2024-04-25T00:46:48.939268+0000 (oldest deadline 2024-04-25T00:47:14.239151+0000) 2024-04-25T00:47:17.487 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:17 smithi121 podman[69116]: 2024-04-25 00:47:17.221112144 +0000 UTC m=+5.761340003 container died 0aa4097e757d7352b9102b0fdb33c8bb06e22855da71cf3368aac6ce5f9b1f90 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-d2c45d9, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-25T00:47:17.488 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:17 smithi121 podman[69116]: 2024-04-25 00:47:17.488103258 +0000 UTC m=+6.028331117 container cleanup 0aa4097e757d7352b9102b0fdb33c8bb06e22855da71cf3368aac6ce5f9b1f90 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=squid-d2c45d9, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-25T00:47:17.747 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:17 smithi121 bash[69116]: ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10 2024-04-25T00:47:18.087 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:17 smithi121 podman[69133]: 2024-04-25 00:47:17.747668766 +0000 UTC m=+0.517596840 container remove 0aa4097e757d7352b9102b0fdb33c8bb06e22855da71cf3368aac6ce5f9b1f90 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-d2c45d9) 2024-04-25T00:47:18.367 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:18 smithi121 podman[69226]: 2024-04-25 00:47:18.098268532 +0000 UTC m=+0.114152061 container create a1772be549d5975712412a7bcc9b924fb04cb3153ee19678c330b9df78095342 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, RELEASE=squid-d2c45d9, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True) 2024-04-25T00:47:18.367 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:18 smithi121 podman[69226]: 2024-04-25 00:47:18.00194806 +0000 UTC m=+0.017831611 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901 2024-04-25T00:47:18.367 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:18 smithi121 podman[69226]: 2024-04-25 00:47:18.292395735 +0000 UTC m=+0.308279275 container init a1772be549d5975712412a7bcc9b924fb04cb3153ee19678c330b9df78095342 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10-deactivate, RELEASE=squid-d2c45d9, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T00:47:18.367 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:18 smithi121 podman[69226]: 2024-04-25 00:47:18.295358124 +0000 UTC m=+0.311241653 container start a1772be549d5975712412a7bcc9b924fb04cb3153ee19678c330b9df78095342 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10-deactivate, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-d2c45d9, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-04-25T00:47:18.367 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:18 smithi121 podman[69226]: 2024-04-25 00:47:18.367239548 +0000 UTC m=+0.383123087 container attach a1772be549d5975712412a7bcc9b924fb04cb3153ee19678c330b9df78095342 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-d2c45d9, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424) 2024-04-25T00:47:18.391 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:17 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:17.955+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:18.391 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:17 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:17.955+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:18.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:17 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:17.955+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:18.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:17 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:17.955+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:18.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:17 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:17.955+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.836754+0000 front 2024-04-25T00:46:30.836803+0000 (oldest deadline 2024-04-25T00:47:00.237230+0000) 2024-04-25T00:47:18.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:17 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:17.955+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:41.338170+0000 front 2024-04-25T00:46:41.338284+0000 (oldest deadline 2024-04-25T00:47:06.637963+0000) 2024-04-25T00:47:18.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:17 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:17.955+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6820 osd.7 since back 2024-04-25T00:46:48.939192+0000 front 2024-04-25T00:46:48.939268+0000 (oldest deadline 2024-04-25T00:47:14.239151+0000) 2024-04-25T00:47:19.054 INFO:journalctl@ceph.osd.10.smithi121.stdout:Apr 25 00:47:18 smithi121 podman[69252]: 2024-04-25 00:47:18.620248543 +0000 UTC m=+0.018714504 container died a1772be549d5975712412a7bcc9b924fb04cb3153ee19678c330b9df78095342 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-10-deactivate, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=squid-d2c45d9, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-25T00:47:19.076 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.10.service' 2024-04-25T00:47:19.114 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:47:19.114 INFO:tasks.cephadm.osd.10:Stopped osd.10 2024-04-25T00:47:19.114 INFO:tasks.cephadm.osd.11:Stopping osd.11... 2024-04-25T00:47:19.114 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.11 2024-04-25T00:47:19.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:18 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:18.931+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:19.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:18 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:18.931+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:19.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:18 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:18.931+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:19.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:18 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:18.931+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:19.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:18 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:18.931+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.836754+0000 front 2024-04-25T00:46:30.836803+0000 (oldest deadline 2024-04-25T00:47:00.237230+0000) 2024-04-25T00:47:19.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:18 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:18.931+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:41.338170+0000 front 2024-04-25T00:46:41.338284+0000 (oldest deadline 2024-04-25T00:47:06.637963+0000) 2024-04-25T00:47:19.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:18 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:18.931+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6820 osd.7 since back 2024-04-25T00:46:48.939192+0000 front 2024-04-25T00:46:48.939268+0000 (oldest deadline 2024-04-25T00:47:14.239151+0000) 2024-04-25T00:47:19.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:19 smithi130 systemd[1]: Stopping Ceph osd.11 for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:47:19.891 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:19 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:19.393+0000 7f81e8f86700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.11 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:47:19.891 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:19 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:19.393+0000 7f81e8f86700 -1 osd.11 111 *** Got signal Terminated *** 2024-04-25T00:47:19.892 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:19 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:19.393+0000 7f81e8f86700 -1 osd.11 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T00:47:19.892 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:19 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:19.891+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:20.391 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:19 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:19.891+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:20.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:19 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:19.891+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:20.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:19 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:19.891+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:20.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:19 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:19.891+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.836754+0000 front 2024-04-25T00:46:30.836803+0000 (oldest deadline 2024-04-25T00:47:00.237230+0000) 2024-04-25T00:47:20.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:19 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:19.891+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:41.338170+0000 front 2024-04-25T00:46:41.338284+0000 (oldest deadline 2024-04-25T00:47:06.637963+0000) 2024-04-25T00:47:20.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:19 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:19.891+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6820 osd.7 since back 2024-04-25T00:46:48.939192+0000 front 2024-04-25T00:46:48.939268+0000 (oldest deadline 2024-04-25T00:47:14.239151+0000) 2024-04-25T00:47:21.391 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:20 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:20.925+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:21.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:20 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:20.925+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:21.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:20 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:20.925+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:21.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:20 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:20.925+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:21.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:20 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:20.925+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.836754+0000 front 2024-04-25T00:46:30.836803+0000 (oldest deadline 2024-04-25T00:47:00.237230+0000) 2024-04-25T00:47:21.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:20 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:20.925+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:41.338170+0000 front 2024-04-25T00:46:41.338284+0000 (oldest deadline 2024-04-25T00:47:06.637963+0000) 2024-04-25T00:47:21.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:20 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:20.925+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6820 osd.7 since back 2024-04-25T00:46:48.939192+0000 front 2024-04-25T00:46:48.939268+0000 (oldest deadline 2024-04-25T00:47:14.239151+0000) 2024-04-25T00:47:22.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:21 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:21.958+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:22.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:21 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:21.958+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:22.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:21 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:21.958+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:22.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:21 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:21.958+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:22.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:21 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:21.958+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.836754+0000 front 2024-04-25T00:46:30.836803+0000 (oldest deadline 2024-04-25T00:47:00.237230+0000) 2024-04-25T00:47:22.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:21 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:21.958+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:41.338170+0000 front 2024-04-25T00:46:41.338284+0000 (oldest deadline 2024-04-25T00:47:06.637963+0000) 2024-04-25T00:47:22.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:21 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:21.958+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6820 osd.7 since back 2024-04-25T00:46:48.939192+0000 front 2024-04-25T00:46:48.939268+0000 (oldest deadline 2024-04-25T00:47:14.239151+0000) 2024-04-25T00:47:23.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:22 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:22.912+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:23.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:22 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:22.912+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:23.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:22 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:22.912+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:23.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:22 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:22.912+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:23.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:22 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:22.912+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.836754+0000 front 2024-04-25T00:46:30.836803+0000 (oldest deadline 2024-04-25T00:47:00.237230+0000) 2024-04-25T00:47:23.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:22 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:22.912+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:41.338170+0000 front 2024-04-25T00:46:41.338284+0000 (oldest deadline 2024-04-25T00:47:06.637963+0000) 2024-04-25T00:47:23.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:22 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:22.912+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6820 osd.7 since back 2024-04-25T00:46:48.939192+0000 front 2024-04-25T00:46:48.939268+0000 (oldest deadline 2024-04-25T00:47:14.239151+0000) 2024-04-25T00:47:24.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:23 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:23.911+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-04-25T00:45:52.231313+0000 front 2024-04-25T00:45:52.231344+0000 (oldest deadline 2024-04-25T00:46:15.131492+0000) 2024-04-25T00:47:24.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:23 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:23.911+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6804 osd.1 since back 2024-04-25T00:46:00.933172+0000 front 2024-04-25T00:46:00.933265+0000 (oldest deadline 2024-04-25T00:46:25.633161+0000) 2024-04-25T00:47:24.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:23 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:23.911+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2024-04-25T00:46:05.633457+0000 front 2024-04-25T00:46:05.633598+0000 (oldest deadline 2024-04-25T00:46:30.933596+0000) 2024-04-25T00:47:24.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:23 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:23.911+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6812 osd.4 since back 2024-04-25T00:46:24.936292+0000 front 2024-04-25T00:46:24.936239+0000 (oldest deadline 2024-04-25T00:46:50.836295+0000) 2024-04-25T00:47:24.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:23 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:23.911+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2024-04-25T00:46:30.836754+0000 front 2024-04-25T00:46:30.836803+0000 (oldest deadline 2024-04-25T00:47:00.237230+0000) 2024-04-25T00:47:24.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:23 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:23.911+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.32:6822 osd.6 since back 2024-04-25T00:46:41.338170+0000 front 2024-04-25T00:46:41.338284+0000 (oldest deadline 2024-04-25T00:47:06.637963+0000) 2024-04-25T00:47:24.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:23 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:23.911+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.121:6820 osd.7 since back 2024-04-25T00:46:48.939192+0000 front 2024-04-25T00:46:48.939268+0000 (oldest deadline 2024-04-25T00:47:14.239151+0000) 2024-04-25T00:47:24.392 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:23 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11[58036]: 2024-04-25T00:47:23.911+0000 7f81e1572700 -1 osd.11 111 heartbeat_check: no reply from 172.21.15.130:6820 osd.8 since back 2024-04-25T00:46:57.740066+0000 front 2024-04-25T00:46:57.740094+0000 (oldest deadline 2024-04-25T00:47:23.040043+0000) 2024-04-25T00:47:25.319 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:25 smithi130 podman[70199]: 2024-04-25 00:47:25.024202129 +0000 UTC m=+5.709600206 container died 1ba7200dfdb2a98dc050f3348bfaf07a4924b4d9ac89152126aedf87fafcb199 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-d2c45d9, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5) 2024-04-25T00:47:25.635 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:25 smithi130 podman[70199]: 2024-04-25 00:47:25.401531964 +0000 UTC m=+6.086930046 container cleanup 1ba7200dfdb2a98dc050f3348bfaf07a4924b4d9ac89152126aedf87fafcb199 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, RELEASE=squid-d2c45d9, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-04-25T00:47:25.635 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:25 smithi130 bash[70199]: ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11 2024-04-25T00:47:25.891 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:25 smithi130 podman[70214]: 2024-04-25 00:47:25.635693666 +0000 UTC m=+0.611638788 container remove 1ba7200dfdb2a98dc050f3348bfaf07a4924b4d9ac89152126aedf87fafcb199 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, RELEASE=squid-d2c45d9) 2024-04-25T00:47:26.255 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:25 smithi130 podman[70305]: 2024-04-25 00:47:25.852619827 +0000 UTC m=+0.018013719 image pull f68e37b807ef3dfd57708ff7362bfa0edaabe0111536b8e8a4c26de22a72557e quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901 2024-04-25T00:47:26.255 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:25 smithi130 podman[70305]: 2024-04-25 00:47:25.961521222 +0000 UTC m=+0.126915104 container create 290c63bf3b52ef28990ff9e08a893d91f3dc4586780671427b30296b90c03102 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11-deactivate, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, RELEASE=squid-d2c45d9, GIT_CLEAN=True) 2024-04-25T00:47:26.255 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:26 smithi130 podman[70305]: 2024-04-25 00:47:26.180609108 +0000 UTC m=+0.346002991 container init 290c63bf3b52ef28990ff9e08a893d91f3dc4586780671427b30296b90c03102 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, RELEASE=squid-d2c45d9, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-25T00:47:26.255 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:26 smithi130 podman[70305]: 2024-04-25 00:47:26.183872713 +0000 UTC m=+0.349266599 container start 290c63bf3b52ef28990ff9e08a893d91f3dc4586780671427b30296b90c03102 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11-deactivate, CEPH_POINT_RELEASE=, RELEASE=squid-d2c45d9, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-25T00:47:26.255 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:26 smithi130 podman[70305]: 2024-04-25 00:47:26.255549794 +0000 UTC m=+0.420943693 container attach 290c63bf3b52ef28990ff9e08a893d91f3dc4586780671427b30296b90c03102 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-d2c45d9, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-25T00:47:26.642 INFO:journalctl@ceph.osd.11.smithi130.stdout:Apr 25 00:47:26 smithi130 podman[70305]: 2024-04-25 00:47:26.474481469 +0000 UTC m=+0.639875347 container died 290c63bf3b52ef28990ff9e08a893d91f3dc4586780671427b30296b90c03102 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-osd-11-deactivate, maintainer=Guillaume Abrioux , RELEASE=squid-d2c45d9, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2) 2024-04-25T00:47:27.012 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@osd.11.service' 2024-04-25T00:47:27.050 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:47:27.050 INFO:tasks.cephadm.osd.11:Stopped osd.11 2024-04-25T00:47:27.051 INFO:tasks.cephadm.mds.a:Stopping mds.a... 2024-04-25T00:47:27.051 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.a 2024-04-25T00:47:27.356 INFO:journalctl@ceph.mds.a.smithi032.stdout:Apr 25 00:47:27 smithi032 systemd[1]: Stopping Ceph mds.a for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:47:27.748 INFO:journalctl@ceph.mds.a.smithi032.stdout:Apr 25 00:47:27 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-a[71240]: 2024-04-25T00:47:27.355+0000 7f58934ed700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:47:27.748 INFO:journalctl@ceph.mds.a.smithi032.stdout:Apr 25 00:47:27 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-a[71240]: 2024-04-25T00:47:27.355+0000 7f58934ed700 -1 mds.a *** got signal Terminated *** 2024-04-25T00:47:31.704 INFO:journalctl@ceph.mds.a.smithi032.stdout:Apr 25 00:47:31 smithi032 podman[94502]: 2024-04-25 00:47:31.398893464 +0000 UTC m=+4.144473467 container died cdcac5998a3b00dafdc26c4a08c31af58d2a95367e4cee926dfe020010b6fec3 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-a, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-d2c45d9, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-25T00:47:31.704 INFO:journalctl@ceph.mds.a.smithi032.stdout:Apr 25 00:47:31 smithi032 podman[94502]: 2024-04-25 00:47:31.70454968 +0000 UTC m=+4.450129679 container cleanup cdcac5998a3b00dafdc26c4a08c31af58d2a95367e4cee926dfe020010b6fec3 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-a, RELEASE=squid-d2c45d9, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-25T00:47:31.998 INFO:journalctl@ceph.mds.a.smithi032.stdout:Apr 25 00:47:31 smithi032 bash[94502]: ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-a 2024-04-25T00:47:31.998 INFO:journalctl@ceph.mds.a.smithi032.stdout:Apr 25 00:47:31 smithi032 podman[94516]: 2024-04-25 00:47:31.930907599 +0000 UTC m=+0.526146422 container remove cdcac5998a3b00dafdc26c4a08c31af58d2a95367e4cee926dfe020010b6fec3 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-a, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, ceph=True, RELEASE=squid-d2c45d9, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2) 2024-04-25T00:47:32.067 DEBUG:teuthology.orchestra.run.smithi032:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.a.service' 2024-04-25T00:47:32.102 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:47:32.103 INFO:tasks.cephadm.mds.a:Stopped mds.a 2024-04-25T00:47:32.103 INFO:tasks.cephadm.mds.d:Stopping mds.d... 2024-04-25T00:47:32.103 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.d 2024-04-25T00:47:32.498 INFO:journalctl@ceph.mds.d.smithi032.stdout:Apr 25 00:47:32 smithi032 systemd[1]: Stopping Ceph mds.d for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:47:32.998 INFO:journalctl@ceph.mds.d.smithi032.stdout:Apr 25 00:47:32 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-d[70285]: 2024-04-25T00:47:32.540+0000 7f09a9cde700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.d -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:47:32.999 INFO:journalctl@ceph.mds.d.smithi032.stdout:Apr 25 00:47:32 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-d[70285]: 2024-04-25T00:47:32.540+0000 7f09a9cde700 -1 mds.d *** got signal Terminated *** 2024-04-25T00:47:35.748 INFO:journalctl@ceph.mds.d.smithi032.stdout:Apr 25 00:47:35 smithi032 podman[94621]: 2024-04-25 00:47:35.386397779 +0000 UTC m=+3.050770980 container died 0003cea6ab589bd36a1d02551fe2ceacc8acc7563e540888692a6b17282ecf79 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-d, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, RELEASE=squid-d2c45d9, org.label-schema.license=GPLv2) 2024-04-25T00:47:36.109 INFO:journalctl@ceph.mds.d.smithi032.stdout:Apr 25 00:47:35 smithi032 podman[94621]: 2024-04-25 00:47:35.850099632 +0000 UTC m=+3.514472831 container cleanup 0003cea6ab589bd36a1d02551fe2ceacc8acc7563e540888692a6b17282ecf79 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-d, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, RELEASE=squid-d2c45d9, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T00:47:36.109 INFO:journalctl@ceph.mds.d.smithi032.stdout:Apr 25 00:47:35 smithi032 bash[94621]: ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-d 2024-04-25T00:47:36.242 DEBUG:teuthology.orchestra.run.smithi032:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.d.service' 2024-04-25T00:47:36.274 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:47:36.275 INFO:tasks.cephadm.mds.d:Stopped mds.d 2024-04-25T00:47:36.275 INFO:tasks.cephadm.mds.g:Stopping mds.g... 2024-04-25T00:47:36.275 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.g 2024-04-25T00:47:36.498 INFO:journalctl@ceph.mds.g.smithi032.stdout:Apr 25 00:47:36 smithi032 systemd[1]: Stopping Ceph mds.g for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:47:36.998 INFO:journalctl@ceph.mds.g.smithi032.stdout:Apr 25 00:47:36 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-g[70883]: 2024-04-25T00:47:36.595+0000 7f8ad4588700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.g -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:47:36.998 INFO:journalctl@ceph.mds.g.smithi032.stdout:Apr 25 00:47:36 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-g[70883]: 2024-04-25T00:47:36.595+0000 7f8ad4588700 -1 mds.g *** got signal Terminated *** 2024-04-25T00:47:40.588 INFO:journalctl@ceph.mds.g.smithi032.stdout:Apr 25 00:47:40 smithi032 podman[94740]: 2024-04-25 00:47:40.322289394 +0000 UTC m=+3.828731430 container died 6f2ee1447833de9d1995ecd9d122c18d0382082b72c3c7774dd2b972497e5e9f (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-g, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_CLEAN=True, RELEASE=squid-d2c45d9) 2024-04-25T00:47:40.988 INFO:journalctl@ceph.mds.g.smithi032.stdout:Apr 25 00:47:40 smithi032 podman[94740]: 2024-04-25 00:47:40.6874872 +0000 UTC m=+4.193929234 container cleanup 6f2ee1447833de9d1995ecd9d122c18d0382082b72c3c7774dd2b972497e5e9f (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-g, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, RELEASE=squid-d2c45d9, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-25T00:47:40.988 INFO:journalctl@ceph.mds.g.smithi032.stdout:Apr 25 00:47:40 smithi032 bash[94740]: ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-g 2024-04-25T00:47:41.129 DEBUG:teuthology.orchestra.run.smithi032:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.g.service' 2024-04-25T00:47:41.162 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:47:41.162 INFO:tasks.cephadm.mds.g:Stopped mds.g 2024-04-25T00:47:41.163 INFO:tasks.cephadm.mds.j:Stopping mds.j... 2024-04-25T00:47:41.163 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.j 2024-04-25T00:47:41.248 INFO:journalctl@ceph.mds.j.smithi032.stdout:Apr 25 00:47:41 smithi032 systemd[1]: Stopping Ceph mds.j for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:47:41.748 INFO:journalctl@ceph.mds.j.smithi032.stdout:Apr 25 00:47:41 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-j[69938]: 2024-04-25T00:47:41.482+0000 7f9edef9f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.j -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:47:41.748 INFO:journalctl@ceph.mds.j.smithi032.stdout:Apr 25 00:47:41 smithi032 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-j[69938]: 2024-04-25T00:47:41.482+0000 7f9edef9f700 -1 mds.j *** got signal Terminated *** 2024-04-25T00:47:44.642 INFO:journalctl@ceph.mds.j.smithi032.stdout:Apr 25 00:47:44 smithi032 podman[94861]: 2024-04-25 00:47:44.386397638 +0000 UTC m=+3.010515379 container died 96ce71e67c0415c22f9ebc82ad29f552940a63c19f88a2ad84807b2e82dd6b33 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-j, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, RELEASE=squid-d2c45d9, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-25T00:47:44.916 INFO:journalctl@ceph.mds.j.smithi032.stdout:Apr 25 00:47:44 smithi032 podman[94861]: 2024-04-25 00:47:44.732403474 +0000 UTC m=+3.356521246 container cleanup 96ce71e67c0415c22f9ebc82ad29f552940a63c19f88a2ad84807b2e82dd6b33 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-j, io.buildah.version=1.33.5, RELEASE=squid-d2c45d9, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-25T00:47:44.916 INFO:journalctl@ceph.mds.j.smithi032.stdout:Apr 25 00:47:44 smithi032 bash[94861]: ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-j 2024-04-25T00:47:45.124 DEBUG:teuthology.orchestra.run.smithi032:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.j.service' 2024-04-25T00:47:45.156 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:47:45.156 INFO:tasks.cephadm.mds.j:Stopped mds.j 2024-04-25T00:47:45.156 INFO:tasks.cephadm.mds.b:Stopping mds.b... 2024-04-25T00:47:45.156 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.b 2024-04-25T00:47:45.465 INFO:journalctl@ceph.mds.b.smithi121.stdout:Apr 25 00:47:45 smithi121 systemd[1]: Stopping Ceph mds.b for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:47:45.726 INFO:journalctl@ceph.mds.b.smithi121.stdout:Apr 25 00:47:45 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-b[62582]: 2024-04-25T00:47:45.527+0000 7feda5482700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:47:45.726 INFO:journalctl@ceph.mds.b.smithi121.stdout:Apr 25 00:47:45 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-b[62582]: 2024-04-25T00:47:45.527+0000 7feda5482700 -1 mds.b *** got signal Terminated *** 2024-04-25T00:47:49.796 INFO:journalctl@ceph.mds.b.smithi121.stdout:Apr 25 00:47:49 smithi121 podman[69330]: 2024-04-25 00:47:49.532818047 +0000 UTC m=+4.189130304 container died 43cd35e25c64f90b2697f7d39f99e8a9c21dc6b8254d631ad4137b6cd092ada1 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-b, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-d2c45d9, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-25T00:47:50.079 INFO:journalctl@ceph.mds.b.smithi121.stdout:Apr 25 00:47:49 smithi121 podman[69330]: 2024-04-25 00:47:49.869726308 +0000 UTC m=+4.526038571 container cleanup 43cd35e25c64f90b2697f7d39f99e8a9c21dc6b8254d631ad4137b6cd092ada1 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-b, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, RELEASE=squid-d2c45d9, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240424) 2024-04-25T00:47:50.079 INFO:journalctl@ceph.mds.b.smithi121.stdout:Apr 25 00:47:49 smithi121 bash[69330]: ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-b 2024-04-25T00:47:50.296 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.b.service' 2024-04-25T00:47:50.331 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:47:50.331 INFO:tasks.cephadm.mds.b:Stopped mds.b 2024-04-25T00:47:50.331 INFO:tasks.cephadm.mds.e:Stopping mds.e... 2024-04-25T00:47:50.332 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.e 2024-04-25T00:47:50.587 INFO:journalctl@ceph.mds.e.smithi121.stdout:Apr 25 00:47:50 smithi121 systemd[1]: Stopping Ceph mds.e for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:47:51.087 INFO:journalctl@ceph.mds.e.smithi121.stdout:Apr 25 00:47:50 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-e[61960]: 2024-04-25T00:47:50.664+0000 7f74edf98700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.e -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:47:51.087 INFO:journalctl@ceph.mds.e.smithi121.stdout:Apr 25 00:47:50 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-e[61960]: 2024-04-25T00:47:50.664+0000 7f74edf98700 -1 mds.e *** got signal Terminated *** 2024-04-25T00:47:54.400 INFO:journalctl@ceph.mds.e.smithi121.stdout:Apr 25 00:47:54 smithi121 podman[69463]: 2024-04-25 00:47:54.098818847 +0000 UTC m=+3.543073283 container died 489b0aa7cac675b6ab6b9303fc31dab886ac498c2f04ed26210219a249bcd217 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-e, org.label-schema.schema-version=1.0, RELEASE=squid-d2c45d9, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=) 2024-04-25T00:47:54.837 INFO:journalctl@ceph.mds.e.smithi121.stdout:Apr 25 00:47:54 smithi121 podman[69463]: 2024-04-25 00:47:54.498550719 +0000 UTC m=+3.942805155 container cleanup 489b0aa7cac675b6ab6b9303fc31dab886ac498c2f04ed26210219a249bcd217 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, RELEASE=squid-d2c45d9, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-25T00:47:54.837 INFO:journalctl@ceph.mds.e.smithi121.stdout:Apr 25 00:47:54 smithi121 bash[69463]: ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-e 2024-04-25T00:47:54.838 INFO:journalctl@ceph.mds.e.smithi121.stdout:Apr 25 00:47:54 smithi121 podman[69477]: 2024-04-25 00:47:54.649406757 +0000 UTC m=+0.513270293 container remove 489b0aa7cac675b6ab6b9303fc31dab886ac498c2f04ed26210219a249bcd217 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-e, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=squid-d2c45d9, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-25T00:47:54.870 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.e.service' 2024-04-25T00:47:54.901 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:47:54.901 INFO:tasks.cephadm.mds.e:Stopped mds.e 2024-04-25T00:47:54.902 INFO:tasks.cephadm.mds.h:Stopping mds.h... 2024-04-25T00:47:54.902 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.h 2024-04-25T00:47:55.210 INFO:journalctl@ceph.mds.h.smithi121.stdout:Apr 25 00:47:54 smithi121 systemd[1]: Stopping Ceph mds.h for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:47:55.587 INFO:journalctl@ceph.mds.h.smithi121.stdout:Apr 25 00:47:55 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-h[62273]: 2024-04-25T00:47:55.209+0000 7fc110e85700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.h -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:47:55.587 INFO:journalctl@ceph.mds.h.smithi121.stdout:Apr 25 00:47:55 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-h[62273]: 2024-04-25T00:47:55.209+0000 7fc110e85700 -1 mds.h *** got signal Terminated *** 2024-04-25T00:47:58.418 INFO:journalctl@ceph.mds.h.smithi121.stdout:Apr 25 00:47:58 smithi121 podman[69584]: 2024-04-25 00:47:58.108557645 +0000 UTC m=+2.991358113 container died 273b11cef7b658743ab666683522aafac5fe665817fdd44ebe3e69be01a91186 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-h, RELEASE=squid-d2c45d9, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS) 2024-04-25T00:47:58.418 INFO:journalctl@ceph.mds.h.smithi121.stdout:Apr 25 00:47:58 smithi121 podman[69584]: 2024-04-25 00:47:58.418241634 +0000 UTC m=+3.301042105 container cleanup 273b11cef7b658743ab666683522aafac5fe665817fdd44ebe3e69be01a91186 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-h, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=squid-d2c45d9) 2024-04-25T00:47:58.772 INFO:journalctl@ceph.mds.h.smithi121.stdout:Apr 25 00:47:58 smithi121 bash[69584]: ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-h 2024-04-25T00:47:58.772 INFO:journalctl@ceph.mds.h.smithi121.stdout:Apr 25 00:47:58 smithi121 podman[69599]: 2024-04-25 00:47:58.560883219 +0000 UTC m=+0.454509141 container remove 273b11cef7b658743ab666683522aafac5fe665817fdd44ebe3e69be01a91186 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-h, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=squid-d2c45d9, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424) 2024-04-25T00:47:58.787 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.h.service' 2024-04-25T00:47:58.818 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:47:58.818 INFO:tasks.cephadm.mds.h:Stopped mds.h 2024-04-25T00:47:58.819 INFO:tasks.cephadm.mds.k:Stopping mds.k... 2024-04-25T00:47:58.819 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.k 2024-04-25T00:47:59.087 INFO:journalctl@ceph.mds.k.smithi121.stdout:Apr 25 00:47:58 smithi121 systemd[1]: Stopping Ceph mds.k for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:47:59.587 INFO:journalctl@ceph.mds.k.smithi121.stdout:Apr 25 00:47:59 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-k[61640]: 2024-04-25T00:47:59.121+0000 7f6b85ff8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.k -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:47:59.587 INFO:journalctl@ceph.mds.k.smithi121.stdout:Apr 25 00:47:59 smithi121 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-k[61640]: 2024-04-25T00:47:59.121+0000 7f6b85ff8700 -1 mds.k *** got signal Terminated *** 2024-04-25T00:48:01.780 INFO:journalctl@ceph.mds.k.smithi121.stdout:Apr 25 00:48:01 smithi121 podman[69706]: 2024-04-25 00:48:01.452554454 +0000 UTC m=+2.427024665 container died ced9fa557c2d2e29e3810afbf831a0099783c77d61ffcaef01f6d1aab59491d6 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-k, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-d2c45d9, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, io.buildah.version=1.33.5) 2024-04-25T00:48:02.054 INFO:journalctl@ceph.mds.k.smithi121.stdout:Apr 25 00:48:01 smithi121 podman[69706]: 2024-04-25 00:48:01.871872037 +0000 UTC m=+2.846342294 container cleanup ced9fa557c2d2e29e3810afbf831a0099783c77d61ffcaef01f6d1aab59491d6 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-k, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, RELEASE=squid-d2c45d9, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-25T00:48:02.054 INFO:journalctl@ceph.mds.k.smithi121.stdout:Apr 25 00:48:01 smithi121 bash[69706]: ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-k 2024-04-25T00:48:02.281 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.k.service' 2024-04-25T00:48:02.311 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:48:02.311 INFO:tasks.cephadm.mds.k:Stopped mds.k 2024-04-25T00:48:02.311 INFO:tasks.cephadm.mds.c:Stopping mds.c... 2024-04-25T00:48:02.311 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.c 2024-04-25T00:48:02.642 INFO:journalctl@ceph.mds.c.smithi130.stdout:Apr 25 00:48:02 smithi130 systemd[1]: Stopping Ceph mds.c for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:48:02.959 INFO:journalctl@ceph.mds.c.smithi130.stdout:Apr 25 00:48:02 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-c[63059]: 2024-04-25T00:48:02.684+0000 7fc3450f8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:48:02.960 INFO:journalctl@ceph.mds.c.smithi130.stdout:Apr 25 00:48:02 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-c[63059]: 2024-04-25T00:48:02.684+0000 7fc3450f8700 -1 mds.c *** got signal Terminated *** 2024-04-25T00:48:06.392 INFO:journalctl@ceph.mds.c.smithi130.stdout:Apr 25 00:48:06 smithi130 podman[70412]: 2024-04-25 00:48:06.089028452 +0000 UTC m=+3.574849893 container died 73fc15c8c09418ae0a61a9cf9a3803ef3dc18d5f91a6659d86e16fe0e2e2db3d (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-c, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-d2c45d9, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-04-25T00:48:06.892 INFO:journalctl@ceph.mds.c.smithi130.stdout:Apr 25 00:48:06 smithi130 podman[70412]: 2024-04-25 00:48:06.684642709 +0000 UTC m=+4.170464148 container cleanup 73fc15c8c09418ae0a61a9cf9a3803ef3dc18d5f91a6659d86e16fe0e2e2db3d (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-c, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-d2c45d9, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-25T00:48:06.892 INFO:journalctl@ceph.mds.c.smithi130.stdout:Apr 25 00:48:06 smithi130 bash[70412]: ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-c 2024-04-25T00:48:06.892 INFO:journalctl@ceph.mds.c.smithi130.stdout:Apr 25 00:48:06 smithi130 podman[70438]: 2024-04-25 00:48:06.835690965 +0000 UTC m=+0.496902634 container remove 73fc15c8c09418ae0a61a9cf9a3803ef3dc18d5f91a6659d86e16fe0e2e2db3d (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-c, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, RELEASE=squid-d2c45d9, org.label-schema.build-date=20240424, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-25T00:48:07.039 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.c.service' 2024-04-25T00:48:07.076 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:48:07.076 INFO:tasks.cephadm.mds.c:Stopped mds.c 2024-04-25T00:48:07.076 INFO:tasks.cephadm.mds.f:Stopping mds.f... 2024-04-25T00:48:07.077 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.f 2024-04-25T00:48:07.388 INFO:journalctl@ceph.mds.f.smithi130.stdout:Apr 25 00:48:07 smithi130 systemd[1]: Stopping Ceph mds.f for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:48:07.641 INFO:journalctl@ceph.mds.f.smithi130.stdout:Apr 25 00:48:07 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-f[62750]: 2024-04-25T00:48:07.387+0000 7f79b90f6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.f -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:48:07.641 INFO:journalctl@ceph.mds.f.smithi130.stdout:Apr 25 00:48:07 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-f[62750]: 2024-04-25T00:48:07.387+0000 7f79b90f6700 -1 mds.f *** got signal Terminated *** 2024-04-25T00:48:10.339 INFO:journalctl@ceph.mds.f.smithi130.stdout:Apr 25 00:48:10 smithi130 podman[70546]: 2024-04-25 00:48:10.082730021 +0000 UTC m=+2.794945773 container died 39daa494086d752c79c258faaf2b5852116a92d1f857a0cb1c66ce4a8ca8461e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-f, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, RELEASE=squid-d2c45d9, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-04-25T00:48:10.623 INFO:journalctl@ceph.mds.f.smithi130.stdout:Apr 25 00:48:10 smithi130 podman[70546]: 2024-04-25 00:48:10.421477646 +0000 UTC m=+3.133693405 container cleanup 39daa494086d752c79c258faaf2b5852116a92d1f857a0cb1c66ce4a8ca8461e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-f, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-d2c45d9, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True) 2024-04-25T00:48:10.623 INFO:journalctl@ceph.mds.f.smithi130.stdout:Apr 25 00:48:10 smithi130 bash[70546]: ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-f 2024-04-25T00:48:10.833 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.f.service' 2024-04-25T00:48:10.865 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:48:10.865 INFO:tasks.cephadm.mds.f:Stopped mds.f 2024-04-25T00:48:10.866 INFO:tasks.cephadm.mds.i:Stopping mds.i... 2024-04-25T00:48:10.866 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.i 2024-04-25T00:48:11.184 INFO:journalctl@ceph.mds.i.smithi130.stdout:Apr 25 00:48:10 smithi130 systemd[1]: Stopping Ceph mds.i for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:48:11.641 INFO:journalctl@ceph.mds.i.smithi130.stdout:Apr 25 00:48:11 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-i[63369]: 2024-04-25T00:48:11.183+0000 7f2dc7832700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.i -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:48:11.642 INFO:journalctl@ceph.mds.i.smithi130.stdout:Apr 25 00:48:11 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-i[63369]: 2024-04-25T00:48:11.183+0000 7f2dc7832700 -1 mds.i *** got signal Terminated *** 2024-04-25T00:48:14.391 INFO:journalctl@ceph.mds.i.smithi130.stdout:Apr 25 00:48:14 smithi130 podman[70669]: 2024-04-25 00:48:14.097488785 +0000 UTC m=+3.014256829 container died 687d1407a90a13bc57c3a8e86839b75b381d397cc1c2003f533cc5f67e872b20 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-i, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-d2c45d9, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-04-25T00:48:14.719 INFO:journalctl@ceph.mds.i.smithi130.stdout:Apr 25 00:48:14 smithi130 podman[70669]: 2024-04-25 00:48:14.492801371 +0000 UTC m=+3.409569424 container cleanup 687d1407a90a13bc57c3a8e86839b75b381d397cc1c2003f533cc5f67e872b20 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-i, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-d2c45d9, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2) 2024-04-25T00:48:14.719 INFO:journalctl@ceph.mds.i.smithi130.stdout:Apr 25 00:48:14 smithi130 bash[70669]: ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-i 2024-04-25T00:48:14.819 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.i.service' 2024-04-25T00:48:14.850 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:48:14.850 INFO:tasks.cephadm.mds.i:Stopped mds.i 2024-04-25T00:48:14.850 INFO:tasks.cephadm.mds.l:Stopping mds.l... 2024-04-25T00:48:14.850 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.l 2024-04-25T00:48:15.141 INFO:journalctl@ceph.mds.l.smithi130.stdout:Apr 25 00:48:14 smithi130 systemd[1]: Stopping Ceph mds.l for cb997156-0299-11ef-bc93-c7b262605968... 2024-04-25T00:48:15.642 INFO:journalctl@ceph.mds.l.smithi130.stdout:Apr 25 00:48:15 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-l[62426]: 2024-04-25T00:48:15.165+0000 7fa408d0d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.l -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T00:48:15.642 INFO:journalctl@ceph.mds.l.smithi130.stdout:Apr 25 00:48:15 smithi130 ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-l[62426]: 2024-04-25T00:48:15.165+0000 7fa408d0d700 -1 mds.l *** got signal Terminated *** 2024-04-25T00:48:18.780 INFO:journalctl@ceph.mds.l.smithi130.stdout:Apr 25 00:48:18 smithi130 podman[70792]: 2024-04-25 00:48:18.40392565 +0000 UTC m=+3.341327467 container died 62b6c422c9477910b984e324fdb53604e84d60c5f02fb8be002c1571b60f0f98 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-l, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=squid-d2c45d9, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True) 2024-04-25T00:48:19.081 INFO:journalctl@ceph.mds.l.smithi130.stdout:Apr 25 00:48:18 smithi130 podman[70792]: 2024-04-25 00:48:18.872141643 +0000 UTC m=+3.809543459 container cleanup 62b6c422c9477910b984e324fdb53604e84d60c5f02fb8be002c1571b60f0f98 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:a7481a6d0dca93e16a29c8acf1c964b318bd33c455aa5b4e4025b7a00d008901, name=ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-l, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, RELEASE=squid-d2c45d9, ceph=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T00:48:19.081 INFO:journalctl@ceph.mds.l.smithi130.stdout:Apr 25 00:48:18 smithi130 bash[70792]: ceph-cb997156-0299-11ef-bc93-c7b262605968-mds-l 2024-04-25T00:48:19.228 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-cb997156-0299-11ef-bc93-c7b262605968@mds.l.service' 2024-04-25T00:48:19.260 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T00:48:19.260 INFO:tasks.cephadm.mds.l:Stopped mds.l 2024-04-25T00:48:19.260 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid cb997156-0299-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-25T00:48:19.461 INFO:teuthology.orchestra.run.smithi032.stdout:Deleting cluster with fsid: cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:48:22.036 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid cb997156-0299-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-25T00:48:22.225 INFO:teuthology.orchestra.run.smithi121.stdout:Deleting cluster with fsid: cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:48:24.856 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid cb997156-0299-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-25T00:48:25.047 INFO:teuthology.orchestra.run.smithi130.stdout:Deleting cluster with fsid: cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T00:48:27.568 DEBUG:teuthology.orchestra.run.smithi032:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-25T00:48:27.601 DEBUG:teuthology.orchestra.run.smithi121:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-25T00:48:27.633 DEBUG:teuthology.orchestra.run.smithi130:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-25T00:48:27.658 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-25T00:48:27.660 DEBUG:teuthology.misc:Transferring archived files from smithi032:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-24_21:24:02-fs-squid-distro-default-smithi/7672261/remote/smithi032/crash 2024-04-25T00:48:27.661 DEBUG:teuthology.orchestra.run.smithi032:> sudo tar c -f - -C /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/crash -- . 2024-04-25T00:48:27.697 INFO:teuthology.orchestra.run.smithi032.stderr:tar: /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-25T00:48:27.697 INFO:teuthology.orchestra.run.smithi032.stderr:tar: Error is not recoverable: exiting now 2024-04-25T00:48:27.699 DEBUG:teuthology.misc:Transferring archived files from smithi121:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-24_21:24:02-fs-squid-distro-default-smithi/7672261/remote/smithi121/crash 2024-04-25T00:48:27.700 DEBUG:teuthology.orchestra.run.smithi121:> sudo tar c -f - -C /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/crash -- . 2024-04-25T00:48:27.742 INFO:teuthology.orchestra.run.smithi121.stderr:tar: /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-25T00:48:27.742 INFO:teuthology.orchestra.run.smithi121.stderr:tar: Error is not recoverable: exiting now 2024-04-25T00:48:27.744 DEBUG:teuthology.misc:Transferring archived files from smithi130:/var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-24_21:24:02-fs-squid-distro-default-smithi/7672261/remote/smithi130/crash 2024-04-25T00:48:27.745 DEBUG:teuthology.orchestra.run.smithi130:> sudo tar c -f - -C /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/crash -- . 2024-04-25T00:48:27.774 INFO:teuthology.orchestra.run.smithi130.stderr:tar: /var/lib/ceph/cb997156-0299-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-25T00:48:27.774 INFO:teuthology.orchestra.run.smithi130.stderr:tar: Error is not recoverable: exiting now 2024-04-25T00:48:27.776 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-25T00:48:27.776 DEBUG:teuthology.orchestra.run.smithi032:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v FS_DEGRADED | egrep -v FS_INLINE_DATA_DEPRECATED | egrep -v FS_WITH_FAILED_MDS | egrep -v MDS_ALL_DOWN | egrep -v MDS_DAMAGE | egrep -v MDS_DEGRADED | egrep -v MDS_FAILED | egrep -v MDS_INSUFFICIENT_STANDBY | egrep -v MDS_UP_LESS_THAN_MAX | egrep -v 'filesystem is online with fewer MDS than max_mds' | egrep -v POOL_APP_NOT_ENABLED | egrep -v 'overall HEALTH_' | egrep -v 'Replacing daemon' | egrep -v 'deprecated feature inline_data' | egrep -v 'overall HEALTH_' | egrep -v '\(OSD_DOWN\)' | egrep -v '\(OSD_' | egrep -v 'but it is still running' | egrep -v 'is not responding' | egrep -v MON_DOWN | egrep -v PG_AVAILABILITY | egrep -v PG_DEGRADED | egrep -v 'Reduced data availability' | egrep -v 'slow metadata IO' | egrep -v SLOW_OPS | egrep -v 'slow request' | head -n 1 2024-04-25T00:48:27.870 INFO:tasks.cephadm:Compressing logs... 2024-04-25T00:48:27.870 DEBUG:teuthology.orchestra.run.smithi032:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-25T00:48:27.873 DEBUG:teuthology.orchestra.run.smithi121:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-25T00:48:27.875 DEBUG:teuthology.orchestra.run.smithi130:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-25T00:48:27.908 INFO:teuthology.orchestra.run.smithi130.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-04-25T00:48:27.913 INFO:teuthology.orchestra.run.smithi130.stderr:gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.log 2024-04-25T00:48:27.913 INFO:teuthology.orchestra.run.smithi130.stderr:gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.2.log 2024-04-25T00:48:27.914 INFO:teuthology.orchestra.run.smithi130.stderr:gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.f.log 2024-04-25T00:48:27.914 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.log: /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-25T00:48:27.914 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.f.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.5.log 2024-04-25T00:48:27.914 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.c.log 2024-04-25T00:48:27.914 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mon.c.log 2024-04-25T00:48:27.914 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.c.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mgr.z.log 2024-04-25T00:48:27.914 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.i.log 2024-04-25T00:48:27.914 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mgr.z.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-25T00:48:27.914 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.i.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.11.log 2024-04-25T00:48:27.914 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.8.log 2024-04-25T00:48:27.914 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.11.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-25T00:48:27.914 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.8.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.l.log 2024-04-25T00:48:27.914 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-25T00:48:27.918 INFO:teuthology.orchestra.run.smithi121.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-04-25T00:48:27.918 INFO:teuthology.orchestra.run.smithi121.stderr:gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.log 2024-04-25T00:48:27.918 INFO:teuthology.orchestra.run.smithi121.stderr:gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-25T00:48:27.922 INFO:teuthology.orchestra.run.smithi121.stderr:gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.7.log 2024-04-25T00:48:27.922 INFO:teuthology.orchestra.run.smithi121.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.log: /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.b.log 2024-04-25T00:48:27.922 INFO:teuthology.orchestra.run.smithi121.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.4.log 2024-04-25T00:48:27.922 INFO:teuthology.orchestra.run.smithi121.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.b.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.1.log 2024-04-25T00:48:27.922 INFO:teuthology.orchestra.run.smithi121.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mgr.y.log 2024-04-25T00:48:27.923 INFO:teuthology.orchestra.run.smithi121.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.h.log 2024-04-25T00:48:27.923 INFO:teuthology.orchestra.run.smithi121.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.10.log 2024-04-25T00:48:27.923 INFO:teuthology.orchestra.run.smithi121.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.h.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-25T00:48:27.923 INFO:teuthology.orchestra.run.smithi121.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.10.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.e.log 2024-04-25T00:48:27.923 INFO:teuthology.orchestra.run.smithi121.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-25T00:48:27.923 INFO:teuthology.orchestra.run.smithi121.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.e.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mon.b.log 2024-04-25T00:48:27.923 INFO:teuthology.orchestra.run.smithi121.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.k.log 2024-04-25T00:48:27.923 INFO:teuthology.orchestra.run.smithi121.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mon.b.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-25T00:48:27.925 INFO:teuthology.orchestra.run.smithi032.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-04-25T00:48:27.925 INFO:teuthology.orchestra.run.smithi032.stderr:gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.log 2024-04-25T00:48:27.925 INFO:teuthology.orchestra.run.smithi032.stderr:gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-25T00:48:27.925 INFO:teuthology.orchestra.run.smithi032.stderr:gzip -5/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.log: --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.g.log 2024-04-25T00:48:27.925 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.j.log 2024-04-25T00:48:27.929 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.g.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.3.log 2024-04-25T00:48:27.929 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.j.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.0.log 2024-04-25T00:48:27.929 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mgr.x.log 2024-04-25T00:48:27.929 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mon.a.log 2024-04-25T00:48:27.929 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.6.log 2024-04-25T00:48:27.929 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-25T00:48:27.929 INFO:teuthology.orchestra.run.smithi032.stderr: 86.3% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-25T00:48:27.930 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.a.log 2024-04-25T00:48:27.930 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-25T00:48:27.930 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.a.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.d.log 2024-04-25T00:48:27.930 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.9.log 2024-04-25T00:48:27.930 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.d.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-25T00:48:27.930 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.9.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.log 2024-04-25T00:48:28.001 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.l.log: /var/log/ceph/cephadm.log: 83.7% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-25T00:48:28.019 INFO:teuthology.orchestra.run.smithi130.stderr: 92.1% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mgr.z.log.gz 2024-04-25T00:48:28.023 INFO:teuthology.orchestra.run.smithi130.stderr: 92.4% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-25T00:48:28.027 INFO:teuthology.orchestra.run.smithi121.stderr:/var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.k.log: /var/log/ceph/cephadm.log: 83.8% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-25T00:48:28.030 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/cephadm.log: /var/log/ceph/ceph-client.admin.log: 84.2% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-25T00:48:28.055 INFO:teuthology.orchestra.run.smithi130.stderr: 90.2% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-25T00:48:28.056 INFO:teuthology.orchestra.run.smithi032.stderr: 94.5% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.a.log.gz 2024-04-25T00:48:28.091 INFO:teuthology.orchestra.run.smithi032.stderr: 90.1% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-25T00:48:28.099 INFO:teuthology.orchestra.run.smithi121.stderr: 92.0% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mgr.y.log.gz 2024-04-25T00:48:28.100 INFO:teuthology.orchestra.run.smithi032.stderr: 92.7% -- replaced with /var/log/ceph/ceph-client.admin.log.gz 2024-04-25T00:48:28.110 INFO:teuthology.orchestra.run.smithi130.stderr: 94.5% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.l.log.gz 2024-04-25T00:48:28.121 INFO:teuthology.orchestra.run.smithi121.stderr: 86.3% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-25T00:48:28.123 INFO:teuthology.orchestra.run.smithi032.stderr: 93.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-25T00:48:28.125 INFO:teuthology.orchestra.run.smithi121.stderr: 90.2% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-25T00:48:28.128 INFO:teuthology.orchestra.run.smithi032.stderr: 94.4% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.g.log.gz 2024-04-25T00:48:28.128 INFO:teuthology.orchestra.run.smithi032.stderr: 94.6% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.j.log.gz 2024-04-25T00:48:28.137 INFO:teuthology.orchestra.run.smithi130.stderr: 86.3% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-25T00:48:28.144 INFO:teuthology.orchestra.run.smithi032.stderr: 94.5% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.d.log.gz 2024-04-25T00:48:28.170 INFO:teuthology.orchestra.run.smithi121.stderr: 92.1% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-25T00:48:28.173 INFO:teuthology.orchestra.run.smithi032.stderr: 91.1% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-25T00:48:28.201 INFO:teuthology.orchestra.run.smithi121.stderr: 91.1% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-25T00:48:28.207 INFO:teuthology.orchestra.run.smithi121.stderr: 94.4% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.k.log.gz 2024-04-25T00:48:28.225 INFO:teuthology.orchestra.run.smithi130.stderr: 91.1% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-25T00:48:29.165 INFO:teuthology.orchestra.run.smithi032.stderr: 90.3% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mgr.x.log.gz 2024-04-25T00:48:29.217 INFO:teuthology.orchestra.run.smithi130.stderr: 92.0% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mon.c.log.gz 2024-04-25T00:48:29.744 INFO:teuthology.orchestra.run.smithi121.stderr: 91.8% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mon.b.log.gz 2024-04-25T00:48:31.905 INFO:teuthology.orchestra.run.smithi130.stderr: 97.4% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.c.log.gz 2024-04-25T00:48:32.203 INFO:teuthology.orchestra.run.smithi032.stderr: 90.5% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mon.a.log.gz 2024-04-25T00:48:33.525 INFO:teuthology.orchestra.run.smithi130.stderr: 96.8% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.i.log.gz 2024-04-25T00:48:33.591 INFO:teuthology.orchestra.run.smithi121.stderr: 96.9% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.h.log.gz 2024-04-25T00:48:45.413 INFO:teuthology.orchestra.run.smithi130.stderr: 97.5% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.f.log.gz 2024-04-25T00:49:06.922 INFO:teuthology.orchestra.run.smithi121.stderr: 97.6% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.e.log.gz 2024-04-25T00:49:27.095 INFO:teuthology.orchestra.run.smithi130.stderr: 92.8% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.11.log.gz 2024-04-25T00:50:04.147 INFO:teuthology.orchestra.run.smithi121.stderr: 92.5% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.7.log.gz 2024-04-25T00:50:19.482 INFO:teuthology.orchestra.run.smithi130.stderr: 92.8% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.8.log.gz 2024-04-25T00:52:47.319 INFO:teuthology.orchestra.run.smithi121.stderr: 92.8% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.10.log.gz 2024-04-25T00:53:27.299 INFO:teuthology.orchestra.run.smithi130.stderr: 92.7% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.2.log.gz 2024-04-25T00:54:12.238 INFO:teuthology.orchestra.run.smithi130.stderr: 92.4% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.5.log.gz 2024-04-25T00:54:12.240 INFO:teuthology.orchestra.run.smithi130.stderr: 2024-04-25T00:54:12.240 INFO:teuthology.orchestra.run.smithi130.stderr:real 5m44.350s 2024-04-25T00:54:12.240 INFO:teuthology.orchestra.run.smithi130.stderr:user 5m7.888s 2024-04-25T00:54:12.240 INFO:teuthology.orchestra.run.smithi130.stderr:sys 0m12.524s 2024-04-25T00:54:34.120 INFO:teuthology.orchestra.run.smithi032.stderr: 92.6% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.3.log.gz 2024-04-25T00:55:41.985 INFO:teuthology.orchestra.run.smithi121.stderr: 92.9% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.1.log.gz 2024-04-25T00:56:13.016 INFO:teuthology.orchestra.run.smithi121.stderr: 95.3% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-mds.b.log.gz 2024-04-25T00:56:49.796 INFO:teuthology.orchestra.run.smithi121.stderr: 92.5% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.4.log.gz 2024-04-25T00:56:49.798 INFO:teuthology.orchestra.run.smithi121.stderr: 2024-04-25T00:56:49.798 INFO:teuthology.orchestra.run.smithi121.stderr:real 8m21.907s 2024-04-25T00:56:49.798 INFO:teuthology.orchestra.run.smithi121.stderr:user 5m58.036s 2024-04-25T00:56:49.799 INFO:teuthology.orchestra.run.smithi121.stderr:sys 0m15.625s 2024-04-25T01:00:46.063 INFO:teuthology.orchestra.run.smithi032.stderr: 92.7% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.0.log.gz 2024-04-25T01:02:02.119 INFO:teuthology.orchestra.run.smithi032.stderr: 92.5% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.6.log.gz 2024-04-25T01:02:25.381 INFO:teuthology.orchestra.run.smithi032.stderr: 92.4% -- replaced with /var/log/ceph/cb997156-0299-11ef-bc93-c7b262605968/ceph-osd.9.log.gz 2024-04-25T01:02:25.384 INFO:teuthology.orchestra.run.smithi032.stderr: 2024-04-25T01:02:25.384 INFO:teuthology.orchestra.run.smithi032.stderr:real 13m57.497s 2024-04-25T01:02:25.384 INFO:teuthology.orchestra.run.smithi032.stderr:user 6m34.041s 2024-04-25T01:02:25.384 INFO:teuthology.orchestra.run.smithi032.stderr:sys 0m17.318s 2024-04-25T01:02:25.384 INFO:tasks.cephadm:Archiving logs... 2024-04-25T01:02:25.385 DEBUG:teuthology.misc:Transferring archived files from smithi032:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-24_21:24:02-fs-squid-distro-default-smithi/7672261/remote/smithi032/log 2024-04-25T01:02:25.386 DEBUG:teuthology.orchestra.run.smithi032:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-25T01:03:27.166 DEBUG:teuthology.misc:Transferring archived files from smithi121:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-24_21:24:02-fs-squid-distro-default-smithi/7672261/remote/smithi121/log 2024-04-25T01:03:27.194 DEBUG:teuthology.orchestra.run.smithi121:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-25T01:04:26.731 DEBUG:teuthology.misc:Transferring archived files from smithi130:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-24_21:24:02-fs-squid-distro-default-smithi/7672261/remote/smithi130/log 2024-04-25T01:04:26.732 DEBUG:teuthology.orchestra.run.smithi130:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-25T01:05:16.130 INFO:tasks.cephadm:Removing cluster... 2024-04-25T01:05:16.131 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid cb997156-0299-11ef-bc93-c7b262605968 --force 2024-04-25T01:05:16.352 INFO:teuthology.orchestra.run.smithi032.stdout:Deleting cluster with fsid: cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T01:05:17.053 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid cb997156-0299-11ef-bc93-c7b262605968 --force 2024-04-25T01:05:17.344 INFO:teuthology.orchestra.run.smithi121.stdout:Deleting cluster with fsid: cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T01:05:18.023 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid cb997156-0299-11ef-bc93-c7b262605968 --force 2024-04-25T01:05:18.214 INFO:teuthology.orchestra.run.smithi130.stdout:Deleting cluster with fsid: cb997156-0299-11ef-bc93-c7b262605968 2024-04-25T01:05:18.849 INFO:tasks.cephadm:Removing cephadm ... 2024-04-25T01:05:18.850 DEBUG:teuthology.orchestra.run.smithi032:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-25T01:05:18.871 DEBUG:teuthology.orchestra.run.smithi121:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-25T01:05:18.903 DEBUG:teuthology.orchestra.run.smithi130:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-25T01:05:18.919 INFO:tasks.cephadm:Teardown complete 2024-04-25T01:05:18.919 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-04-25T01:05:18.930 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-04-25T01:05:18.931 DEBUG:teuthology.orchestra.run.smithi032:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-25T01:05:18.933 DEBUG:teuthology.orchestra.run.smithi121:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-25T01:05:18.946 DEBUG:teuthology.orchestra.run.smithi130:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-25T01:05:19.069 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-25T01:05:19.079 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-25T01:05:19.080 DEBUG:teuthology.orchestra.run.smithi032:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-25T01:05:19.082 DEBUG:teuthology.orchestra.run.smithi121:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-25T01:05:19.085 DEBUG:teuthology.orchestra.run.smithi130:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-25T01:05:19.097 INFO:teuthology.orchestra.run.smithi032.stderr:bash: line 1: ntpq: command not found 2024-04-25T01:05:19.098 INFO:teuthology.orchestra.run.smithi121.stderr:bash: line 1: ntpq: command not found 2024-04-25T01:05:19.098 INFO:teuthology.orchestra.run.smithi130.stderr:bash: line 1: ntpq: command not found 2024-04-25T01:05:19.108 INFO:teuthology.orchestra.run.smithi130.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-25T01:05:19.109 INFO:teuthology.orchestra.run.smithi130.stdout:=============================================================================== 2024-04-25T01:05:19.109 INFO:teuthology.orchestra.run.smithi130.stdout:^- hv01.front.sepia.ceph.com 2 6 377 5 +1026us[+1026us] +/- 575ms 2024-04-25T01:05:19.109 INFO:teuthology.orchestra.run.smithi130.stdout:^* hv02.front.sepia.ceph.com 3 6 377 6 -123us[ -111us] +/- 77ms 2024-04-25T01:05:19.109 INFO:teuthology.orchestra.run.smithi130.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 3 +134us[ +134us] +/- 86ms 2024-04-25T01:05:19.109 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T01:05:19.113 INFO:teuthology.orchestra.run.smithi121.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-25T01:05:19.113 INFO:teuthology.orchestra.run.smithi121.stdout:=============================================================================== 2024-04-25T01:05:19.113 INFO:teuthology.orchestra.run.smithi121.stdout:^- hv01.front.sepia.ceph.com 2 6 377 3 +1028us[+1038us] +/- 575ms 2024-04-25T01:05:19.114 INFO:teuthology.orchestra.run.smithi121.stdout:^* hv02.front.sepia.ceph.com 3 6 377 3 -134us[ -124us] +/- 77ms 2024-04-25T01:05:19.114 INFO:teuthology.orchestra.run.smithi121.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 2 +139us[ +139us] +/- 86ms 2024-04-25T01:05:19.114 INFO:teuthology.orchestra.run.smithi121.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T01:05:19.116 INFO:teuthology.orchestra.run.smithi032.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-25T01:05:19.116 INFO:teuthology.orchestra.run.smithi032.stdout:=============================================================================== 2024-04-25T01:05:19.116 INFO:teuthology.orchestra.run.smithi032.stdout:^- hv01.front.sepia.ceph.com 2 6 377 7 +1025us[+1039us] +/- 575ms 2024-04-25T01:05:19.116 INFO:teuthology.orchestra.run.smithi032.stdout:^* hv02.front.sepia.ceph.com 3 6 377 1 -120us[ -105us] +/- 77ms 2024-04-25T01:05:19.116 INFO:teuthology.orchestra.run.smithi032.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 7 +132us[ +146us] +/- 86ms 2024-04-25T01:05:19.116 INFO:teuthology.orchestra.run.smithi032.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T01:05:19.117 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-25T01:05:19.127 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-25T01:05:19.128 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-25T01:05:19.138 DEBUG:teuthology.orchestra.run.smithi032:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-25T01:05:19.429 DEBUG:teuthology.orchestra.run.smithi121:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-25T01:05:19.516 DEBUG:teuthology.orchestra.run.smithi130:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-25T01:05:19.592 DEBUG:teuthology.orchestra.run.smithi032:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-25T01:05:19.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-25T01:05:19.626 DEBUG:teuthology.orchestra.run.smithi121:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-25T01:05:19.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-25T01:05:19.660 DEBUG:teuthology.orchestra.run.smithi130:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-25T01:05:19.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-25T01:05:19.690 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-25T01:05:19.706 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-25T01:05:19.721 INFO:teuthology.task.internal:Duration was 2967.035114 seconds 2024-04-25T01:05:19.722 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-25T01:05:19.746 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-25T01:05:19.746 DEBUG:teuthology.orchestra.run.smithi032:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-25T01:05:19.750 DEBUG:teuthology.orchestra.run.smithi121:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-25T01:05:19.754 DEBUG:teuthology.orchestra.run.smithi130:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-25T01:05:19.802 INFO:teuthology.orchestra.run.smithi130.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-25T01:05:19.808 INFO:teuthology.orchestra.run.smithi032.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-25T01:05:19.815 INFO:teuthology.orchestra.run.smithi121.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-25T01:05:20.296 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-25T01:05:20.296 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi032.front.sepia.ceph.com 2024-04-25T01:05:20.297 DEBUG:teuthology.orchestra.run.smithi032:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-25T01:05:20.320 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi121.front.sepia.ceph.com 2024-04-25T01:05:20.320 DEBUG:teuthology.orchestra.run.smithi121:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-25T01:05:20.347 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi130.front.sepia.ceph.com 2024-04-25T01:05:20.347 DEBUG:teuthology.orchestra.run.smithi130:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-25T01:05:20.381 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-25T01:05:20.381 DEBUG:teuthology.orchestra.run.smithi032:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-25T01:05:20.384 DEBUG:teuthology.orchestra.run.smithi121:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-25T01:05:20.390 DEBUG:teuthology.orchestra.run.smithi130:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-25T01:05:20.450 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-25T01:05:20.450 DEBUG:teuthology.orchestra.run.smithi032:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-25T01:05:20.452 DEBUG:teuthology.orchestra.run.smithi121:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-25T01:05:20.459 DEBUG:teuthology.orchestra.run.smithi130:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-25T01:05:20.839 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-25T01:05:20.852 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-25T01:05:20.852 DEBUG:teuthology.orchestra.run.smithi032:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-25T01:05:20.876 DEBUG:teuthology.orchestra.run.smithi121:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-25T01:05:20.901 DEBUG:teuthology.orchestra.run.smithi130:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-25T01:05:20.947 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-25T01:05:20.958 DEBUG:teuthology.orchestra.run.smithi032:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-25T01:05:20.961 DEBUG:teuthology.orchestra.run.smithi121:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-25T01:05:20.963 DEBUG:teuthology.orchestra.run.smithi130:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-25T01:05:20.985 INFO:teuthology.orchestra.run.smithi130.stdout:kernel.core_pattern = core 2024-04-25T01:05:20.993 INFO:teuthology.orchestra.run.smithi121.stdout:kernel.core_pattern = core 2024-04-25T01:05:21.006 INFO:teuthology.orchestra.run.smithi032.stdout:kernel.core_pattern = core 2024-04-25T01:05:21.039 DEBUG:teuthology.orchestra.run.smithi032:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-25T01:05:21.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-25T01:05:21.054 DEBUG:teuthology.orchestra.run.smithi121:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-25T01:05:21.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-25T01:05:21.092 DEBUG:teuthology.orchestra.run.smithi130:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-25T01:05:21.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-25T01:05:21.106 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-25T01:05:21.118 INFO:teuthology.task.internal:Transferring archived files... 2024-04-25T01:05:21.120 DEBUG:teuthology.misc:Transferring archived files from smithi032:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-24_21:24:02-fs-squid-distro-default-smithi/7672261/remote/smithi032 2024-04-25T01:05:21.120 DEBUG:teuthology.orchestra.run.smithi032:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-25T01:05:21.162 DEBUG:teuthology.misc:Transferring archived files from smithi121:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-24_21:24:02-fs-squid-distro-default-smithi/7672261/remote/smithi121 2024-04-25T01:05:21.163 DEBUG:teuthology.orchestra.run.smithi121:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-25T01:05:21.197 DEBUG:teuthology.misc:Transferring archived files from smithi130:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-24_21:24:02-fs-squid-distro-default-smithi/7672261/remote/smithi130 2024-04-25T01:05:21.197 DEBUG:teuthology.orchestra.run.smithi130:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-25T01:05:21.234 INFO:teuthology.task.internal:Removing archive directory... 2024-04-25T01:05:21.234 DEBUG:teuthology.orchestra.run.smithi032:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-25T01:05:21.237 DEBUG:teuthology.orchestra.run.smithi121:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-25T01:05:21.239 DEBUG:teuthology.orchestra.run.smithi130:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-25T01:05:21.282 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-25T01:05:21.296 INFO:teuthology.task.internal:Not uploading archives. 2024-04-25T01:05:21.296 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-25T01:05:21.310 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-25T01:05:21.310 DEBUG:teuthology.orchestra.run.smithi032:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-25T01:05:21.312 DEBUG:teuthology.orchestra.run.smithi121:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-25T01:05:21.315 DEBUG:teuthology.orchestra.run.smithi130:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-25T01:05:21.327 INFO:teuthology.orchestra.run.smithi032.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 25 01:05 /home/ubuntu/cephtest 2024-04-25T01:05:21.327 INFO:teuthology.orchestra.run.smithi121.stdout: 262161 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 25 01:05 /home/ubuntu/cephtest 2024-04-25T01:05:21.337 INFO:teuthology.orchestra.run.smithi130.stdout: 262161 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 25 01:05 /home/ubuntu/cephtest 2024-04-25T01:05:21.338 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-25T01:05:21.351 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-25T01:05:21.438 INFO:teuthology.nuke:Checking targets against current locks 2024-04-25T01:05:21.467 DEBUG:teuthology.nuke:shortname: smithi032 2024-04-25T01:05:21.468 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-25T01:05:21.491 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi032.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-24_21:24:02-fs-squid-distro-default-smithi/7672261', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-25 00:10:23.147174', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBD7DrckUBqtLVOh4N21KpzV2c5DYCQ4bNPwoz1xlRWR3s5dZJe2UHzqCWVTXjf2RXCo+Uax58IeB8SHs1nwGyDQ='} 2024-04-25T01:05:21.495 DEBUG:teuthology.nuke:shortname: smithi121 2024-04-25T01:05:21.495 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-25T01:05:21.522 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi121.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-24_21:24:02-fs-squid-distro-default-smithi/7672261', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-25 00:10:23.149934', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDc3ni7OjyBoF/W+5fbYLxj3kDcEq+yuFnaEXYGj35ajB7s1IGOeGWTpkBzEK3HH72BAhhtYvuA9ng2PWcevtHA='} 2024-04-25T01:05:21.523 DEBUG:teuthology.nuke:shortname: smithi130 2024-04-25T01:05:21.523 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-25T01:05:21.549 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi130.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-24_21:24:02-fs-squid-distro-default-smithi/7672261', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-25 00:10:23.148903', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJEqQth7lAp0ElBkxSulfTqJDM8G2Xx2ESozNk909PMdxh3PD482xBapBNkUUkg7JMf7+qYQSeTk7wRTvFInNGw='} 2024-04-25T01:05:21.555 INFO:teuthology.orchestra.console.smithi032:Power off 2024-04-25T01:05:21.556 DEBUG:teuthology.orchestra.console.smithi032:pexpect command: ipmitool -H smithi032.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-25T01:05:21.580 DEBUG:teuthology.orchestra.console.smithi032:power off output: Chassis Power Control: Down/Off 2024-04-25T01:05:21.581 DEBUG:teuthology.orchestra.console.smithi032:pexpect command: ipmitool -H smithi032.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-25T01:05:21.597 INFO:teuthology.orchestra.console.smithi121:Power off 2024-04-25T01:05:21.597 DEBUG:teuthology.orchestra.console.smithi121:pexpect command: ipmitool -H smithi121.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-25T01:05:21.612 DEBUG:teuthology.orchestra.console.smithi032:check power output: Chassis Power is on 2024-04-25T01:05:21.622 DEBUG:teuthology.orchestra.console.smithi121:power off output: Chassis Power Control: Down/Off 2024-04-25T01:05:21.622 DEBUG:teuthology.orchestra.console.smithi121:pexpect command: ipmitool -H smithi121.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-25T01:05:21.642 INFO:teuthology.orchestra.console.smithi130:Power off 2024-04-25T01:05:21.642 DEBUG:teuthology.orchestra.console.smithi130:pexpect command: ipmitool -H smithi130.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-25T01:05:21.652 DEBUG:teuthology.orchestra.console.smithi121:check power output: Chassis Power is on 2024-04-25T01:05:21.665 DEBUG:teuthology.orchestra.console.smithi130:power off output: Chassis Power Control: Down/Off 2024-04-25T01:05:21.665 DEBUG:teuthology.orchestra.console.smithi130:pexpect command: ipmitool -H smithi130.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-25T01:05:21.728 DEBUG:teuthology.orchestra.console.smithi130:check power output: Chassis Power is on 2024-04-25T01:05:25.613 DEBUG:teuthology.orchestra.console.smithi032:pexpect command: ipmitool -H smithi032.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-25T01:05:25.653 DEBUG:teuthology.orchestra.console.smithi121:pexpect command: ipmitool -H smithi121.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-25T01:05:25.723 DEBUG:teuthology.orchestra.console.smithi032:check power output: Chassis Power is on 2024-04-25T01:05:25.729 DEBUG:teuthology.orchestra.console.smithi130:pexpect command: ipmitool -H smithi130.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-25T01:05:25.763 DEBUG:teuthology.orchestra.console.smithi121:check power output: Chassis Power is on 2024-04-25T01:05:25.840 DEBUG:teuthology.orchestra.console.smithi130:check power output: Chassis Power is on 2024-04-25T01:05:29.728 DEBUG:teuthology.orchestra.console.smithi032:pexpect command: ipmitool -H smithi032.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-25T01:05:29.764 DEBUG:teuthology.orchestra.console.smithi121:pexpect command: ipmitool -H smithi121.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-25T01:05:29.838 DEBUG:teuthology.orchestra.console.smithi032:check power output: Chassis Power is off 2024-04-25T01:05:29.841 DEBUG:teuthology.orchestra.console.smithi130:pexpect command: ipmitool -H smithi130.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-25T01:05:29.876 DEBUG:teuthology.orchestra.console.smithi121:check power output: Chassis Power is off 2024-04-25T01:05:29.939 INFO:teuthology.orchestra.console.smithi032:Power off completed 2024-04-25T01:05:29.950 DEBUG:teuthology.orchestra.console.smithi130:check power output: Chassis Power is on 2024-04-25T01:05:29.977 INFO:teuthology.orchestra.console.smithi121:Power off completed 2024-04-25T01:05:33.954 DEBUG:teuthology.orchestra.console.smithi130:pexpect command: ipmitool -H smithi130.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-25T01:05:34.064 DEBUG:teuthology.orchestra.console.smithi130:check power output: Chassis Power is off 2024-04-25T01:05:34.165 INFO:teuthology.orchestra.console.smithi130:Power off completed 2024-04-25T01:05:34.270 INFO:teuthology.run:Summary data: description: fs/workload/{0-centos_9.stream begin/{0-install 1-cephadm 2-logrotate 3-modules} clusters/1a11s-mds-1c-client-3node conf/{client mds mgr mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/stock/{centos_9.stream k-stock} ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts pg_health session_timeout} ranks/multi/{balancer/automatic export-check n/3 replication/default} standby-replay tasks/{0-subvolume/{with-no-extra-options} 1-check-counter 2-scrub/yes 3-snaps/yes 4-flush/no 5-workunit/suites/iogen}} duration: 2967.0351140499115 failure_reason: 'error during scrub thrashing: rank damage found: {''backtrace''}' flavor: default owner: scheduled_teuthology@teuthology status: fail success: false 2024-04-25T01:05:34.271 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-25T01:05:34.356 INFO:teuthology.run:FAIL